{
      "annotations": {
         "list": [ ]
      },
      "editable": true,
      "gnetId": null,
      "graphTooltip": 0,
      "hideControls": false,
      "links": [
         {
            "asDropdown": true,
            "icon": "external link",
            "includeVars": true,
            "keepTime": true,
            "tags": [
               "loki"
            ],
            "targetBlank": false,
            "title": "Loki Dashboards",
            "type": "dashboards"
         }
      ],
      "refresh": "10s",
      "rows": [
         {
            "collapse": false,
            "height": "250px",
            "panels": [
               {
                  "aliasColors": { },
                  "bars": false,
                  "dashLength": 10,
                  "dashes": false,
                  "datasource": "$datasource",
                  "fill": 1,
                  "id": 1,
                  "legend": {
                     "avg": false,
                     "current": false,
                     "max": false,
                     "min": false,
                     "show": true,
                     "total": false,
                     "values": false
                  },
                  "lines": true,
                  "linewidth": 1,
                  "links": [ ],
                  "nullPointMode": "null as zero",
                  "percentage": false,
                  "pointradius": 5,
                  "points": false,
                  "renderer": "flot",
                  "seriesOverrides": [
                     {
                        "alias": "limit",
                        "color": "#E02F44",
                        "fill": 0
                     }
                  ],
                  "spaceLength": 10,
                  "span": 4,
                  "stack": false,
                  "steppedLine": false,
                  "targets": [
                     {
                        "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"}[$__rate_interval]))",
                        "format": "time_series",
                        "intervalFactor": 2,
                        "legendFormat": "{{pod}}",
                        "legendLink": null,
                        "step": 10
                     },
                     {
                        "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"})",
                        "format": "time_series",
                        "intervalFactor": 2,
                        "legendFormat": "limit",
                        "legendLink": null,
                        "step": 10
                     }
                  ],
                  "thresholds": [ ],
                  "timeFrom": null,
                  "timeShift": null,
                  "title": "CPU",
                  "tooltip": {
                     "sort": 2
                  },
                  "type": "graph",
                  "xaxis": {
                     "buckets": null,
                     "mode": "time",
                     "name": null,
                     "show": true,
                     "values": [ ]
                  },
                  "yaxes": [
                     {
                        "format": "short",
                        "label": null,
                        "logBase": 1,
                        "max": null,
                        "min": 0,
                        "show": true
                     },
                     {
                        "format": "short",
                        "label": null,
                        "logBase": 1,
                        "max": null,
                        "min": null,
                        "show": false
                     }
                  ]
               },
               {
                  "aliasColors": { },
                  "bars": false,
                  "dashLength": 10,
                  "dashes": false,
                  "datasource": "$datasource",
                  "fill": 1,
                  "id": 2,
                  "legend": {
                     "avg": false,
                     "current": false,
                     "max": false,
                     "min": false,
                     "show": true,
                     "total": false,
                     "values": false
                  },
                  "lines": true,
                  "linewidth": 1,
                  "links": [ ],
                  "nullPointMode": "null as zero",
                  "percentage": false,
                  "pointradius": 5,
                  "points": false,
                  "renderer": "flot",
                  "seriesOverrides": [
                     {
                        "alias": "limit",
                        "color": "#E02F44",
                        "fill": 0
                     }
                  ],
                  "spaceLength": 10,
                  "span": 4,
                  "stack": false,
                  "steppedLine": false,
                  "targets": [
                     {
                        "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"})",
                        "format": "time_series",
                        "intervalFactor": 2,
                        "legendFormat": "{{pod}}",
                        "legendLink": null,
                        "step": 10
                     },
                     {
                        "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"} > 0)",
                        "format": "time_series",
                        "intervalFactor": 2,
                        "legendFormat": "limit",
                        "legendLink": null,
                        "step": 10
                     }
                  ],
                  "thresholds": [ ],
                  "timeFrom": null,
                  "timeShift": null,
                  "title": "Memory (workingset)",
                  "tooltip": {
                     "sort": 2
                  },
                  "type": "graph",
                  "xaxis": {
                     "buckets": null,
                     "mode": "time",
                     "name": null,
                     "show": true,
                     "values": [ ]
                  },
                  "yaxes": [
                     {
                        "format": "bytes",
                        "label": null,
                        "logBase": 1,
                        "max": null,
                        "min": 0,
                        "show": true
                     },
                     {
                        "format": "short",
                        "label": null,
                        "logBase": 1,
                        "max": null,
                        "min": null,
                        "show": false
                     }
                  ]
               },
               {
                  "aliasColors": { },
                  "bars": false,
                  "dashLength": 10,
                  "dashes": false,
                  "datasource": "$datasource",
                  "fill": 1,
                  "id": 3,
                  "legend": {
                     "avg": false,
                     "current": false,
                     "max": false,
                     "min": false,
                     "show": true,
                     "total": false,
                     "values": false
                  },
                  "lines": true,
                  "linewidth": 1,
                  "links": [ ],
                  "nullPointMode": "null as zero",
                  "percentage": false,
                  "pointradius": 5,
                  "points": false,
                  "renderer": "flot",
                  "seriesOverrides": [ ],
                  "spaceLength": 10,
                  "span": 4,
                  "stack": false,
                  "steppedLine": false,
                  "targets": [
                     {
                        "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-read\"})",
                        "format": "time_series",
                        "intervalFactor": 2,
                        "legendFormat": "{{pod}}",
                        "legendLink": null,
                        "step": 10
                     }
                  ],
                  "thresholds": [ ],
                  "timeFrom": null,
                  "timeShift": null,
                  "title": "Memory (go heap inuse)",
                  "tooltip": {
                     "sort": 2
                  },
                  "type": "graph",
                  "xaxis": {
                     "buckets": null,
                     "mode": "time",
                     "name": null,
                     "show": true,
                     "values": [ ]
                  },
                  "yaxes": [
                     {
                        "format": "bytes",
                        "label": null,
                        "logBase": 1,
                        "max": null,
                        "min": 0,
                        "show": true
                     },
                     {
                        "format": "short",
                        "label": null,
                        "logBase": 1,
                        "max": null,
                        "min": null,
                        "show": false
                     }
                  ]
               }
            ],
            "repeat": null,
            "repeatIteration": null,
            "repeatRowId": null,
            "showTitle": true,
            "title": "Resource Usage",
            "titleSize": "h6"
         },
         {
            "collapse": false,
            "height": "250px",
            "panels": [
               {
                  "aliasColors": { },
                  "bars": false,
                  "dashLength": 10,
                  "dashes": false,
                  "datasource": "$datasource",
                  "fieldConfig": {
                     "defaults": {
                        "color": {
                           "fixedColor": "blue",
                           "mode": "fixed"
                        },
                        "custom": { },
                        "thresholds": {
                           "mode": "absolute",
                           "steps": [
                              {
                                 "color": "green",
                                 "value": null
                              }
                           ]
                        },
                        "unit": "dateTimeFromNow"
                     }
                  },
                  "fill": 1,
                  "id": 4,
                  "legend": {
                     "avg": false,
                     "current": false,
                     "max": false,
                     "min": false,
                     "show": true,
                     "total": false,
                     "values": false
                  },
                  "lines": true,
                  "linewidth": 1,
                  "links": [ ],
                  "nullPointMode": "null as zero",
                  "options": {
                     "colorMode": "value",
                     "graphMode": "area",
                     "justifyMode": "auto",
                     "orientation": "auto",
                     "reduceOptions": {
                        "calcs": [
                           "lastNotNull"
                        ],
                        "fields": "",
                        "values": false
                     },
                     "text": { },
                     "textMode": "auto"
                  },
                  "percentage": false,
                  "pointradius": 5,
                  "points": false,
                  "renderer": "flot",
                  "seriesOverrides": [ ],
                  "spaceLength": 10,
                  "span": 4,
                  "stack": false,
                  "steppedLine": false,
                  "targets": [
                     {
                        "expr": "loki_boltdb_shipper_compact_tables_operation_last_successful_run_timestamp_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"} * 1e3",
                        "format": "time_series",
                        "instant": true,
                        "refId": "A"
                     }
                  ],
                  "thresholds": [ ],
                  "timeFrom": null,
                  "timeShift": null,
                  "title": "Last Compact and Mark Operation Success",
                  "tooltip": {
                     "shared": true,
                     "sort": 2,
                     "value_type": "individual"
                  },
                  "type": "stat",
                  "xaxis": {
                     "buckets": null,
                     "mode": "time",
                     "name": null,
                     "show": true,
                     "values": [ ]
                  },
                  "yaxes": [
                     {
                        "format": "short",
                        "label": null,
                        "logBase": 1,
                        "max": null,
                        "min": 0,
                        "show": true
                     },
                     {
                        "format": "short",
                        "label": null,
                        "logBase": 1,
                        "max": null,
                        "min": null,
                        "show": false
                     }
                  ]
               },
               {
                  "aliasColors": { },
                  "bars": false,
                  "dashLength": 10,
                  "dashes": false,
                  "datasource": "$datasource",
                  "fill": 1,
                  "id": 5,
                  "legend": {
                     "avg": false,
                     "current": false,
                     "max": false,
                     "min": false,
                     "show": true,
                     "total": false,
                     "values": false
                  },
                  "lines": true,
                  "linewidth": 1,
                  "links": [ ],
                  "nullPointMode": "null as zero",
                  "percentage": false,
                  "pointradius": 5,
                  "points": false,
                  "renderer": "flot",
                  "seriesOverrides": [ ],
                  "spaceLength": 10,
                  "span": 4,
                  "stack": false,
                  "steppedLine": false,
                  "targets": [
                     {
                        "expr": "loki_boltdb_shipper_compact_tables_operation_duration_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"}",
                        "format": "time_series",
                        "intervalFactor": 2,
                        "legendFormat": "duration",
                        "legendLink": null,
                        "step": 10
                     }
                  ],
                  "thresholds": [ ],
                  "timeFrom": null,
                  "timeShift": null,
                  "title": "Compact and Mark Operations Duration",
                  "tooltip": {
                     "shared": true,
                     "sort": 2,
                     "value_type": "individual"
                  },
                  "type": "graph",
                  "xaxis": {
                     "buckets": null,
                     "mode": "time",
                     "name": null,
                     "show": true,
                     "values": [ ]
                  },
                  "yaxes": [
                     {
                        "format": "s",
                        "label": null,
                        "logBase": 1,
                        "max": null,
                        "min": 0,
                        "show": true
                     },
                     {
                        "format": "short",
                        "label": null,
                        "logBase": 1,
                        "max": null,
                        "min": null,
                        "show": false
                     }
                  ]
               },
               {
                  "aliasColors": { },
                  "bars": false,
                  "dashLength": 10,
                  "dashes": false,
                  "datasource": "$datasource",
                  "fill": 1,
                  "id": 6,
                  "legend": {
                     "avg": false,
                     "current": false,
                     "max": false,
                     "min": false,
                     "show": true,
                     "total": false,
                     "values": false
                  },
                  "lines": true,
                  "linewidth": 1,
                  "links": [ ],
                  "nullPointMode": "null as zero",
                  "percentage": false,
                  "pointradius": 5,
                  "points": false,
                  "renderer": "flot",
                  "seriesOverrides": [ ],
                  "spaceLength": 10,
                  "span": 4,
                  "stack": false,
                  "steppedLine": false,
                  "targets": [
                     {
                        "expr": "sum by (status)(rate(loki_boltdb_shipper_compact_tables_operation_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))",
                        "format": "time_series",
                        "intervalFactor": 2,
                        "legendFormat": "{{success}}",
                        "legendLink": null,
                        "step": 10
                     }
                  ],
                  "thresholds": [ ],
                  "timeFrom": null,
                  "timeShift": null,
                  "title": "Compact and Mark Operations Per Status",
                  "tooltip": {
                     "shared": true,
                     "sort": 2,
                     "value_type": "individual"
                  },
                  "type": "graph",
                  "xaxis": {
                     "buckets": null,
                     "mode": "time",
                     "name": null,
                     "show": true,
                     "values": [ ]
                  },
                  "yaxes": [
                     {
                        "format": "short",
                        "label": null,
                        "logBase": 1,
                        "max": null,
                        "min": 0,
                        "show": true
                     },
                     {
                        "format": "short",
                        "label": null,
                        "logBase": 1,
                        "max": null,
                        "min": null,
                        "show": false
                     }
                  ]
               }
            ],
            "repeat": null,
            "repeatIteration": null,
            "repeatRowId": null,
            "showTitle": true,
            "title": "Compact and Mark",
            "titleSize": "h6"
         },
         {
            "collapse": false,
            "height": "250px",
            "panels": [
               {
                  "aliasColors": { },
                  "bars": false,
                  "dashLength": 10,
                  "dashes": false,
                  "datasource": "$datasource",
                  "fill": 10,
                  "id": 7,
                  "legend": {
                     "avg": false,
                     "current": false,
                     "max": false,
                     "min": false,
                     "show": true,
                     "total": false,
                     "values": false
                  },
                  "lines": true,
                  "linewidth": 0,
                  "links": [ ],
                  "nullPointMode": "null as zero",
                  "percentage": false,
                  "pointradius": 5,
                  "points": false,
                  "renderer": "flot",
                  "seriesOverrides": [ ],
                  "spaceLength": 10,
                  "span": 4,
                  "stack": true,
                  "steppedLine": false,
                  "targets": [
                     {
                        "expr": "count by(action)(loki_boltdb_shipper_retention_marker_table_processed_total{cluster=~\"$cluster\", namespace=~\"$namespace\"})",
                        "format": "time_series",
                        "intervalFactor": 2,
                        "legendFormat": "{{action}}",
                        "legendLink": null,
                        "step": 10
                     }
                  ],
                  "thresholds": [ ],
                  "timeFrom": null,
                  "timeShift": null,
                  "title": "Processed Tables Per Action",
                  "tooltip": {
                     "shared": true,
                     "sort": 2,
                     "value_type": "individual"
                  },
                  "type": "graph",
                  "xaxis": {
                     "buckets": null,
                     "mode": "time",
                     "name": null,
                     "show": true,
                     "values": [ ]
                  },
                  "yaxes": [
                     {
                        "format": "short",
                        "label": null,
                        "logBase": 1,
                        "max": null,
                        "min": 0,
                        "show": true
                     },
                     {
                        "format": "short",
                        "label": null,
                        "logBase": 1,
                        "max": null,
                        "min": null,
                        "show": false
                     }
                  ]
               },
               {
                  "aliasColors": { },
                  "bars": false,
                  "dashLength": 10,
                  "dashes": false,
                  "datasource": "$datasource",
                  "fill": 10,
                  "id": 8,
                  "legend": {
                     "avg": false,
                     "current": false,
                     "max": false,
                     "min": false,
                     "show": true,
                     "total": false,
                     "values": false
                  },
                  "lines": true,
                  "linewidth": 0,
                  "links": [ ],
                  "nullPointMode": "null as zero",
                  "percentage": false,
                  "pointradius": 5,
                  "points": false,
                  "renderer": "flot",
                  "seriesOverrides": [ ],
                  "spaceLength": 10,
                  "span": 4,
                  "stack": true,
                  "steppedLine": false,
                  "targets": [
                     {
                        "expr": "count by(table,action)(loki_boltdb_shipper_retention_marker_table_processed_total{cluster=~\"$cluster\", namespace=~\"$namespace\" , action=~\"modified|deleted\"})",
                        "format": "time_series",
                        "intervalFactor": 2,
                        "legendFormat": "{{table}}-{{action}}",
                        "legendLink": null,
                        "step": 10
                     }
                  ],
                  "thresholds": [ ],
                  "timeFrom": null,
                  "timeShift": null,
                  "title": "Modified Tables",
                  "tooltip": {
                     "shared": true,
                     "sort": 2,
                     "value_type": "individual"
                  },
                  "type": "graph",
                  "xaxis": {
                     "buckets": null,
                     "mode": "time",
                     "name": null,
                     "show": true,
                     "values": [ ]
                  },
                  "yaxes": [
                     {
                        "format": "short",
                        "label": null,
                        "logBase": 1,
                        "max": null,
                        "min": 0,
                        "show": true
                     },
                     {
                        "format": "short",
                        "label": null,
                        "logBase": 1,
                        "max": null,
                        "min": null,
                        "show": false
                     }
                  ]
               },
               {
                  "aliasColors": { },
                  "bars": false,
                  "dashLength": 10,
                  "dashes": false,
                  "datasource": "$datasource",
                  "fill": 10,
                  "id": 9,
                  "legend": {
                     "avg": false,
                     "current": false,
                     "max": false,
                     "min": false,
                     "show": true,
                     "total": false,
                     "values": false
                  },
                  "lines": true,
                  "linewidth": 0,
                  "links": [ ],
                  "nullPointMode": "null as zero",
                  "percentage": false,
                  "pointradius": 5,
                  "points": false,
                  "renderer": "flot",
                  "seriesOverrides": [ ],
                  "spaceLength": 10,
                  "span": 4,
                  "stack": true,
                  "steppedLine": false,
                  "targets": [
                     {
                        "expr": "sum by (table)(rate(loki_boltdb_shipper_retention_marker_count_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) >0",
                        "format": "time_series",
                        "intervalFactor": 2,
                        "legendFormat": "{{table}}",
                        "legendLink": null,
                        "step": 10
                     }
                  ],
                  "thresholds": [ ],
                  "timeFrom": null,
                  "timeShift": null,
                  "title": "Marks Creation Rate Per Table",
                  "tooltip": {
                     "shared": true,
                     "sort": 2,
                     "value_type": "individual"
                  },
                  "type": "graph",
                  "xaxis": {
                     "buckets": null,
                     "mode": "time",
                     "name": null,
                     "show": true,
                     "values": [ ]
                  },
                  "yaxes": [
                     {
                        "format": "short",
                        "label": null,
                        "logBase": 1,
                        "max": null,
                        "min": 0,
                        "show": true
                     },
                     {
                        "format": "short",
                        "label": null,
                        "logBase": 1,
                        "max": null,
                        "min": null,
                        "show": false
                     }
                  ]
               }
            ],
            "repeat": null,
            "repeatIteration": null,
            "repeatRowId": null,
            "showTitle": true,
            "title": "Per Table Marker",
            "titleSize": "h6"
         },
         {
            "collapse": false,
            "height": "250px",
            "panels": [
               {
                  "aliasColors": { },
                  "bars": false,
                  "dashLength": 10,
                  "dashes": false,
                  "datasource": "$datasource",
                  "fill": 1,
                  "format": "short",
                  "id": 10,
                  "legend": {
                     "avg": false,
                     "current": false,
                     "max": false,
                     "min": false,
                     "show": true,
                     "total": false,
                     "values": false
                  },
                  "lines": true,
                  "linewidth": 1,
                  "links": [ ],
                  "nullPointMode": "null as zero",
                  "percentage": false,
                  "pointradius": 5,
                  "points": false,
                  "renderer": "flot",
                  "seriesOverrides": [ ],
                  "spaceLength": 10,
                  "span": 6,
                  "stack": false,
                  "steppedLine": false,
                  "targets": [
                     {
                        "expr": "sum (increase(loki_boltdb_shipper_retention_marker_count_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[24h]))",
                        "format": "time_series",
                        "instant": true,
                        "intervalFactor": 2,
                        "refId": "A"
                     }
                  ],
                  "thresholds": "70,80",
                  "timeFrom": null,
                  "timeShift": null,
                  "title": "Marked Chunks (24h)",
                  "tooltip": {
                     "shared": true,
                     "sort": 2,
                     "value_type": "individual"
                  },
                  "type": "singlestat",
                  "xaxis": {
                     "buckets": null,
                     "mode": "time",
                     "name": null,
                     "show": true,
                     "values": [ ]
                  },
                  "yaxes": [
                     {
                        "format": "short",
                        "label": null,
                        "logBase": 1,
                        "max": null,
                        "min": 0,
                        "show": true
                     },
                     {
                        "format": "short",
                        "label": null,
                        "logBase": 1,
                        "max": null,
                        "min": null,
                        "show": false
                     }
                  ]
               },
               {
                  "aliasColors": { },
                  "bars": false,
                  "dashLength": 10,
                  "dashes": false,
                  "datasource": "$datasource",
                  "fill": 1,
                  "id": 11,
                  "legend": {
                     "avg": false,
                     "current": false,
                     "max": false,
                     "min": false,
                     "show": true,
                     "total": false,
                     "values": false
                  },
                  "lines": true,
                  "linewidth": 1,
                  "links": [ ],
                  "nullPointMode": "null as zero",
                  "percentage": false,
                  "pointradius": 5,
                  "points": false,
                  "renderer": "flot",
                  "seriesOverrides": [ ],
                  "spaceLength": 10,
                  "span": 6,
                  "stack": false,
                  "steppedLine": false,
                  "targets": [
                     {
                        "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_retention_marker_table_processed_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (le)) * 1e3",
                        "format": "time_series",
                        "intervalFactor": 2,
                        "legendFormat": "99th Percentile",
                        "refId": "A",
                        "step": 10
                     },
                     {
                        "expr": "histogram_quantile(0.50, sum(rate(loki_boltdb_shipper_retention_marker_table_processed_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (le)) * 1e3",
                        "format": "time_series",
                        "intervalFactor": 2,
                        "legendFormat": "50th Percentile",
                        "refId": "B",
                        "step": 10
                     },
                     {
                        "expr": "sum(rate(loki_boltdb_shipper_retention_marker_table_processed_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) * 1e3 / sum(rate(loki_boltdb_shipper_retention_marker_table_processed_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))",
                        "format": "time_series",
                        "intervalFactor": 2,
                        "legendFormat": "Average",
                        "refId": "C",
                        "step": 10
                     }
                  ],
                  "thresholds": [ ],
                  "timeFrom": null,
                  "timeShift": null,
                  "title": "Mark Table Latency",
                  "tooltip": {
                     "shared": true,
                     "sort": 2,
                     "value_type": "individual"
                  },
                  "type": "graph",
                  "xaxis": {
                     "buckets": null,
                     "mode": "time",
                     "name": null,
                     "show": true,
                     "values": [ ]
                  },
                  "yaxes": [
                     {
                        "format": "ms",
                        "label": null,
                        "logBase": 1,
                        "max": null,
                        "min": 0,
                        "show": true
                     },
                     {
                        "format": "short",
                        "label": null,
                        "logBase": 1,
                        "max": null,
                        "min": null,
                        "show": false
                     }
                  ]
               }
            ],
            "repeat": null,
            "repeatIteration": null,
            "repeatRowId": null,
            "showTitle": true,
            "title": "",
            "titleSize": "h6"
         },
         {
            "collapse": false,
            "height": "250px",
            "panels": [
               {
                  "aliasColors": { },
                  "bars": false,
                  "dashLength": 10,
                  "dashes": false,
                  "datasource": "$datasource",
                  "fill": 1,
                  "format": "short",
                  "id": 12,
                  "legend": {
                     "avg": false,
                     "current": false,
                     "max": false,
                     "min": false,
                     "show": true,
                     "total": false,
                     "values": false
                  },
                  "lines": true,
                  "linewidth": 1,
                  "links": [ ],
                  "nullPointMode": "null as zero",
                  "percentage": false,
                  "pointradius": 5,
                  "points": false,
                  "renderer": "flot",
                  "seriesOverrides": [ ],
                  "spaceLength": 10,
                  "span": 6,
                  "stack": false,
                  "steppedLine": false,
                  "targets": [
                     {
                        "expr": "sum (increase(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\"}[24h]))",
                        "format": "time_series",
                        "instant": true,
                        "intervalFactor": 2,
                        "refId": "A"
                     }
                  ],
                  "thresholds": "70,80",
                  "timeFrom": null,
                  "timeShift": null,
                  "title": "Delete Chunks (24h)",
                  "tooltip": {
                     "shared": true,
                     "sort": 2,
                     "value_type": "individual"
                  },
                  "type": "singlestat",
                  "xaxis": {
                     "buckets": null,
                     "mode": "time",
                     "name": null,
                     "show": true,
                     "values": [ ]
                  },
                  "yaxes": [
                     {
                        "format": "short",
                        "label": null,
                        "logBase": 1,
                        "max": null,
                        "min": 0,
                        "show": true
                     },
                     {
                        "format": "short",
                        "label": null,
                        "logBase": 1,
                        "max": null,
                        "min": null,
                        "show": false
                     }
                  ]
               },
               {
                  "aliasColors": { },
                  "bars": false,
                  "dashLength": 10,
                  "dashes": false,
                  "datasource": "$datasource",
                  "fill": 1,
                  "id": 13,
                  "legend": {
                     "avg": false,
                     "current": false,
                     "max": false,
                     "min": false,
                     "show": true,
                     "total": false,
                     "values": false
                  },
                  "lines": true,
                  "linewidth": 1,
                  "links": [ ],
                  "nullPointMode": "null as zero",
                  "percentage": false,
                  "pointradius": 5,
                  "points": false,
                  "renderer": "flot",
                  "seriesOverrides": [ ],
                  "spaceLength": 10,
                  "span": 6,
                  "stack": false,
                  "steppedLine": false,
                  "targets": [
                     {
                        "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (le)) * 1e3",
                        "format": "time_series",
                        "intervalFactor": 2,
                        "legendFormat": "99th Percentile",
                        "refId": "A",
                        "step": 10
                     },
                     {
                        "expr": "histogram_quantile(0.50, sum(rate(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (le)) * 1e3",
                        "format": "time_series",
                        "intervalFactor": 2,
                        "legendFormat": "50th Percentile",
                        "refId": "B",
                        "step": 10
                     },
                     {
                        "expr": "sum(rate(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) * 1e3 / sum(rate(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))",
                        "format": "time_series",
                        "intervalFactor": 2,
                        "legendFormat": "Average",
                        "refId": "C",
                        "step": 10
                     }
                  ],
                  "thresholds": [ ],
                  "timeFrom": null,
                  "timeShift": null,
                  "title": "Delete Latency",
                  "tooltip": {
                     "shared": true,
                     "sort": 2,
                     "value_type": "individual"
                  },
                  "type": "graph",
                  "xaxis": {
                     "buckets": null,
                     "mode": "time",
                     "name": null,
                     "show": true,
                     "values": [ ]
                  },
                  "yaxes": [
                     {
                        "format": "ms",
                        "label": null,
                        "logBase": 1,
                        "max": null,
                        "min": 0,
                        "show": true
                     },
                     {
                        "format": "short",
                        "label": null,
                        "logBase": 1,
                        "max": null,
                        "min": null,
                        "show": false
                     }
                  ]
               }
            ],
            "repeat": null,
            "repeatIteration": null,
            "repeatRowId": null,
            "showTitle": true,
            "title": "Sweeper",
            "titleSize": "h6"
         },
         {
            "collapse": false,
            "height": "250px",
            "panels": [
               {
                  "aliasColors": { },
                  "bars": false,
                  "dashLength": 10,
                  "dashes": false,
                  "datasource": "$datasource",
                  "fill": 1,
                  "id": 14,
                  "legend": {
                     "avg": false,
                     "current": false,
                     "max": false,
                     "min": false,
                     "show": true,
                     "total": false,
                     "values": false
                  },
                  "lines": true,
                  "linewidth": 1,
                  "links": [ ],
                  "nullPointMode": "null as zero",
                  "percentage": false,
                  "pointradius": 5,
                  "points": false,
                  "renderer": "flot",
                  "seriesOverrides": [ ],
                  "spaceLength": 10,
                  "span": 4,
                  "stack": false,
                  "steppedLine": false,
                  "targets": [
                     {
                        "expr": "time() - (loki_boltdb_shipper_retention_sweeper_marker_file_processing_current_time{cluster=~\"$cluster\", namespace=~\"$namespace\"} > 0)",
                        "format": "time_series",
                        "intervalFactor": 2,
                        "legendFormat": "lag",
                        "legendLink": null,
                        "step": 10
                     }
                  ],
                  "thresholds": [ ],
                  "timeFrom": null,
                  "timeShift": null,
                  "title": "Sweeper Lag",
                  "tooltip": {
                     "shared": true,
                     "sort": 2,
                     "value_type": "individual"
                  },
                  "type": "graph",
                  "xaxis": {
                     "buckets": null,
                     "mode": "time",
                     "name": null,
                     "show": true,
                     "values": [ ]
                  },
                  "yaxes": [
                     {
                        "format": "s",
                        "label": null,
                        "logBase": 1,
                        "max": null,
                        "min": null,
                        "show": true
                     },
                     {
                        "format": "short",
                        "label": null,
                        "logBase": 1,
                        "max": null,
                        "min": null,
                        "show": false
                     }
                  ]
               },
               {
                  "aliasColors": { },
                  "bars": false,
                  "dashLength": 10,
                  "dashes": false,
                  "datasource": "$datasource",
                  "fill": 1,
                  "id": 15,
                  "legend": {
                     "avg": false,
                     "current": false,
                     "max": false,
                     "min": false,
                     "show": true,
                     "total": false,
                     "values": false
                  },
                  "lines": true,
                  "linewidth": 1,
                  "links": [ ],
                  "nullPointMode": "null as zero",
                  "percentage": false,
                  "pointradius": 5,
                  "points": false,
                  "renderer": "flot",
                  "seriesOverrides": [ ],
                  "spaceLength": 10,
                  "span": 4,
                  "stack": false,
                  "steppedLine": false,
                  "targets": [
                     {
                        "expr": "sum(loki_boltdb_shipper_retention_sweeper_marker_files_current{cluster=~\"$cluster\", namespace=~\"$namespace\"})",
                        "format": "time_series",
                        "intervalFactor": 2,
                        "legendFormat": "count",
                        "legendLink": null,
                        "step": 10
                     }
                  ],
                  "thresholds": [ ],
                  "timeFrom": null,
                  "timeShift": null,
                  "title": "Marks Files to Process",
                  "tooltip": {
                     "shared": true,
                     "sort": 2,
                     "value_type": "individual"
                  },
                  "type": "graph",
                  "xaxis": {
                     "buckets": null,
                     "mode": "time",
                     "name": null,
                     "show": true,
                     "values": [ ]
                  },
                  "yaxes": [
                     {
                        "format": "short",
                        "label": null,
                        "logBase": 1,
                        "max": null,
                        "min": 0,
                        "show": true
                     },
                     {
                        "format": "short",
                        "label": null,
                        "logBase": 1,
                        "max": null,
                        "min": null,
                        "show": false
                     }
                  ]
               },
               {
                  "aliasColors": { },
                  "bars": false,
                  "dashLength": 10,
                  "dashes": false,
                  "datasource": "$datasource",
                  "fill": 1,
                  "id": 16,
                  "legend": {
                     "avg": false,
                     "current": false,
                     "max": false,
                     "min": false,
                     "show": true,
                     "total": false,
                     "values": false
                  },
                  "lines": true,
                  "linewidth": 1,
                  "links": [ ],
                  "nullPointMode": "null as zero",
                  "percentage": false,
                  "pointradius": 5,
                  "points": false,
                  "renderer": "flot",
                  "seriesOverrides": [ ],
                  "spaceLength": 10,
                  "span": 4,
                  "stack": false,
                  "steppedLine": false,
                  "targets": [
                     {
                        "expr": "sum by (status)(rate(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))",
                        "format": "time_series",
                        "intervalFactor": 2,
                        "legendFormat": "{{status}}",
                        "legendLink": null,
                        "step": 10
                     }
                  ],
                  "thresholds": [ ],
                  "timeFrom": null,
                  "timeShift": null,
                  "title": "Delete Rate Per Status",
                  "tooltip": {
                     "shared": true,
                     "sort": 2,
                     "value_type": "individual"
                  },
                  "type": "graph",
                  "xaxis": {
                     "buckets": null,
                     "mode": "time",
                     "name": null,
                     "show": true,
                     "values": [ ]
                  },
                  "yaxes": [
                     {
                        "format": "short",
                        "label": null,
                        "logBase": 1,
                        "max": null,
                        "min": 0,
                        "show": true
                     },
                     {
                        "format": "short",
                        "label": null,
                        "logBase": 1,
                        "max": null,
                        "min": null,
                        "show": false
                     }
                  ]
               }
            ],
            "repeat": null,
            "repeatIteration": null,
            "repeatRowId": null,
            "showTitle": true,
            "title": "",
            "titleSize": "h6"
         },
         {
            "collapse": false,
            "height": "250px",
            "panels": [
               {
                  "datasource": "$logs",
                  "id": 17,
                  "span": 12,
                  "targets": [
                     {
                        "expr": "{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-read\"}",
                        "refId": "A"
                     }
                  ],
                  "title": "Compactor Logs",
                  "type": "logs"
               }
            ],
            "repeat": null,
            "repeatIteration": null,
            "repeatRowId": null,
            "showTitle": true,
            "title": "Logs",
            "titleSize": "h6"
         }
      ],
      "schemaVersion": 14,
      "style": "dark",
      "tags": [
         "loki"
      ],
      "templating": {
         "list": [
            {
               "current": {
                  "text": "default",
                  "value": "default"
               },
               "hide": 0,
               "label": "Data Source",
               "name": "datasource",
               "options": [ ],
               "query": "prometheus",
               "refresh": 1,
               "regex": "",
               "type": "datasource"
            },
            {
               "allValue": null,
               "current": {
                  "text": "prod",
                  "value": "prod"
               },
               "datasource": "$datasource",
               "hide": 0,
               "includeAll": false,
               "label": "cluster",
               "multi": false,
               "name": "cluster",
               "options": [ ],
               "query": "label_values(loki_build_info, cluster)",
               "refresh": 1,
               "regex": "",
               "sort": 2,
               "tagValuesQuery": "",
               "tags": [ ],
               "tagsQuery": "",
               "type": "query",
               "useTags": false
            },
            {
               "allValue": null,
               "current": {
                  "text": "prod",
                  "value": "prod"
               },
               "datasource": "$datasource",
               "hide": 0,
               "includeAll": false,
               "label": "namespace",
               "multi": false,
               "name": "namespace",
               "options": [ ],
               "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)",
               "refresh": 1,
               "regex": "",
               "sort": 2,
               "tagValuesQuery": "",
               "tags": [ ],
               "tagsQuery": "",
               "type": "query",
               "useTags": false
            },
            {
               "hide": 0,
               "label": null,
               "name": "logs",
               "options": [ ],
               "query": "loki",
               "refresh": 1,
               "regex": "",
               "type": "datasource"
            }
         ]
      },
      "time": {
         "from": "now-1h",
         "to": "now"
      },
      "timepicker": {
         "refresh_intervals": [
            "5s",
            "10s",
            "30s",
            "1m",
            "5m",
            "15m",
            "30m",
            "1h",
            "2h",
            "1d"
         ],
         "time_options": [
            "5m",
            "15m",
            "1h",
            "6h",
            "12h",
            "24h",
            "2d",
            "7d",
            "30d"
         ]
      },
      "timezone": "utc",
      "title": "Loki / Retention",
      "uid": "retention",
      "version": 0
   }