{
  "cluster_id": "1379661944579024068",
  "instance": "db-tidb-1",
  "occurrence_time": 1672385214,
  "detect_time": 1672385497,
  "analyze_time": 1673340371,
  "evidences": {
    "source": "k8s eventrouter",
    "content": [
      "{\"verb\":\"ADDED\",\"event\":{\"metadata\":{\"name\":\"db-tidb-1.1735817e02612001\",\"namespace\":\"tidb1379661944579024068\",\"selfLink\":\"/api/v1/namespaces/tidb1379661944579024068/events/db-tidb-1.1735817e02612001\",\"uid\":\"5797f131-fdeb-4bdb-99b6-de2a3f94a81e\",\"resourceVersion\":\"667007\",\"creationTimestamp\":\"2022-12-30T07:26:54Z\",\"annotations\":{\"offending_containers\":\"tidb\",\"offending_containers_usage\":\"12390584Ki\",\"starved_resource\":\"memory\"},\"managedFields\":[{\"manager\":\"kubelet\",\"operation\":\"Update\",\"apiVersion\":\"v1\",\"time\":\"2022-12-30T07:26:54Z\"}]},\"involvedObject\":{\"kind\":\"Pod\",\"namespace\":\"tidb1379661944579024068\",\"name\":\"db-tidb-1\",\"uid\":\"b4d7b34e-3b5f-44dc-98bc-4d3af915b436\",\"apiVersion\":\"v1\",\"resourceVersion\":\"909938468\"},\"reason\":\"Evicted\",\"message\":\"The node was low on resource: memory. Container tidb was using 12390584Ki, which exceeds its request of 200Mi. \",\"source\":{\"component\":\"kubelet\",\"host\":\"ip-10-250-17-97.us-west-2.compute.internal\"},\"firstTimestamp\":\"2022-12-30T07:26:54Z\",\"lastTimestamp\":\"2022-12-30T07:26:54Z\",\"count\":1,\"type\":\"Warning\",\"eventTime\":null,\"reportingComponent\":\"\",\"reportingInstance\":\"\"}}"
    ]
  },
  "running_sqls_topmem": [
    {
      "time": 1672385170.302,
      "cluster": "1379661944579024068",
      "component": "tidb",
      "instance": "tidb-1",
      "content": {
        "cost_time": " 40.355987616s",
        "cop_time": " 34.058606149s",
        "request_count": " 238",
        "total_keys": " 37",
        "process_keys": " 36",
        "num_cop_tasks": " 238",
        "process_avg_time": " 0s",
        "process_p90_time": " 0s",
        "process_max_time": " 0s",
        "process_max_addr": " db-tiflash-0.db-tiflash-peer.tidb1379661944579024068.svc:3930",
        "wait_avg_time": " 0s",
        "wait_p90_time": " 0s",
        "wait_max_time": " 0s",
        "wait_max_addr": " db-tiflash-0.db-tiflash-peer.tidb1379661944579024068.svc:3930",
        "stats": " tidb_version_tags:pseudo,telemetry_tidb_instance:438404718330380314,telemetry_tidb_basic_info:438404969975513114",
        "conn_id": " 4037657385844605639",
        "user": " mimir",
        "database": " telemetry-global",
        "table_ids": " [1436,450,432]",
        "txn_start_ts": " 438405738083647516",
        "mem_max": " 3297952336 Bytes (3.07 GB)",
        "sql": " select date ( `telemetry ti db basic info with date` . `ts` ) as `ts` , `tidb_version_tags` . `tag_value` as `version_series` , count ( distinct `telemetry ti db basic info with date` . `tracking_id` ) as `count` from `telemetry_tidb_instance` inner join `telemetry_tidb_basic_info_with_date` `telemetry ti db basic info with date` on `telemetry_tidb_instance` . `basic_info_id` = `telemetry ti db basic info with date` . `pk_id` inner join `tidb_version_tags` on `telemetry_tidb_instance` . `version` = `tidb_version_tags` . `version` where `lower` ( `telemetry_tidb_instance` . `up_time` ) like ? and not ( `substring_index` ( `telemetry_tidb_instance` . `up_time` , ... ) between - ? and ? ) and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `pk_id` in ( select `basic_info_id` from ( select count ( `basic_info_id` ) as `tikv_count` , `basic_info_id` from `telemetry_tidb_instance` where `instance_type` = ? group by `basic_info_id` ) as `t` where `tikv_count` >= ? ) and `tidb_version_tags` . `tag_value` is not ? and date ( `telemetry ti db basic info with date` . `ts` ) between date ( date_add ( now ( ) , interval - ? day ) ) and date ( date_add ( now ( ) , interval - ? day ) ) group by date ( `telemetry ti db basic info with date` . `ts` ) , `tidb_version_tags` . `tag_value` order by date ( `telemetry ti db basic info with date` . `ts` ) asc",
        "mem_max_human": "3.07 GB"
      }
    },
    {
      "time": 0,
      "cluster": "1379661944579024068",
      "component": "tidb",
      "instance": "tidb-1",
      "content": {
        "cost_time": " 1214.981152374s",
        "conn_id": " 4037657385844604931",
        "txn_start_ts": " 0",
        "mem_max": " 56274156 Bytes (53.7 MB)",
        "sql": " analyze table `dbaas-sli`.`reporter_service_level_probe_cluster_counter`",
        "mem_max_human": "53.7 MB"
      }
    }
  ],
  "running_sqls_toptime": [
    {
      "time": 0,
      "cluster": "1379661944579024068",
      "component": "tidb",
      "instance": "tidb-1",
      "content": {
        "cost_time": " 1214.981152374s",
        "conn_id": " 4037657385844604931",
        "txn_start_ts": " 0",
        "mem_max": " 56274156 Bytes (53.7 MB)",
        "sql": " analyze table `dbaas-sli`.`reporter_service_level_probe_cluster_counter`",
        "mem_max_human": "53.7 MB"
      }
    },
    {
      "time": 1672385170.302,
      "cluster": "1379661944579024068",
      "component": "tidb",
      "instance": "tidb-1",
      "content": {
        "cost_time": " 40.355987616s",
        "cop_time": " 34.058606149s",
        "request_count": " 238",
        "total_keys": " 37",
        "process_keys": " 36",
        "num_cop_tasks": " 238",
        "process_avg_time": " 0s",
        "process_p90_time": " 0s",
        "process_max_time": " 0s",
        "process_max_addr": " db-tiflash-0.db-tiflash-peer.tidb1379661944579024068.svc:3930",
        "wait_avg_time": " 0s",
        "wait_p90_time": " 0s",
        "wait_max_time": " 0s",
        "wait_max_addr": " db-tiflash-0.db-tiflash-peer.tidb1379661944579024068.svc:3930",
        "stats": " tidb_version_tags:pseudo,telemetry_tidb_instance:438404718330380314,telemetry_tidb_basic_info:438404969975513114",
        "conn_id": " 4037657385844605639",
        "user": " mimir",
        "database": " telemetry-global",
        "table_ids": " [1436,450,432]",
        "txn_start_ts": " 438405738083647516",
        "mem_max": " 3297952336 Bytes (3.07 GB)",
        "sql": " select date ( `telemetry ti db basic info with date` . `ts` ) as `ts` , `tidb_version_tags` . `tag_value` as `version_series` , count ( distinct `telemetry ti db basic info with date` . `tracking_id` ) as `count` from `telemetry_tidb_instance` inner join `telemetry_tidb_basic_info_with_date` `telemetry ti db basic info with date` on `telemetry_tidb_instance` . `basic_info_id` = `telemetry ti db basic info with date` . `pk_id` inner join `tidb_version_tags` on `telemetry_tidb_instance` . `version` = `tidb_version_tags` . `version` where `lower` ( `telemetry_tidb_instance` . `up_time` ) like ? and not ( `substring_index` ( `telemetry_tidb_instance` . `up_time` , ... ) between - ? and ? ) and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `pk_id` in ( select `basic_info_id` from ( select count ( `basic_info_id` ) as `tikv_count` , `basic_info_id` from `telemetry_tidb_instance` where `instance_type` = ? group by `basic_info_id` ) as `t` where `tikv_count` >= ? ) and `tidb_version_tags` . `tag_value` is not ? and date ( `telemetry ti db basic info with date` . `ts` ) between date ( date_add ( now ( ) , interval - ? day ) ) and date ( date_add ( now ( ) , interval - ? day ) ) group by date ( `telemetry ti db basic info with date` . `ts` ) , `tidb_version_tags` . `tag_value` order by date ( `telemetry ti db basic info with date` . `ts` ) asc",
        "mem_max_human": "3.07 GB"
      }
    }
  ],
  "expensive_sqls": [
    {
      "time": 1672384055.790425,
      "cluster": "tidb1379661944579024068",
      "component": "tidb",
      "instance": "db-tidb-1",
      "content": {
        "cost_time": "60.080985136s",
        "conn_id": "4037657385844604931",
        "txn_start_ts": "0",
        "mem_max": "4134234 Bytes (3.94 MB)",
        "sql": "analyze table `dbaas-sli`.`reporter_service_level_probe_cluster_counter`",
        "mem_max_human": "3.94 MB"
      }
    },
    {
      "time": 1672385020.790401,
      "cluster": "tidb1379661944579024068",
      "component": "tidb",
      "instance": "db-tidb-1",
      "content": {
        "cost_time": "60.07342332s",
        "cop_time": "29.645493189s",
        "request_count": "254",
        "total_keys": "37",
        "process_keys": "36",
        "num_cop_tasks": "254",
        "process_avg_time": "0s",
        "process_p90_time": "0s",
        "process_max_time": "0s",
        "process_max_addr": "db-tiflash-0.db-tiflash-peer.tidb1379661944579024068.svc:3930",
        "wait_avg_time": "0s",
        "wait_p90_time": "0s",
        "wait_max_time": "0s",
        "wait_max_addr": "db-tiflash-0.db-tiflash-peer.tidb1379661944579024068.svc:3930",
        "stats": "telemetry_tidb_instance:438404718330380314,telemetry_tidb_basic_info:438404969975513114,tidb_version_tags:pseudo",
        "conn_id": "4037657385844605639",
        "user": "mimir",
        "database": "telemetry-global",
        "table_ids": "[1436,450,432]",
        "txn_start_ts": "438405683138265095",
        "mem_max": "3316755833 Bytes (3.09 GB)",
        "sql": "select date ( `telemetry ti db basic info with date` . `ts` ) as `ts` , `tidb_version_tags` . `tag_value` as `version_series` , count ( distinct `telemetry ti db basic info with date` . `tracking_id` ) as `count` from `telemetry_tidb_instance` inner join `telemetry_tidb_basic_info_with_date` `telemetry ti db basic info with date` on `telemetry_tidb_instance` . `basic_info_id` = `telemetry ti db basic info with date` . `pk_id` inner join `tidb_version_tags` on `telemetry_tidb_instance` . `version` = `tidb_version_tags` . `version` where `lower` ( `telemetry_tidb_instance` . `up_time` ) like ? and not ( `substring_index` ( `telemetry_tidb_instance` . `up_time` , ... ) between - ? and ? ) and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `ip` <> ? and `telemetry ti db basic info with date` . `pk_id` in ( select `basic_info_id` from ( select count ( `basic_info_id` ) as `tikv_count` , `basic_info_id` from `telemetry_tidb_instance` where `instance_type` = ? group by `basic_info_id` ) as `t` where `tikv_count` >= ? ) and `tidb_version_tags` . `tag_value` is not ? and date ( `telemetry ti db basic info with date` . `ts` ) between date ( date_add ( now ( ) , interval - ? day ) ) and date ( date_add ( now ( ) , interval - ? day ) ) group by date ( `telemetry ti db basic info with date` . `ts` ) , `tidb_version_tags` . `tag_value` order by date ( `telemetry ti db basic info with date` . `ts` ) asc",
        "mem_max_human": "3.09 GB"
      }
    }
  ],
  "logs": {
    "out_of_memory_quota": [],
    "memory_exceeds_quota": [
      {
        "time": 1672383658.068745,
        "content": "[2022/12/30 07:00:58.064 +00:00] [INFO] [coprocessor.go:1281] [\"memory exceeds quota, destroy one token now.\"] [consumed=1078903041] [quota=1073741824] [\"total token count\"=15] [\"remaining token count\"=15]"
      },
      {
        "time": 1672383658.103418,
        "content": "[2022/12/30 07:00:58.103 +00:00] [INFO] [coprocessor.go:1281] [\"memory exceeds quota, destroy one token now.\"] [consumed=1073791521] [quota=1073741824] [\"total token count\"=15] [\"remaining token count\"=14]"
      }
    ]
  }
}