"""Lambda Functions for calculate SLA of O11y prometheus available""" import json from datetime import datetime, timezone import pymysql import sliutil.connection import sliutil.histogram import sliutil.string import sliutil.tidb import sliutil.time SLI_TABLE_NAME = "sli_o11y_dedicated_prometheus_upload_delay" SLA_TABLE_NAME = "sla_o11y_dedicated_prometheus_upload_delay" def lambda_handler(event, context): # pylint:disable=unused-argument """AWS Lambda Function Handler Parameters ---------- event: dict, required API Gateway Lambda Proxy Input Format Event doc: https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html#api-gateway-simple-proxy-for-lambda-input-format # pylint:disable=line-too-long # noqa context: object, required Lambda Context runtime methods and attributes Context doc: https://docs.aws.amazon.com/lambda/latest/dg/python-context-object.html Returns ------ API Gateway Lambda Proxy Output Format: dict Return doc: https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html # pylint:disable=line-too-long """ tidb_conn = sliutil.connection.get_tidb_connection() month_str = event.get("month", "") begin_ts, end_ts = sliutil.time.get_month_begin_end_ts(month=month_str) handler = Handler(connection=tidb_conn) handler.run(begin_ts=begin_ts, end_ts=end_ts) return { "statusCode": 200, "body": json.dumps( { "message": "hello world", # "location": ip.text.replace("\n", "") } ), } class Handler: """Handle metrics `reporter_service_level_probe_cluster_counter`""" def __init__(self, connection: pymysql.Connection) -> None: self.connection = connection def run(self, begin_ts: float, end_ts: float) -> None: """Calculate SLA of O11y prometheus available from begin_ts to end_ts and save the results into a new table """ print(f"Start to calculate SLA, begin_ts: {begin_ts}, end_ts: {end_ts}") o11y_clusters = self.list_clusters(begin_ts=begin_ts, end_ts=end_ts) o11y_cluster_ids = [cluster["o11y_cluster_id"] for cluster in o11y_clusters] results = self.search( begin_ts=begin_ts, end_ts=end_ts, o11y_cluster_ids=o11y_cluster_ids, ) cluster_buckets = {} for row in results: key = row["o11y_cluster_id"] if key not in cluster_buckets: cluster_buckets[key] = [] cluster_buckets[key].append((float(row["le"]), float(row["sum_increment"]))) data = [] for cluster in o11y_clusters: value = sliutil.histogram.quantile( tuples=cluster_buckets.get(cluster["o11y_cluster_id"], []), q_value=0.999, ) data.append( ( cluster["o11y_cluster_id"], sliutil.time.get_month_from_timestamp(timestamp=begin_ts), value, cluster["region"], cluster["tenant_id"], cluster["project_id"], cluster["application_id"], ) ) self.save(data=data) print("Finish calculate SLA") def search(self, begin_ts: float, end_ts: float, o11y_cluster_ids: list) -> list: """List downtime minutes from begin_ts to end_ts""" results = [] start = datetime.now(timezone.utc).timestamp() with self.connection.cursor() as cursor: for i in range(0, len(o11y_cluster_ids), 100): end = min(i + 100, len(o11y_cluster_ids)) sql = f""" SELECT o11y_cluster_id, le, SUM(increment) AS sum_increment FROM `{SLI_TABLE_NAME}` WHERE timestamp >= {begin_ts:.0f} AND timestamp <= {end_ts:.0f} AND status != "skipped_by_not_active" AND status != "skipped_by_not_exclusive" AND o11y_cluster_id IN ({sliutil.string.list_to_string(o11y_cluster_ids[i:end])}) GROUP BY o11y_cluster_id, le """ # print("search, sql: ", sql) cursor.execute(sql) results.extend(cursor.fetchall()) end = datetime.now(timezone.utc).timestamp() print(f"search, take {end - start:.2f}s") return results def list_clusters(self, begin_ts: float, end_ts: float) -> list: """List clusters""" with self.connection.cursor() as cursor: sql = f""" SELECT DISTINCT o11y_cluster_id, region, tenant_id, project_id, application_id FROM `{SLI_TABLE_NAME}` WHERE timestamp >= {begin_ts:.0f} and timestamp <= {end_ts:.0f} AND status != "skipped_by_not_active" AND status != "skipped_by_not_exclusive" """ # print("search, sql: ", sql) cursor.execute(sql) result = cursor.fetchall() print(f"list o11 clusters, len: {len(result)}") return result def save(self, data: list) -> None: """Save data into table""" print(f"save data, len: {len(data)}") with self.connection.cursor() as cursor: sql = f""" INSERT INTO `{SLA_TABLE_NAME}` ( o11y_cluster_id, month, p999, region, tenant_id, project_id, application_id ) VALUES (%s, %s, %s, %s, %s, %s, %s) ON DUPLICATE KEY UPDATE p999=VALUES(p999) """ cursor.executemany(sql, data) self.connection.commit()