import json import os import sliutil.alert import sliutil.secret from sendgrid import SendGridAPIClient from sendgrid.helpers.mail import Mail SENDGRID_API_KEY = sliutil.secret.get_secret("SENDGRID_API_KEY") SLI_STAGE = os.environ["SLI_STAGE"] JIRA_PROJECT = os.environ["JIRA_PROJECT"] EMAIL_SUFFIX = "@notifications.statuspage.io" def lambda_handler(event, context): # pylint:disable=unused-argument """AWS Lambda Function Handler Parameters ---------- event: dict, required API Gateway Lambda Proxy Input Format Event doc: https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html#api-gateway-simple-proxy-for-lambda-input-format # pylint:disable=line-too-long # noqa context: object, required Lambda Context runtime methods and attributes Context doc: https://docs.aws.amazon.com/lambda/latest/dg/python-context-object.html Returns ------ API Gateway Lambda Proxy Output Format: dict Return doc: https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html # pylint:disable=line-too-long """ body = json.loads(event["body"]) print("request body:", body) dry_run = event.get("dry_run", False) website_name = body["website_name"] alert_type = body["alert_type"] website_url = body.get("website_url", "") alert_detail = body.get("alert_detail", "") email_prefix = get_component_prefix(SLI_STAGE, website_name) if email_prefix == "": send_email( "chensiyu@pingcap.com", "[ERROR] uptimerobot sync error, unknown website", event["body"], ) return {"statusCode": 200, "body": "skip, unknown website"} to_email = "" subject = alert_type.upper() needAlert = False if subject == "DOWN": needAlert = True to_email = f"{email_prefix}+partial_outage{EMAIL_SUFFIX}" elif subject == "UP": to_email = f"{email_prefix}{EMAIL_SUFFIX}" else: send_email( "chensiyu@pingcap.com", "[ERROR] uptimerobot sync error, no matched subject", event["body"], ) return {"statusCode": 200, "body": f"skip, unknown alert type, {event['body']}"} if not dry_run: send_email(to_email, subject, event["body"]) if needAlert: alert = generate_alert_content( website_name=website_name, website_url=website_url, alert_type=alert_type, alert_detail=alert_detail, ) print("jira alert:", alert) if alert is not None and not dry_run: sliutil.alert.create_alerts(alerts=[alert]) result = { "statusCode": 200, "body": { "to_email": to_email, "subject": subject, }, } print(f"result: {result}") return result def get_component_prefix(stage: str, website_name: str) -> str: if stage.upper() == "PROD": component_email_map = { "TiDB Cloud Portal Backend": "component+f1e31e6970cf460b958ceabe1d647530", "TiDB Cloud Portal Frontend": "component+ff8ea9f52c6247c1a29e6e614b734cde", "TiDB Cloud Starter(aws/ap-northeast-1)": "component+2e7a52e0ca0f4312a2a7cfd5cc403bca", "TiDB Cloud Starter(aws/ap-southeast-1)": "component+8dded71a9d9b4d309de65d446b2673f1", "TiDB Cloud Starter(aws/eu-central-1)": "component+27f795b23aa04d4bbd72a7722d704848", "TiDB Cloud Starter(aws/us-east-1)": "component+6407e7f6a4694d4ab550de3a452581ca", "TiDB Cloud Starter(aws/us-west-2)": "component+c30ea0b3149247d88c3151da659fea36", "TiDB Cloud Starter(alicloud/ap-southeast-1)": "component+269d575df223402f994673dadfde479a", "TiDB Cloud Essential(aws/ap-northeast-1)": "component+4eda5e6b96504d7fbec9fab8888ef38a", "TiDB Cloud Essential(aws/ap-southeast-1)": "component+78a2adc28aa142a6a844215ad463bf9c", "TiDB Cloud Essential(aws/eu-central-1)": "component+ae80471b25c24642b5b1018072a51a13", "TiDB Cloud Essential(aws/us-east-1)": "component+22263609ec8842ed975946e681173881", "TiDB Cloud Essential(aws/us-west-2)": "component+91933fdb44954f3a9b4c12b76894890c", "TiDB Cloud Essential(alicloud/ap-southeast-1)": "component+98684be13d2a473cbe9aad2a5c0aec2c", "TiDB Cloud Essential(alicloud/ap-northeast-1)": "component+84df16f6381d4430be18406c37675845", "TiDB Cloud Essential(alicloud/ap-southeast-5)": "component+555d3a730ed34b5483ff268f955375ed", "TiDB Cloud Essential(alicloud/na-south-1)": "component+d2122bfd19a142d7b471128c5e91eefa", } return component_email_map.get(website_name, "") component_email_map = { "TiDB Cloud Portal Backend": "component+a9b6a2f8cffd4174b835b1c99f36c345", "TiDB Cloud Portal Frontend": "component+4cab9c4a482848f1b9c5baa2a6b84793", "TiDB Cloud Starter(aws/eu-central-1)": "component+1475d7bdcfae403a9f2a101c2df3146b", "TiDB Cloud Starter(aws/us-west-2)": "component+41fcde2a780f444f9341ed2f24b2ce94", } return component_email_map.get(website_name, "") def send_email(to_email, subject, content): message = Mail( from_email="no-reply@tidbcloud.com", to_emails=to_email, subject=subject, html_content=content, ) try: sg = SendGridAPIClient(SENDGRID_API_KEY) response = sg.send(message) print( f"send email success, to_email: {to_email}, subject: {subject}, response code: {response.status_code}, response body: {response.body}" ) except Exception as e: print( f"send email failed, {e}, to_email: {to_email}, subject: {subject}, content: {content}" ) def generate_alert_content( website_name: str, website_url: str, alert_type: str, alert_detail: str ) -> dict | None: """Generate content of alert""" if website_name == "TiDB Cloud Portal Backend": alert_name = "IncidentWebConsoleBackendIsDown" component_name = "resilience" elif website_name == "TiDB Cloud Portal Frontend": alert_name = "IncidentCustomFacingUIDown" component_name = "frontend" else: return None return { "project": {"key": JIRA_PROJECT}, "summary": f"[{SLI_STAGE.upper()}] {alert_name}", "description": f""" Annotations: - message = Website {website_name}({website_url}) is currently {alert_type}, detail: {alert_detail} """, "issuetype": {"name": "Task"}, "priority": {"name": "Critical"}, "components": [{"name": component_name}], } if __name__ == "__main__": # ev = { # "body": json.dumps( # { # "website_name": "TiDB Cloud Portal Backend", # "website_url": "https://staging.tidbcloud.com", # "alert_type": "DOWN", # "alert_detail": "Test alert detail", # } # ), # "dry_run": True, # } ev = { "body": json.dumps( { "website_name": "TiDB Cloud Starter(aws/eu-central-1)", "alert_type": "DOWN", } ), "dry_run": True, } lambda_handler(ev, None)