AWSTemplateFormatVersion: "2010-09-09"
Transform: AWS::Serverless-2016-10-31
Description: LATAM tenant monitoring, scoring, metrics, and summary pipeline.

Parameters:
  DevOpsDBHost:
    Type: String
  DevOpsDBPort:
    Type: String
    Default: "3306"
  DevOpsDBUser:
    Type: String
  DevOpsDBPassword:
    Type: String
    NoEcho: true
  DevOpsDBName:
    Type: String
  TargetDatabaseURL:
    Type: String
    Description: TiDB connection URL whose database path is latam_regional_support.
    NoEcho: true
  HubSpotWritebackPrivateAppToken:
    Type: String
    NoEcho: true
    Default: ""
  HubSpotRequestTimeoutSeconds:
    Type: String
    Default: "30"
  HubSpotRetryCount:
    Type: String
    Default: "3"
  DevOpsAPIToken:
    Type: String
    NoEcho: true
  ClinicAPIKey:
    Type: String
    NoEcho: true
  QwenAPIKey:
    Type: String
    NoEcho: true
  QwenWorkspaceID:
    Type: String
    NoEcho: true
  DashScopeEndpoint:
    Type: String
  QwenRegion:
    Type: String
  FeishuWebhookURL:
    Type: String
    NoEcho: true
  OperatorEmail:
    Type: String
    Default: ""
  DevOpsBaseURL:
    Type: String
    Default: https://prod.devops.pingcap.com
  ClinicBaseURL:
    Type: String
    Default: https://clinic.pingcap.com
  QwenWebSearchModel:
    Type: String
    Default: qwen3.8-max
  DomainIntelligenceTTLDays:
    Type: String
    Default: "30"
  LLMTimeoutSeconds:
    Type: String
    Default: "60"
  BillingRequestIntervalSeconds:
    Type: String
    Default: "2.2"
  BillingRetryCount:
    Type: String
    Default: "3"
  KeyLeadsCacheTTLSeconds:
    Type: String
    Default: "900"
  TargetTiDBSSLCAPath:
    Type: String
    Default: ""
  ImportEventCountTable:
    Type: String
    Default: latam_tenant_events
    AllowedPattern: "^[A-Za-z0-9_]+$"
  SummarySectionLimit:
    Type: String
    Default: "10"
    AllowedValues:
      - "10"
  ScheduleState:
    Type: String
    Default: DISABLED
    AllowedValues:
      - ENABLED
      - DISABLED
  DispatcherScheduleExpression:
    Type: String
    Default: cron(5 0 * * ? *)
  SummaryScheduleExpression:
    Type: String
    Default: cron(0 2 * * ? *)
  WeeklyMetricsScheduleExpression:
    Type: String
    Default: cron(0 4 ? * MON *)
  LogRetentionDays:
    Type: Number
    Default: 30
    AllowedValues:
      - 7
      - 14
      - 30
      - 60
      - 90
  AlarmTopicArn:
    Type: String
    Default: ""

Conditions:
  HasAlarmTopic: !Not [!Equals [!Ref AlarmTopicArn, ""]]

Globals:
  Function:
    Runtime: python3.12
    Architectures:
      - x86_64
    Tracing: Active

Resources:
  MonitorDeadLetterQueue:
    Type: AWS::SQS::Queue
    Properties:
      QueueName: LATAM-Tenants-Monitor-Fanout-DLQ
      MessageRetentionPeriod: 1209600
      SqsManagedSseEnabled: true

  MonitorQueue:
    Type: AWS::SQS::Queue
    Properties:
      QueueName: LATAM-Tenants-Monitor-Fanout-Queue
      VisibilityTimeout: 5400
      MessageRetentionPeriod: 345600
      ReceiveMessageWaitTimeSeconds: 20
      SqsManagedSseEnabled: true
      RedrivePolicy:
        deadLetterTargetArn: !GetAtt MonitorDeadLetterQueue.Arn
        maxReceiveCount: 5

  ScoringDeadLetterQueue:
    Type: AWS::SQS::Queue
    Properties:
      QueueName: LATAM-Tenants-Scoring-DLQ
      MessageRetentionPeriod: 1209600
      SqsManagedSseEnabled: true

  ScoringQueue:
    Type: AWS::SQS::Queue
    Properties:
      QueueName: LATAM-Tenants-Scoring-Queue
      VisibilityTimeout: 720
      MessageRetentionPeriod: 345600
      ReceiveMessageWaitTimeSeconds: 20
      SqsManagedSseEnabled: true
      RedrivePolicy:
        deadLetterTargetArn: !GetAtt ScoringDeadLetterQueue.Arn
        maxReceiveCount: 5

  DispatcherFunction:
    Type: AWS::Serverless::Function
    Properties:
      FunctionName: LATAM-Tenants-Monitor-Fanout-Dispatcher-Task
      CodeUri: LATAM-Tenants-Monitor-Fanout-Dispatcher-Task/
      Handler: dispatcher_function.lambda_handler
      Description: Select LATAM tenants and fan out monitor messages.
      Timeout: 120
      MemorySize: 256
      Environment:
        Variables:
          DEVOPS_DB_HOST: !Ref DevOpsDBHost
          DEVOPS_DB_PORT: !Ref DevOpsDBPort
          DEVOPS_DB_USER: !Ref DevOpsDBUser
          DEVOPS_DB_PASSWORD: !Ref DevOpsDBPassword
          DEVOPS_DB_NAME: !Ref DevOpsDBName
          MONITOR_QUEUE_URL: !Ref MonitorQueue
      Policies:
        - AWSLambdaBasicExecutionRole
        - AWSXRayDaemonWriteAccess
        - SQSSendMessagePolicy:
            QueueName: !GetAtt MonitorQueue.QueueName

  WorkerFunction:
    Type: AWS::Serverless::Function
    Properties:
      FunctionName: LATAM-Tenants-Monitor-Fanout-Worker-Task
      CodeUri: LATAM-Tenants-Monitor-Fanout-Worker-Task/
      Handler: worker_function.lambda_handler
      Description: Enrich one LATAM tenant per monitor queue message.
      Timeout: 900
      MemorySize: 512
      ReservedConcurrentExecutions: 2
      Environment:
        Variables:
          TARGET_DATABASE_URL: !Ref TargetDatabaseURL
          SCORING_QUEUE_URL: !Ref ScoringQueue
          DEVOPS_API_TOKEN: !Ref DevOpsAPIToken
          CLINIC_API_KEY: !Ref ClinicAPIKey
          OPERATOR_EMAIL: !Ref OperatorEmail
          API_KEY: !Ref QwenAPIKey
          WORKSPACE_ID: !Ref QwenWorkspaceID
          DASHSCOPE_ENDPOINT: !Ref DashScopeEndpoint
          REGION: !Ref QwenRegion
          DEVOPS_BASE_URL: !Ref DevOpsBaseURL
          CLINIC_BASE_URL: !Ref ClinicBaseURL
          QWEN_WEB_SEARCH_MODEL: !Ref QwenWebSearchModel
          DOMAIN_INTELLIGENCE_TTL_DAYS: !Ref DomainIntelligenceTTLDays
          LLM_TIMEOUT_SECONDS: !Ref LLMTimeoutSeconds
          BILLING_REQUEST_INTERVAL_SECONDS: !Ref BillingRequestIntervalSeconds
          BILLING_RETRY_COUNT: !Ref BillingRetryCount
          TARGET_TIDB_SSL_CA_PATH: !Ref TargetTiDBSSLCAPath
      Policies:
        - AWSLambdaBasicExecutionRole
        - AWSXRayDaemonWriteAccess
        - SQSPollerPolicy:
            QueueName: !GetAtt MonitorQueue.QueueName
        - SQSSendMessagePolicy:
            QueueName: !GetAtt ScoringQueue.QueueName
      Events:
        MonitorQueueEvent:
          Type: SQS
          Properties:
            Queue: !GetAtt MonitorQueue.Arn
            BatchSize: 1
            FunctionResponseTypes:
              - ReportBatchItemFailures
            ScalingConfig:
              MaximumConcurrency: 2

  ScoringFunction:
    Type: AWS::Serverless::Function
    Properties:
      FunctionName: LATAM-Tenants-Scoring-Task
      CodeUri: LATAM-Tenants-Scoring-Task/
      Handler: scoring_function.lambda_handler
      Description: Score LATAM tenants and update daily score snapshots.
      Timeout: 180
      MemorySize: 256
      Environment:
        Variables:
          TARGET_DATABASE_URL: !Ref TargetDatabaseURL
          TARGET_TIDB_SSL_CA_PATH: !Ref TargetTiDBSSLCAPath
          KEY_LEADS_CACHE_TTL_SECONDS: !Ref KeyLeadsCacheTTLSeconds
          HUBSPOT_WRITEBACK_TOKEN: !Ref HubSpotWritebackPrivateAppToken
          HUBSPOT_WRITEBACK_PROPERTY_NAME: latam_pql_priority_tier
          HUBSPOT_WRITEBACK_TIMEOUT_SECONDS: !Ref HubSpotRequestTimeoutSeconds
          HUBSPOT_WRITEBACK_RETRY_COUNT: !Ref HubSpotRetryCount
      Policies:
        - AWSLambdaBasicExecutionRole
        - AWSXRayDaemonWriteAccess
        - SQSPollerPolicy:
            QueueName: !GetAtt ScoringQueue.QueueName
      Events:
        ScoringQueueEvent:
          Type: SQS
          Properties:
            Queue: !GetAtt ScoringQueue.Arn
            BatchSize: 1
            MaximumBatchingWindowInSeconds: 0
            ScalingConfig:
              MaximumConcurrency: 2

  WeeklyMetricsFunction:
    Type: AWS::Serverless::Function
    Properties:
      FunctionName: LATAM-Weekly-Tenant-Metrics-Task
      CodeUri: LATAM-Weekly-Tenant-Metrics-Task/
      Handler: weekly_metrics_function.lambda_handler
      Description: Send LATAM weekly tenant metrics to Feishu.
      Timeout: 120
      MemorySize: 256
      Environment:
        Variables:
          TARGET_DATABASE_URL: !Ref TargetDatabaseURL
          TARGET_TIDB_SSL_CA_PATH: !Ref TargetTiDBSSLCAPath
          FEISHU_WEBHOOK_URL: !Ref FeishuWebhookURL
      Policies:
        - AWSLambdaBasicExecutionRole
        - AWSXRayDaemonWriteAccess

  SummaryFunction:
    Type: AWS::Serverless::Function
    Properties:
      FunctionName: LATAM-Tenants-Summary-Task
      CodeUri: LATAM-Tenants-Summary-Task/
      Handler: summary_function.lambda_handler
      Description: Send the LATAM daily tenant summary to Feishu.
      Timeout: 120
      MemorySize: 512
      Environment:
        Variables:
          TARGET_DATABASE_URL: !Ref TargetDatabaseURL
          TARGET_TIDB_SSL_CA_PATH: !Ref TargetTiDBSSLCAPath
          FEISHU_WEBHOOK_URL: !Ref FeishuWebhookURL
          IMPORT_EVENT_COUNT_TABLE: !Ref ImportEventCountTable
          SUMMARY_SECTION_LIMIT: !Ref SummarySectionLimit
      Policies:
        - AWSLambdaBasicExecutionRole
        - AWSXRayDaemonWriteAccess

  DispatcherScheduleRule:
    Type: AWS::Events::Rule
    Properties:
      Name: LATAM-Tenants-Monitor-Fanout-Dispatcher-Task-Daily
      Description: Dispatch LATAM tenant monitor messages daily.
      ScheduleExpression: !Ref DispatcherScheduleExpression
      State: !Ref ScheduleState
      Targets:
        - Arn: !GetAtt DispatcherFunction.Arn
          Id: DispatcherFunction
          RetryPolicy:
            MaximumEventAgeInSeconds: 21600
            MaximumRetryAttempts: 2

  DispatcherSchedulePermission:
    Type: AWS::Lambda::Permission
    Properties:
      Action: lambda:InvokeFunction
      FunctionName: !Ref DispatcherFunction
      Principal: events.amazonaws.com
      SourceArn: !GetAtt DispatcherScheduleRule.Arn

  SummaryScheduleRule:
    Type: AWS::Events::Rule
    Properties:
      Name: LATAM-Tenants-Summary-Task-Daily
      Description: Send the LATAM daily tenant summary.
      ScheduleExpression: !Ref SummaryScheduleExpression
      State: !Ref ScheduleState
      Targets:
        - Arn: !GetAtt SummaryFunction.Arn
          Id: SummaryFunction
          RetryPolicy:
            MaximumEventAgeInSeconds: 21600
            MaximumRetryAttempts: 2

  SummarySchedulePermission:
    Type: AWS::Lambda::Permission
    Properties:
      Action: lambda:InvokeFunction
      FunctionName: !Ref SummaryFunction
      Principal: events.amazonaws.com
      SourceArn: !GetAtt SummaryScheduleRule.Arn

  WeeklyMetricsScheduleRule:
    Type: AWS::Events::Rule
    Properties:
      Name: LATAM-Weekly-Tenant-Metrics-Task-Weekly
      Description: Send LATAM weekly tenant metrics every Monday.
      ScheduleExpression: !Ref WeeklyMetricsScheduleExpression
      State: !Ref ScheduleState
      Targets:
        - Arn: !GetAtt WeeklyMetricsFunction.Arn
          Id: WeeklyMetricsFunction
          RetryPolicy:
            MaximumEventAgeInSeconds: 21600
            MaximumRetryAttempts: 2

  WeeklyMetricsSchedulePermission:
    Type: AWS::Lambda::Permission
    Properties:
      Action: lambda:InvokeFunction
      FunctionName: !Ref WeeklyMetricsFunction
      Principal: events.amazonaws.com
      SourceArn: !GetAtt WeeklyMetricsScheduleRule.Arn

  DispatcherLogGroup:
    Type: AWS::Logs::LogGroup
    DeletionPolicy: Retain
    UpdateReplacePolicy: Retain
    Properties:
      LogGroupName: !Sub /aws/lambda/${DispatcherFunction}
      RetentionInDays: !Ref LogRetentionDays

  WorkerLogGroup:
    Type: AWS::Logs::LogGroup
    DeletionPolicy: Retain
    UpdateReplacePolicy: Retain
    Properties:
      LogGroupName: !Sub /aws/lambda/${WorkerFunction}
      RetentionInDays: !Ref LogRetentionDays

  ScoringLogGroup:
    Type: AWS::Logs::LogGroup
    DeletionPolicy: Retain
    UpdateReplacePolicy: Retain
    Properties:
      LogGroupName: !Sub /aws/lambda/${ScoringFunction}
      RetentionInDays: !Ref LogRetentionDays

  WeeklyMetricsLogGroup:
    Type: AWS::Logs::LogGroup
    DeletionPolicy: Retain
    UpdateReplacePolicy: Retain
    Properties:
      LogGroupName: !Sub /aws/lambda/${WeeklyMetricsFunction}
      RetentionInDays: !Ref LogRetentionDays

  SummaryLogGroup:
    Type: AWS::Logs::LogGroup
    DeletionPolicy: Retain
    UpdateReplacePolicy: Retain
    Properties:
      LogGroupName: !Sub /aws/lambda/${SummaryFunction}
      RetentionInDays: !Ref LogRetentionDays

  MonitorDLQAlarm:
    Type: AWS::CloudWatch::Alarm
    Properties:
      AlarmName: LATAM-Tenants-Monitor-Fanout-DLQ-Visible
      AlarmDescription: Monitor messages reached the LATAM dead-letter queue.
      Namespace: AWS/SQS
      MetricName: ApproximateNumberOfMessagesVisible
      Dimensions:
        - Name: QueueName
          Value: !GetAtt MonitorDeadLetterQueue.QueueName
      Statistic: Maximum
      Period: 300
      EvaluationPeriods: 1
      Threshold: 1
      ComparisonOperator: GreaterThanOrEqualToThreshold
      TreatMissingData: notBreaching
      AlarmActions: !If [HasAlarmTopic, [!Ref AlarmTopicArn], !Ref AWS::NoValue]

  ScoringDLQAlarm:
    Type: AWS::CloudWatch::Alarm
    Properties:
      AlarmName: LATAM-Tenants-Scoring-DLQ-Visible
      AlarmDescription: Scoring messages reached the LATAM dead-letter queue.
      Namespace: AWS/SQS
      MetricName: ApproximateNumberOfMessagesVisible
      Dimensions:
        - Name: QueueName
          Value: !GetAtt ScoringDeadLetterQueue.QueueName
      Statistic: Maximum
      Period: 300
      EvaluationPeriods: 1
      Threshold: 1
      ComparisonOperator: GreaterThanOrEqualToThreshold
      TreatMissingData: notBreaching
      AlarmActions: !If [HasAlarmTopic, [!Ref AlarmTopicArn], !Ref AWS::NoValue]

  DispatcherErrorAlarm:
    Type: AWS::CloudWatch::Alarm
    Properties: &LambdaErrorAlarm
      AlarmName: LATAM-Tenants-Monitor-Fanout-Dispatcher-Task-Errors
      AlarmDescription: LATAM Dispatcher Lambda reported an error.
      Namespace: AWS/Lambda
      MetricName: Errors
      Dimensions:
        - Name: FunctionName
          Value: !Ref DispatcherFunction
      Statistic: Sum
      Period: 300
      EvaluationPeriods: 1
      Threshold: 1
      ComparisonOperator: GreaterThanOrEqualToThreshold
      TreatMissingData: notBreaching
      AlarmActions: !If [HasAlarmTopic, [!Ref AlarmTopicArn], !Ref AWS::NoValue]

  WorkerErrorAlarm:
    Type: AWS::CloudWatch::Alarm
    Properties:
      <<: *LambdaErrorAlarm
      AlarmName: LATAM-Tenants-Monitor-Fanout-Worker-Task-Errors
      AlarmDescription: LATAM Worker Lambda reported an error.
      Dimensions:
        - Name: FunctionName
          Value: !Ref WorkerFunction

  ScoringErrorAlarm:
    Type: AWS::CloudWatch::Alarm
    Properties:
      <<: *LambdaErrorAlarm
      AlarmName: LATAM-Tenants-Scoring-Task-Errors
      AlarmDescription: LATAM Scoring Lambda reported an error.
      Dimensions:
        - Name: FunctionName
          Value: !Ref ScoringFunction

  WeeklyMetricsErrorAlarm:
    Type: AWS::CloudWatch::Alarm
    Properties:
      <<: *LambdaErrorAlarm
      AlarmName: LATAM-Weekly-Tenant-Metrics-Task-Errors
      AlarmDescription: LATAM Weekly Metrics Lambda reported an error.
      Dimensions:
        - Name: FunctionName
          Value: !Ref WeeklyMetricsFunction

  SummaryErrorAlarm:
    Type: AWS::CloudWatch::Alarm
    Properties:
      <<: *LambdaErrorAlarm
      AlarmName: LATAM-Tenants-Summary-Task-Errors
      AlarmDescription: LATAM Summary Lambda reported an error.
      Dimensions:
        - Name: FunctionName
          Value: !Ref SummaryFunction

Outputs:
  DispatcherFunctionName:
    Value: !Ref DispatcherFunction
  WorkerFunctionName:
    Value: !Ref WorkerFunction
  ScoringFunctionName:
    Value: !Ref ScoringFunction
  WeeklyMetricsFunctionName:
    Value: !Ref WeeklyMetricsFunction
  SummaryFunctionName:
    Value: !Ref SummaryFunction
  MonitorQueueURL:
    Value: !Ref MonitorQueue
  MonitorQueueARN:
    Value: !GetAtt MonitorQueue.Arn
  ScoringQueueURL:
    Value: !Ref ScoringQueue
  ScoringQueueARN:
    Value: !GetAtt ScoringQueue.Arn