AWSTemplateFormatVersion: "2010-09-09"
Transform: AWS::Serverless-2016-10-31
Description: APAC tenant monitoring, scoring, metrics, and summary pipeline.

Parameters:
  DevOpsDBHost:
    Type: String
  DevOpsDBPort:
    Type: String
    Default: "3306"
  DevOpsDBUser:
    Type: String
  DevOpsDBPassword:
    Type: String
    NoEcho: true
  DevOpsDBName:
    Type: String
  TargetDatabaseURL:
    Type: String
    NoEcho: true
  HubSpotPrivateAppToken:
    Type: String
    NoEcho: true
    Default: ""
  HubSpotWritebackPrivateAppToken:
    Type: String
    NoEcho: true
    Default: ""
  HubSpotTargetListID:
    Type: String
    Default: "19544"
  HubSpotRequestTimeoutSeconds:
    Type: String
    Default: "30"
  HubSpotRetryCount:
    Type: String
    Default: "3"
  HubSpotSnapshotCacheTTLSeconds:
    Type: String
    Default: "300"
  APACSalesEmails:
    Type: String
    NoEcho: true
    Default: ""
  GCPServiceAccountSecretArn:
    Type: String
    Description: ARN of the Secrets Manager secret containing the GCP service account JSON.
  DevOpsAPIToken:
    Type: String
    NoEcho: true
  ClinicAPIKey:
    Type: String
    NoEcho: true
  OpenAIAPIKey:
    Type: String
    NoEcho: true
  FeishuWebhookURL:
    Type: String
    NoEcho: true
  OperatorEmail:
    Type: String
    Default: ""
  DevOpsBaseURL:
    Type: String
    Default: https://prod.devops.pingcap.com
  ClinicBaseURL:
    Type: String
    Default: https://clinic.pingcap.com
  OpenAIWebSearchModel:
    Type: String
    Default: gpt-5.4-mini
  DomainIntelligenceTTLDays:
    Type: String
    Default: "30"
  LLMTimeoutSeconds:
    Type: String
    Default: "60"
  BillingRequestIntervalSeconds:
    Type: String
    Default: "2.2"
  BillingRetryCount:
    Type: String
    Default: "3"
  TargetTiDBSSLCAPath:
    Type: String
    Default: ""
  ImportEventCountTable:
    Type: String
    Default: apac_tenant_events
    AllowedPattern: "^[A-Za-z0-9_]+$"
  SummarySectionLimit:
    Type: String
    Default: "10"
    AllowedValues:
      - "10"
  DispatcherScheduleState:
    Type: String
    Default: ENABLED
    AllowedValues:
      - ENABLED
      - DISABLED
  SummaryScheduleState:
    Type: String
    Default: ENABLED
    AllowedValues:
      - ENABLED
      - DISABLED
  WeeklyMetricsScheduleState:
    Type: String
    Default: DISABLED
    AllowedValues:
      - ENABLED
      - DISABLED
  DispatcherScheduleExpression:
    Type: String
    Default: cron(30 0 * * ? *)
  SummaryScheduleExpression:
    Type: String
    Default: cron(0 4 * * ? *)
  WeeklyMetricsScheduleExpression:
    Type: String
    Default: cron(0 7 ? * MON *)
  LogRetentionDays:
    Type: Number
    Default: 30
    AllowedValues:
      - 7
      - 14
      - 30
      - 60
      - 90
  AlarmTopicArn:
    Type: String
    Default: ""

Conditions:
  HasAlarmTopic: !Not [!Equals [!Ref AlarmTopicArn, ""]]

Globals:
  Function:
    Runtime: python3.12
    Architectures:
      - x86_64
    Tracing: Active

Resources:
  MonitorDeadLetterQueue:
    Type: AWS::SQS::Queue
    Properties:
      QueueName: APAC-Tenants-Monitor-Fanout-DLQ
      MessageRetentionPeriod: 1209600
      SqsManagedSseEnabled: true

  MonitorQueue:
    Type: AWS::SQS::Queue
    Properties:
      QueueName: APAC-Tenants-Monitor-Fanout-Queue
      VisibilityTimeout: 5400
      MessageRetentionPeriod: 345600
      ReceiveMessageWaitTimeSeconds: 20
      SqsManagedSseEnabled: true
      RedrivePolicy:
        deadLetterTargetArn: !GetAtt MonitorDeadLetterQueue.Arn
        maxReceiveCount: 5

  ScoringDeadLetterQueue:
    Type: AWS::SQS::Queue
    Properties:
      QueueName: APAC-Tenants-Scoring-DLQ
      MessageRetentionPeriod: 1209600
      SqsManagedSseEnabled: true

  ScoringQueue:
    Type: AWS::SQS::Queue
    Properties:
      QueueName: APAC-Tenants-Scoring-Queue
      VisibilityTimeout: 720
      MessageRetentionPeriod: 345600
      ReceiveMessageWaitTimeSeconds: 20
      SqsManagedSseEnabled: true
      RedrivePolicy:
        deadLetterTargetArn: !GetAtt ScoringDeadLetterQueue.Arn
        maxReceiveCount: 5

  DispatcherFunction:
    Type: AWS::Serverless::Function
    Properties:
      FunctionName: APAC-Tenants-Monitor-Fanout-Dispatcher-Task
      CodeUri: APAC-Tenants-Monitor-Fanout-Dispatcher-Task/
      Handler: dispatcher_function.lambda_handler
      Description: Select APAC tenants and fan out monitor messages.
      Timeout: 900
      MemorySize: 256
      Environment:
        Variables:
          DEVOPS_DB_HOST: !Ref DevOpsDBHost
          DEVOPS_DB_PORT: !Ref DevOpsDBPort
          DEVOPS_DB_USER: !Ref DevOpsDBUser
          DEVOPS_DB_PASSWORD: !Ref DevOpsDBPassword
          DEVOPS_DB_NAME: !Ref DevOpsDBName
          TARGET_DATABASE_URL: !Ref TargetDatabaseURL
          TARGET_TIDB_SSL_CA_PATH: !Ref TargetTiDBSSLCAPath
          HUBSPOT_PRIVATE_APP_TOKEN: !Ref HubSpotPrivateAppToken
          HUBSPOT_TARGET_LIST_ID: !Ref HubSpotTargetListID
          HUBSPOT_REQUEST_TIMEOUT_SECONDS: !Ref HubSpotRequestTimeoutSeconds
          HUBSPOT_RETRY_COUNT: !Ref HubSpotRetryCount
          MONITOR_QUEUE_URL: !Ref MonitorQueue
          EFFECTIVE_OWNER_SYNC_FUNCTION_NAME: !Ref EffectiveSalesOwnerSyncFunction
          EFFECTIVE_OWNER_SYNC_INVOKE_TIMEOUT_SECONDS: "720"
      Policies:
        - AWSLambdaBasicExecutionRole
        - AWSXRayDaemonWriteAccess
        - SQSSendMessagePolicy:
            QueueName: !GetAtt MonitorQueue.QueueName
        - LambdaInvokePolicy:
            FunctionName: !Ref EffectiveSalesOwnerSyncFunction

  WorkerFunction:
    Type: AWS::Serverless::Function
    Properties:
      FunctionName: APAC-Tenants-Monitor-Fanout-Worker-Task
      CodeUri: APAC-Tenants-Monitor-Fanout-Worker-Task/
      Handler: worker_function.lambda_handler
      Description: Enrich one APAC tenant per monitor queue message.
      Timeout: 900
      MemorySize: 512
      ReservedConcurrentExecutions: 2
      Environment:
        Variables:
          TARGET_DATABASE_URL: !Ref TargetDatabaseURL
          SCORING_QUEUE_URL: !Ref ScoringQueue
          DEVOPS_API_TOKEN: !Ref DevOpsAPIToken
          CLINIC_API_KEY: !Ref ClinicAPIKey
          OPERATOR_EMAIL: !Ref OperatorEmail
          OPENAI_API_KEY: !Ref OpenAIAPIKey
          DEVOPS_BASE_URL: !Ref DevOpsBaseURL
          CLINIC_BASE_URL: !Ref ClinicBaseURL
          OPENAI_WEB_SEARCH_MODEL: !Ref OpenAIWebSearchModel
          DOMAIN_INTELLIGENCE_TTL_DAYS: !Ref DomainIntelligenceTTLDays
          LLM_TIMEOUT_SECONDS: !Ref LLMTimeoutSeconds
          BILLING_REQUEST_INTERVAL_SECONDS: !Ref BillingRequestIntervalSeconds
          BILLING_RETRY_COUNT: !Ref BillingRetryCount
          TARGET_TIDB_SSL_CA_PATH: !Ref TargetTiDBSSLCAPath
          APAC_SALES_EMAILS: !Ref APACSalesEmails
          HUBSPOT_TARGET_LIST_ID: !Ref HubSpotTargetListID
          HUBSPOT_SNAPSHOT_CACHE_TTL_SECONDS: !Ref HubSpotSnapshotCacheTTLSeconds
      Policies:
        - AWSLambdaBasicExecutionRole
        - AWSXRayDaemonWriteAccess
        - SQSPollerPolicy:
            QueueName: !GetAtt MonitorQueue.QueueName
        - SQSSendMessagePolicy:
            QueueName: !GetAtt ScoringQueue.QueueName
      Events:
        MonitorQueueEvent:
          Type: SQS
          Properties:
            Queue: !GetAtt MonitorQueue.Arn
            BatchSize: 1
            FunctionResponseTypes:
              - ReportBatchItemFailures
            ScalingConfig:
              MaximumConcurrency: 2

  ScoringFunction:
    Type: AWS::Serverless::Function
    Properties:
      FunctionName: APAC-Tenants-Scoring-Task
      CodeUri: APAC-Tenants-Scoring-Task/
      Handler: scoring_function.lambda_handler
      Description: Score APAC tenants and update daily score snapshots.
      Timeout: 180
      MemorySize: 256
      Environment:
        Variables:
          TARGET_DATABASE_URL: !Ref TargetDatabaseURL
          TARGET_TIDB_SSL_CA_PATH: !Ref TargetTiDBSSLCAPath
          HUBSPOT_WRITEBACK_TOKEN: !Ref HubSpotWritebackPrivateAppToken
          HUBSPOT_WRITEBACK_PROPERTY_NAME: apac_pql_priority_tier
          HUBSPOT_WRITEBACK_TIMEOUT_SECONDS: !Ref HubSpotRequestTimeoutSeconds
          HUBSPOT_WRITEBACK_RETRY_COUNT: !Ref HubSpotRetryCount
      Policies:
        - AWSLambdaBasicExecutionRole
        - AWSXRayDaemonWriteAccess
        - SQSPollerPolicy:
            QueueName: !GetAtt ScoringQueue.QueueName
      Events:
        ScoringQueueEvent:
          Type: SQS
          Properties:
            Queue: !GetAtt ScoringQueue.Arn
            BatchSize: 1
            MaximumBatchingWindowInSeconds: 0
            ScalingConfig:
              MaximumConcurrency: 2

  WeeklyMetricsFunction:
    Type: AWS::Serverless::Function
    Properties:
      FunctionName: APAC-Weekly-Tenant-Metrics-Task
      CodeUri: APAC-Weekly-Tenant-Metrics-Task/
      Handler: weekly_metrics_function.lambda_handler
      Description: Send APAC weekly tenant metrics to Feishu.
      Timeout: 120
      MemorySize: 256
      Environment:
        Variables:
          TARGET_DATABASE_URL: !Ref TargetDatabaseURL
          TARGET_TIDB_SSL_CA_PATH: !Ref TargetTiDBSSLCAPath
          FEISHU_WEBHOOK_URL: !Ref FeishuWebhookURL
      Policies:
        - AWSLambdaBasicExecutionRole
        - AWSXRayDaemonWriteAccess

  SummaryFunction:
    Type: AWS::Serverless::Function
    Properties:
      FunctionName: APAC-Tenants-Summary-Task
      CodeUri: APAC-Tenants-Summary-Task/
      Handler: summary_function.lambda_handler
      Description: Send the APAC daily tenant summary to Feishu.
      Timeout: 120
      MemorySize: 512
      Environment:
        Variables:
          TARGET_DATABASE_URL: !Ref TargetDatabaseURL
          TARGET_TIDB_SSL_CA_PATH: !Ref TargetTiDBSSLCAPath
          FEISHU_WEBHOOK_URL: !Ref FeishuWebhookURL
          IMPORT_EVENT_COUNT_TABLE: !Ref ImportEventCountTable
          SUMMARY_SECTION_LIMIT: !Ref SummarySectionLimit
      Policies:
        - AWSLambdaBasicExecutionRole
        - AWSXRayDaemonWriteAccess

  EffectiveSalesOwnerSyncFunction:
    Type: AWS::Serverless::Function
    Properties:
      FunctionName: APAC-Effective-Sales-Owner-Sync-Task
      CodeUri: APAC-Effective-Sales-Owner-Sync-Task/
      Handler: owner_sync_function.lambda_handler
      Description: Build a versioned APAC Effective Sales Owner snapshot from Nexus CRM.
      Timeout: 720
      MemorySize: 512
      Environment:
        Variables:
          DEVOPS_DB_HOST: !Ref DevOpsDBHost
          DEVOPS_DB_PORT: !Ref DevOpsDBPort
          DEVOPS_DB_USER: !Ref DevOpsDBUser
          DEVOPS_DB_PASSWORD: !Ref DevOpsDBPassword
          DEVOPS_DB_NAME: !Ref DevOpsDBName
          TARGET_DATABASE_URL: !Ref TargetDatabaseURL
          TARGET_TIDB_SSL_CA_PATH: !Ref TargetTiDBSSLCAPath
          GCP_SERVICE_ACCOUNT_SECRET_ARN: !Ref GCPServiceAccountSecretArn
          BIGQUERY_JOB_PROJECT: gcp-resilience-dev
          BIGQUERY_QUERY_TIMEOUT_SECONDS: "600"
          OWNER_SYNC_MAX_TENANTS: "20000"
          OWNER_SYNC_DB_BATCH_SIZE: "500"
          OWNER_SYNC_RETENTION_DAYS: "14"
      Policies:
        - AWSLambdaBasicExecutionRole
        - AWSXRayDaemonWriteAccess
        - Statement:
            - Sid: ReadGCPServiceAccountSecret
              Effect: Allow
              Action:
                - secretsmanager:GetSecretValue
              Resource: !Ref GCPServiceAccountSecretArn

  DispatcherScheduleRule:
    Type: AWS::Events::Rule
    Properties:
      Name: APAC-Tenants-Monitor-Fanout-Dispatcher-Task-Daily
      Description: Dispatch APAC tenant monitor messages daily.
      ScheduleExpression: !Ref DispatcherScheduleExpression
      State: !Ref DispatcherScheduleState
      Targets:
        - Arn: !GetAtt DispatcherFunction.Arn
          Id: DispatcherFunction
          RetryPolicy:
            MaximumEventAgeInSeconds: 21600
            MaximumRetryAttempts: 2

  DispatcherSchedulePermission:
    Type: AWS::Lambda::Permission
    Properties:
      Action: lambda:InvokeFunction
      FunctionName: !Ref DispatcherFunction
      Principal: events.amazonaws.com
      SourceArn: !GetAtt DispatcherScheduleRule.Arn

  SummaryScheduleRule:
    Type: AWS::Events::Rule
    Properties:
      Name: APAC-Tenants-Summary-Task-Daily
      Description: Send the APAC daily tenant summary.
      ScheduleExpression: !Ref SummaryScheduleExpression
      State: !Ref SummaryScheduleState
      Targets:
        - Arn: !GetAtt SummaryFunction.Arn
          Id: SummaryFunction
          RetryPolicy:
            MaximumEventAgeInSeconds: 21600
            MaximumRetryAttempts: 2

  SummarySchedulePermission:
    Type: AWS::Lambda::Permission
    Properties:
      Action: lambda:InvokeFunction
      FunctionName: !Ref SummaryFunction
      Principal: events.amazonaws.com
      SourceArn: !GetAtt SummaryScheduleRule.Arn

  WeeklyMetricsScheduleRule:
    Type: AWS::Events::Rule
    Properties:
      Name: APAC-Weekly-Tenant-Metrics-Task-Weekly
      Description: Send APAC weekly tenant metrics every Monday.
      ScheduleExpression: !Ref WeeklyMetricsScheduleExpression
      State: !Ref WeeklyMetricsScheduleState
      Targets:
        - Arn: !GetAtt WeeklyMetricsFunction.Arn
          Id: WeeklyMetricsFunction
          RetryPolicy:
            MaximumEventAgeInSeconds: 21600
            MaximumRetryAttempts: 2

  WeeklyMetricsSchedulePermission:
    Type: AWS::Lambda::Permission
    Properties:
      Action: lambda:InvokeFunction
      FunctionName: !Ref WeeklyMetricsFunction
      Principal: events.amazonaws.com
      SourceArn: !GetAtt WeeklyMetricsScheduleRule.Arn

  DispatcherLogGroup:
    Type: AWS::Logs::LogGroup
    DeletionPolicy: Retain
    UpdateReplacePolicy: Retain
    Properties:
      LogGroupName: !Sub /aws/lambda/${DispatcherFunction}
      RetentionInDays: !Ref LogRetentionDays

  WorkerLogGroup:
    Type: AWS::Logs::LogGroup
    DeletionPolicy: Retain
    UpdateReplacePolicy: Retain
    Properties:
      LogGroupName: !Sub /aws/lambda/${WorkerFunction}
      RetentionInDays: !Ref LogRetentionDays

  ScoringLogGroup:
    Type: AWS::Logs::LogGroup
    DeletionPolicy: Retain
    UpdateReplacePolicy: Retain
    Properties:
      LogGroupName: !Sub /aws/lambda/${ScoringFunction}
      RetentionInDays: !Ref LogRetentionDays

  WeeklyMetricsLogGroup:
    Type: AWS::Logs::LogGroup
    DeletionPolicy: Retain
    UpdateReplacePolicy: Retain
    Properties:
      LogGroupName: !Sub /aws/lambda/${WeeklyMetricsFunction}
      RetentionInDays: !Ref LogRetentionDays

  SummaryLogGroup:
    Type: AWS::Logs::LogGroup
    DeletionPolicy: Retain
    UpdateReplacePolicy: Retain
    Properties:
      LogGroupName: !Sub /aws/lambda/${SummaryFunction}
      RetentionInDays: !Ref LogRetentionDays

  EffectiveSalesOwnerSyncLogGroup:
    Type: AWS::Logs::LogGroup
    DeletionPolicy: Retain
    UpdateReplacePolicy: Retain
    Properties:
      LogGroupName: !Sub /aws/lambda/${EffectiveSalesOwnerSyncFunction}
      RetentionInDays: !Ref LogRetentionDays

  MonitorDLQAlarm:
    Type: AWS::CloudWatch::Alarm
    Properties:
      AlarmName: APAC-Tenants-Monitor-Fanout-DLQ-Visible
      AlarmDescription: Monitor messages reached the APAC dead-letter queue.
      Namespace: AWS/SQS
      MetricName: ApproximateNumberOfMessagesVisible
      Dimensions:
        - Name: QueueName
          Value: !GetAtt MonitorDeadLetterQueue.QueueName
      Statistic: Maximum
      Period: 300
      EvaluationPeriods: 1
      Threshold: 1
      ComparisonOperator: GreaterThanOrEqualToThreshold
      TreatMissingData: notBreaching
      AlarmActions: !If [HasAlarmTopic, [!Ref AlarmTopicArn], !Ref AWS::NoValue]

  ScoringDLQAlarm:
    Type: AWS::CloudWatch::Alarm
    Properties:
      AlarmName: APAC-Tenants-Scoring-DLQ-Visible
      AlarmDescription: Scoring messages reached the APAC dead-letter queue.
      Namespace: AWS/SQS
      MetricName: ApproximateNumberOfMessagesVisible
      Dimensions:
        - Name: QueueName
          Value: !GetAtt ScoringDeadLetterQueue.QueueName
      Statistic: Maximum
      Period: 300
      EvaluationPeriods: 1
      Threshold: 1
      ComparisonOperator: GreaterThanOrEqualToThreshold
      TreatMissingData: notBreaching
      AlarmActions: !If [HasAlarmTopic, [!Ref AlarmTopicArn], !Ref AWS::NoValue]

  DispatcherErrorAlarm:
    Type: AWS::CloudWatch::Alarm
    Properties: &LambdaErrorAlarm
      AlarmName: APAC-Tenants-Monitor-Fanout-Dispatcher-Task-Errors
      AlarmDescription: APAC Dispatcher Lambda reported an error.
      Namespace: AWS/Lambda
      MetricName: Errors
      Dimensions:
        - Name: FunctionName
          Value: !Ref DispatcherFunction
      Statistic: Sum
      Period: 300
      EvaluationPeriods: 1
      Threshold: 1
      ComparisonOperator: GreaterThanOrEqualToThreshold
      TreatMissingData: notBreaching
      AlarmActions: !If [HasAlarmTopic, [!Ref AlarmTopicArn], !Ref AWS::NoValue]

  WorkerErrorAlarm:
    Type: AWS::CloudWatch::Alarm
    Properties:
      <<: *LambdaErrorAlarm
      AlarmName: APAC-Tenants-Monitor-Fanout-Worker-Task-Errors
      AlarmDescription: APAC Worker Lambda reported an error.
      Dimensions:
        - Name: FunctionName
          Value: !Ref WorkerFunction

  ScoringErrorAlarm:
    Type: AWS::CloudWatch::Alarm
    Properties:
      <<: *LambdaErrorAlarm
      AlarmName: APAC-Tenants-Scoring-Task-Errors
      AlarmDescription: APAC Scoring Lambda reported an error.
      Dimensions:
        - Name: FunctionName
          Value: !Ref ScoringFunction

  WeeklyMetricsErrorAlarm:
    Type: AWS::CloudWatch::Alarm
    Properties:
      <<: *LambdaErrorAlarm
      AlarmName: APAC-Weekly-Tenant-Metrics-Task-Errors
      AlarmDescription: APAC Weekly Metrics Lambda reported an error.
      Dimensions:
        - Name: FunctionName
          Value: !Ref WeeklyMetricsFunction

  SummaryErrorAlarm:
    Type: AWS::CloudWatch::Alarm
    Properties:
      <<: *LambdaErrorAlarm
      AlarmName: APAC-Tenants-Summary-Task-Errors
      AlarmDescription: APAC Summary Lambda reported an error.
      Dimensions:
        - Name: FunctionName
          Value: !Ref SummaryFunction

  EffectiveSalesOwnerSyncErrorAlarm:
    Type: AWS::CloudWatch::Alarm
    Properties:
      <<: *LambdaErrorAlarm
      AlarmName: APAC-Effective-Sales-Owner-Sync-Task-Errors
      AlarmDescription: APAC Effective Sales Owner Sync Lambda reported an error.
      Dimensions:
        - Name: FunctionName
          Value: !Ref EffectiveSalesOwnerSyncFunction

Outputs:
  DispatcherFunctionName:
    Value: !Ref DispatcherFunction
  WorkerFunctionName:
    Value: !Ref WorkerFunction
  ScoringFunctionName:
    Value: !Ref ScoringFunction
  WeeklyMetricsFunctionName:
    Value: !Ref WeeklyMetricsFunction
  SummaryFunctionName:
    Value: !Ref SummaryFunction
  EffectiveSalesOwnerSyncFunctionName:
    Value: !Ref EffectiveSalesOwnerSyncFunction
  MonitorQueueURL:
    Value: !Ref MonitorQueue
  MonitorQueueARN:
    Value: !GetAtt MonitorQueue.Arn
  ScoringQueueURL:
    Value: !Ref ScoringQueue
  ScoringQueueARN:
    Value: !GetAtt ScoringQueue.Arn