groups:
- name: dataplane-manager-resource-operations
  rules:
  - alert: DataplaneCreationDurationTooLong
    annotations:
      message: The Dataplane Resource {{ $labels.name }} in {{ $labels.tidb_cluster_id }}/{{ $labels.tenant_id }} creation time exceeds 25 minutes
    expr: |
      max(dataplane_manager_dataplane_info{service="dataplane-controller", status=~"Pending|ReadyForCreate|Creating|Failed"})by(name, region, tenant_id) >= 1
    for: 30m
    labels:
      severity: major
      response_class: ticket
      component: dataplane-controller
      oncall_team: operator
  - alert: DataplaneCreationDurationTooLong
    annotations:
      message: The Dataplane Resource {{ $labels.name }} in {{ $labels.tidb_cluster_id }}/{{ $labels.tenant_id }} creation time exceeds 35 minutes
    expr: |
      max(dataplane_manager_dataplane_info{service="dataplane-controller", status=~"Pending|ReadyForCreate|Creating|Failed"})by(name, region, tenant_id) >= 1
    for: 35m
    labels:
      severity: critical
      response_class: page
      component: dataplane-controller
      oncall_team: operator
  - alert: DataplaneDeletionDurationTooLong
    annotations:
      message: The Dataplane Resource {{ $labels.name }} in {{ $labels.tidb_cluster_id }}/{{ $labels.tenant_id }} deletion time exceeds 30 minutes
    expr: |
      max(dataplane_manager_dataplane_info{service="dataplane-controller", status=~"Deleting"})by(name, region, tenant_id) >= 1
    for: 30m
    labels:
      severity: major
      response_class: ticket
      component: dataplane-controller
      oncall_team: operator
  - alert: VPCCreationDurationTooLong 
    annotations:
      message: The VPC Resource {{ $labels.name }} in {{ $labels.tidb_cluster_id }}/{{ $labels.tenant_id }} creation time exceeds 5 minutes
    expr: |
      max(dataplane_manager_vpc_info{service="dataplane-controller", status=~"Pending|ReadyForCreate|Creating|Failed"})by(name, region, tenant_id) >= 1
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: dataplane-controller
      oncall_team: operator
  - alert: VPCCreationDurationTooLong
    annotations:
      message: The VPC Resource {{ $labels.name }} in {{ $labels.tidb_cluster_id }}/{{ $labels.tenant_id }} creation time exceeds 10 minutes
    expr: |
      max(dataplane_manager_vpc_info{service="dataplane-controller", status=~"Pending|ReadyForCreate|Creating|Failed"})by(name, region, tenant_id) >= 1
    for: 10m
    labels:
      severity: critical
      response_class: page
      component: dataplane-controller
      oncall_team: operator
  - alert: VPCDeletionDurationTooLong
    annotations:
      message: The VPC Resource {{ $labels.name }} in {{ $labels.tidb_cluster_id }}/{{ $labels.tenant_id }} deletion time exceeds 20 minutes
    expr: |
      max(dataplane_manager_vpc_info{service="dataplane-controller", status=~"Deleting"})by(name, region, tenant_id) >= 1
    for: 20m
    labels:
      severity: major
      response_class: ticket
      component: dataplane-controller
      oncall_team: operator
  - alert: InternetResourceCreationDurationTooLong
    annotations:
      message: The Internet Resource {{ $labels.name }} in {{ $labels.tidb_cluster_id }}/{{ $labels.tenant_id }} creation time exceeds 10 minutes
    expr: |
      max(dataplane_manager_internet_info{service="dataplane-controller", status=~"ReadyForCreate|Creating|Failed"})by(name, region, tenant_id) >= 1
    for: 10m
    labels:
      severity: major
      response_class: ticket
      component: dataplane-controller
      oncall_team: operator
  - alert: InternetResourceCreationDurationTooLong
    annotations:
      message: The Internet Resource {{ $labels.name }} in {{ $labels.tidb_cluster_id }}/{{ $labels.tenant_id }} creation time exceeds 20 minutes
    expr: |
      max(dataplane_manager_internet_info{service="dataplane-controller", status=~"ReadyForCreate|Creating|Failed"})by(name, region, tenant_id) >= 1
    for: 20m
    labels:
      severity: critical
      response_class: page
      component: dataplane-controller
      oncall_team: operator
  - alert: InternetResourceDeletionDurationTooLong
    annotations:
      message: The Internet Resource {{ $labels.name }} in {{ $labels.tidb_cluster_id }}/{{ $labels.tenant_id }} deletion time exceeds 20 minutes
    expr: |
      max(dataplane_manager_internet_info{service="dataplane-controller", status=~"Deleting"})by(name, region, tenant_id) >= 1
    for: 20m
    labels:
      severity: major
      response_class: ticket
      component: dataplane-controller
      oncall_team: operator
  - alert: InternetResourceDeletionDurationTooLong
    annotations:
      message: The Internet Resource {{ $labels.name }} in {{ $labels.tidb_cluster_id }}/{{ $labels.tenant_id }} deletion time exceeds 24 hours
    expr: |
      max(dataplane_manager_internet_info{service="dataplane-controller", status=~"Deleting"})by(name, region, tenant_id) >= 1
    for: 24h
    labels:
      severity: critical
      response_class: page
      component: dataplane-controller
      oncall_team: operator
  - alert: K8sCreationDurationTooLong
    annotations:
      message: K8s Resource(APPBox CR) {{ $labels.name }} in {{ $labels.tidb_cluster_id }}/{{ $labels.tenant_id }} creation time exceeds 25 minutes
    expr: |
      max(dataplane_manager_appbox_info{service="dataplane-controller", service_type!="byoc", status=~"ReadyForCreate|Creating|Failed"})by(name, region, tenant_id) >= 1
    for: 25m
    labels:
      severity: major
      response_class: ticket
      component: dataplane-controller
      oncall_team: operator
  - alert: K8sCreationDurationTooLong
    annotations:
      message: K8s Resource(APPBox CR) {{ $labels.name }} in {{ $labels.tidb_cluster_id }}/{{ $labels.tenant_id }} creation time exceeds 30 minutes
    expr: |
      max(dataplane_manager_appbox_info{service="dataplane-controller", service_type!="byoc", status=~"ReadyForCreate|Creating|Failed"})by(name, region, tenant_id) >= 1
    for: 30m
    labels:
      severity: critical
      response_class: page
      component: dataplane-controller
      oncall_team: operator
  - alert: BYOCK8sCreationDurationTooLong
    annotations:
      message: BYOC K8s Resource(APPBox CR) {{ $labels.name }} in {{ $labels.tidb_cluster_id }}/{{ $labels.tenant_id }} creation time exceeds 30 minutes
    expr: |
      max(dataplane_manager_appbox_info{service="dataplane-controller", service_type="byoc", status=~"ReadyForCreate|Creating|Failed"})by(name, region, tenant_id) >= 1
    for: 30m
    labels:
      severity: major
      response_class: ticket
      component: dataplane-controller
      oncall_team: operator
  - alert: BYOCK8sCreationDurationTooLong
    annotations:
      message: BYOC K8s Resource(APPBox CR) {{ $labels.name }} in {{ $labels.tidb_cluster_id }}/{{ $labels.tenant_id }} creation time exceeds 40 minutes
    expr: |
      max(dataplane_manager_appbox_info{service="dataplane-controller", service_type="byoc", status=~"ReadyForCreate|Creating|Failed"})by(name, region, tenant_id) >= 1
    for: 40m
    labels:
      severity: critical
      response_class: page
      component: dataplane-controller
      oncall_team: operator
  - alert: K8sDeletionDurationTooLong
    annotations:
      message: K8s Resource(APPBox CR) {{ $labels.name }} in {{ $labels.tidb_cluster_id }}/{{ $labels.tenant_id }} deletion time exceeds 30 minutes
    expr: |
      max(dataplane_manager_appbox_info{service="dataplane-controller", status=~"Deleting"})by(name, region, tenant_id) >= 1
    for: 30m
    labels:
      severity: major
      response_class: ticket
      component: dataplane-controller
      oncall_team: operator
  - alert: K8sDeletionDurationTooLong
    annotations:
      message: K8s Resource(APPBox CR) {{ $labels.name }} in {{ $labels.tidb_cluster_id }}/{{ $labels.tenant_id }} deletion time exceeds 24 hours
    expr: |
      max(dataplane_manager_appbox_info{service="dataplane-controller", status=~"Deleting"})by(name, region, tenant_id) >= 1
    for: 24h
    labels:
      severity: critical
      response_class: page
      component: dataplane-controller
      oncall_team: operator
  - alert: PrivateLinkServiceCreationDurationTooLong
    annotations:
      message: The Private Link Service Resource {{ $labels.name }} in {{ $labels.tidb_cluster_id }}/{{ $labels.tenant_id }} creation time exceeds 15 minutes
    expr: |
      max(dataplane_manager_privatelinkservice_info{service="dataplane-controller", status=~"Pending|Creating|Failed"})by(name, region, tenant_id) >= 1
    for: 15m
    labels:
      severity: major
      response_class: ticket
      component: dataplane-controller
      oncall_team: operator
  - alert: PrivateLinkServiceCreationDurationTooLong
    annotations:
      message: The Private Link Service Resource {{ $labels.name }} in {{ $labels.tidb_cluster_id }}/{{ $labels.tenant_id }} creation time exceeds 30 minutes
    expr: |
      max(dataplane_manager_privatelinkservice_info{service="dataplane-controller", status=~"Pending|Creating|Failed"})by(name, region, tenant_id) >= 1
    for: 30m
    labels:
      severity: critical
      response_class: page
      component: dataplane-controller
      oncall_team: operator
  - alert: PrivateLinkEndpointCreationDurationTooLong
    annotations:
      message: The Private Link Endpoint Resource {{ $labels.name }} (provider={{ $labels.provider }}, region={{ $labels.region }}, tenant={{ $labels.tenant_id }}) creation time exceeds 10 minutes
    expr: |
      max(dataplane_manager_privatelinkendpoint_info{service="dataplane-controller", status=~"^$|Pending|Creating|Failed"})by(name, provider, region, tenant_id) >= 1
    for: 10m
    labels:
      severity: major
      response_class: ticket
      component: dataplane-controller
      oncall_team: operator
  - alert: PrivateLinkEndpointCreationDurationTooLong
    annotations:
      message: The Private Link Endpoint Resource {{ $labels.name }} (provider={{ $labels.provider }}, region={{ $labels.region }}, tenant={{ $labels.tenant_id }}) creation time exceeds 15 minutes
    expr: |
      max(dataplane_manager_privatelinkendpoint_info{service="dataplane-controller", status=~"^$|Pending|Creating|Failed"})by(name, provider, region, tenant_id) >= 1
    for: 15m
    labels:
      severity: critical
      response_class: page
      component: dataplane-controller
      oncall_team: operator
  - alert: PrivateLinkServiceDeletionDurationTooLong
    annotations:
      message: The Private Link Service Resource {{ $labels.name }} in {{ $labels.tidb_cluster_id }}/{{ $labels.tenant_id }} deletion time exceeds 20 minutes
    expr: |
      max(dataplane_manager_privatelinkservice_info{service="dataplane-controller", status=~"Deleting"})by(name, region, tenant_id) >= 1
    for: 20m
    labels:
      severity: major
      response_class: ticket
      component: dataplane-controller
      oncall_team: operator
  - alert: LoadBalancerCreationDurationTooLong
    annotations:
      message: The Load Balancer Resource {{ $labels.name }} in {{ $labels.tidb_cluster_id }}/{{ $labels.tenant_id }} creation time exceeds 10 minutes
    expr: |
      max(dataplane_manager_loadbalancer_info{service="dataplane-controller", status=~"Pending|ReadyForCreate|Creating|Failed"})by(name, region, tenant_id) >= 1
    for: 10m
    labels:
      severity: major
      response_class: ticket
      component: dataplane-controller
      oncall_team: operator
  - alert: LoadBalancerCreationDurationTooLong
    annotations:
      message: The Load Balancer Resource {{ $labels.name }} in {{ $labels.tidb_cluster_id }}/{{ $labels.tenant_id }} creation time exceeds 20 minutes
    expr: |
      max(dataplane_manager_loadbalancer_info{service="dataplane-controller", status=~"Pending|ReadyForCreate|Creating|Failed"})by(name, region, tenant_id) >= 1
    for: 20m
    labels:
      severity: critical
      response_class: page
      component: dataplane-controller
      oncall_team: operator
  - alert: PublicLoadBalancerDNSNotReachable
    annotations:
      message: Public Load Balancer DNS {{ $labels.dns }} for resource {{ $labels.name }} (namespace={{ $labels.namespace }}, tenant={{ $labels.tenant_id }}, cluster={{ $labels.cluster_id }}, tidb_group={{ $labels.tidb_group_id }}) is not reachable from public DNS for 30 minutes. Query the DNS name against public resolvers such as 1.1.1.1 and 8.8.8.8, inspect the LoadBalancer and Route53 Record CRs, and verify the Route53 hosted-zone record and parent-domain delegation or registration. Restore the record or delegation, then confirm the DNS name returns an A record.
    expr: |
      max by(name, namespace, tenant_id, cluster_id, tidb_group_id, dns, reason) (
        dataplane_manager_public_loadbalancer_dns_reachable{service="dataplane-controller", reason="DNS_NOT_REACHABLE"}
      ) < 1
    for: 30m
    labels:
      severity: critical
      response_class: page
      component: dataplane-controller
      oncall_team: operator
  - alert: PublicLoadBalancerDNSProbeFailed
    annotations:
      message: Public Load Balancer DNS probe failed for {{ $labels.dns }} on resource {{ $labels.name }} (namespace={{ $labels.namespace }}, tenant={{ $labels.tenant_id }}, cluster={{ $labels.cluster_id }}, tidb_group={{ $labels.tidb_group_id }}, reason={{ $labels.reason }}) for 30 minutes. Inspect dataplane-controller logs for the per-resolver error, then verify DNS, TLS, and HTTPS egress from the DPM pod to dns.google and cloudflare-dns.com. Restore probe connectivity, wait for the automatic retry, and confirm the metric returns 1; investigate Route53 only if public resolver queries also fail.
    expr: |
      max by(name, namespace, tenant_id, cluster_id, tidb_group_id, dns, reason) (
        dataplane_manager_public_loadbalancer_dns_reachable{service="dataplane-controller", reason="DNS_PROBE_FAILED"}
      ) < 1
    for: 30m
    labels:
      severity: critical
      response_class: page
      component: dataplane-controller
      oncall_team: operator
  - alert: LoadBalancerDeletionDurationTooLong
    annotations:
      message: The Load Balancer Resource {{ $labels.name }} in {{ $labels.tidb_cluster_id }}/{{ $labels.tenant_id }} deletion time exceeds 15 minutes
    expr: |
      max(dataplane_manager_loadbalancer_info{service="dataplane-controller", status=~"Deleting"})by(name, region, tenant_id) >= 1
    for: 15m
    labels:
      severity: major
      response_class: ticket
      component: dataplane-controller
      oncall_team: operator
  - alert: LoadBalancerDeletionDurationTooLong
    annotations:
      message: The Load Balancer Resource {{ $labels.name }} in {{ $labels.tidb_cluster_id }}/{{ $labels.tenant_id }} deletion time exceeds 24 hours
    expr: |
      max(dataplane_manager_loadbalancer_info{service="dataplane-controller", status=~"Deleting"})by(name, region, tenant_id) >= 1
    for: 24h
    labels:
      severity: critical
      response_class: page
      component: dataplane-controller
      oncall_team: operator
  - alert: BucketDeletionDurationTooLong
    annotations:
      message: The Bucket Resource {{ $labels.name }} in {{ $labels.tidb_cluster_id }}/{{ $labels.tenant_id }} deletion time exceeds 24 hours
    expr: |
      max(dataplane_manager_bucket_info{service="dataplane-controller", status=~"Deleting"})by(name, region, tenant_id) >= 1
    for: 24h
    labels:
      severity: major
      response_class: ticket
      component: dataplane-controller
      oncall_team: operator