package dashboard

import (
	"context"
	"fmt"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/go-logr/logr"
	promapi "github.com/prometheus/client_golang/api"
	promv1 "github.com/prometheus/client_golang/api/prometheus/v1"
	prommodel "github.com/prometheus/common/model"
	"github.com/tidbcloud/devops/entity"
)

const (
	TimeRangePlaceholder = "$TIME_RANGE"
	TimeRange7d          = "7d"
)

type SLOCache struct {
	basePrometheusPrefix string
	syncInterval         *time.Duration

	logger logr.Logger
	cache  []entity.ApplicationSLOSummary
	lock   sync.Mutex
	once   sync.Once
}

func newSLOCache(logger logr.Logger,
	basePrometheusPrefix string,
	syncInterval *time.Duration,
) *SLOCache {
	return &SLOCache{
		logger:               logger.WithName("SLOCache"),
		basePrometheusPrefix: basePrometheusPrefix,
		syncInterval:         syncInterval,
		cache:                make([]entity.ApplicationSLOSummary, 0),
	}
}

func (s *SLOCache) GetSLOCache() ([]entity.ApplicationSLOSummary, error) {
	if len(s.cache) > 0 {
		return s.cache, nil
	}

	s.lock.Lock()
	defer s.lock.Unlock()
	if len(s.cache) > 0 {
		return s.cache, nil
	}

	cache, err := s.sync()
	if err != nil {
		return nil, err
	}
	s.cache = cache
	s.once.Do(func() {
		go s.syncScheduler()
	})
	return s.cache, nil
}

type ApplicationSLOResult struct {
	Application string
	SLOName     string
	Met         bool
	Error       error
}

func (s *SLOCache) syncScheduler() {
	syncTicker := time.NewTicker(*s.syncInterval)
	for {
		<-syncTicker.C
		s.lock.Lock()
		cache, err := s.sync()
		if err != nil {
			s.logger.Error(err, "sync SLO failed")
			s.lock.Unlock()
			continue
		}
		s.cache = cache
		s.lock.Unlock()
	}
}

func (s *SLOCache) sync() ([]entity.ApplicationSLOSummary, error) {
	s.logger.Info("syncing SLO")
	sloChan := make(chan ApplicationSLOResult, 100)
	go func() {
		wg := sync.WaitGroup{}
		for application, item := range SLORegistry {
			for _, slo := range item {
				wg.Add(1)
				go func(application string, slo SLOQuery) {
					defer wg.Done()
					met, err := s.metSLO(strings.ReplaceAll(slo.Query, TimeRangePlaceholder, TimeRange7d))
					sloChan <- ApplicationSLOResult{
						Application: application,
						SLOName:     slo.Name,
						Met:         met,
						Error:       err,
					}
				}(application, slo)
			}
		}
		wg.Wait()
		close(sloChan)
	}()

	result := make(map[string][]entity.SLO)
	for sloResult := range sloChan {
		if _, ok := result[sloResult.Application]; !ok {
			result[sloResult.Application] = make([]entity.SLO, 0)
		}
		slo := entity.SLO{
			Name: sloResult.SLOName,
			Met:  sloResult.Met,
		}
		if sloResult.Error != nil {
			slo.Reason = fmt.Sprintf("fetch SLO failed: %v, application: %s, slo: %s", sloResult.Error, sloResult.Application, sloResult.SLOName)
		}
		result[sloResult.Application] = append(result[sloResult.Application], slo)
	}

	resp := make([]entity.ApplicationSLOSummary, 0)
	for application, slos := range result {
		resp = append(resp, entity.ApplicationSLOSummary{
			Application: application,
			SLOs:        slos,
		})
	}
	sort.Slice(resp, func(i, j int) bool {
		return resp[i].Application < resp[j].Application
	})
	s.logger.Info("sync SLO done")
	return resp, nil
}

func (s *SLOCache) metSLO(query string) (bool, error) {
	client, err := promapi.NewClient(promapi.Config{
		Address: s.basePrometheusPrefix,
	})
	if err != nil {
		return false, fmt.Errorf("error creating client: %v", err)
	}
	v1api := promv1.NewAPI(client)
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	result, _, err := v1api.Query(ctx,
		query,
		time.Now(),
		promv1.WithTimeout(5*time.Second),
	)
	if err != nil {
		return false, fmt.Errorf("error querying Prometheus: %v", err)
	}
	vector, ok := result.(prommodel.Vector)
	if !ok {
		return false, fmt.Errorf("prometheus query result is not vector")
	}
	if len(vector) == 0 {
		return false, fmt.Errorf("prometheus query result is empty, result: %s", vector.String())
	}
	if len(vector) > 1 {
		return false, fmt.Errorf("prometheus query result is not scalar, result: %s", vector.String())
	}
	return vector[0].Value > 0, nil
}

type SLOQuery struct {
	Name  string
	Query string
}

var SLORegistry = map[string][]SLOQuery{
	"Account": {
		{
			Name:  "HTTP API Availability",
			Query: fmt.Sprintf("sum(rate(kong_custom_http_status{kube_service=~\"(nightly-ms|staging-ms|prod-ms).account-central-http.*\", code!~\"5.*\"}[%s])) / sum(rate(kong_custom_http_status{kube_service=~\"(nightly-ms|staging-ms|prod-ms).account-central-http.*\"}[%s])) >= bool 0.999", TimeRangePlaceholder, TimeRangePlaceholder),
		},
		{
			Name:  "HTTP API Latency p90",
			Query: fmt.Sprintf("sum(rate(kong_custom_request_latency_ms_bucket{kube_service=~\"(nightly-ms|staging-ms|prod-ms).account-central-http.*\", le=\"700\"}[%s]))\n/\nsum(rate(kong_custom_request_latency_ms_count{kube_service=~\"(nightly-ms|staging-ms|prod-ms).account-central-http.*\"}[%s])) >= bool 0.90", TimeRangePlaceholder, TimeRangePlaceholder),
		},
		{
			Name:  "HTTP API Latency p99",
			Query: fmt.Sprintf("sum(rate(kong_custom_request_latency_ms_bucket{kube_service=~\"(nightly-ms|staging-ms|prod-ms).account-central-http.*\", le=\"5000\"}[%s]))\n/\nsum(rate(kong_custom_request_latency_ms_count{kube_service=~\"(nightly-ms|staging-ms|prod-ms).account-central-http.*\"}[%s])) >= bool 0.99", TimeRangePlaceholder, TimeRangePlaceholder),
		},
		{
			Name:  "GRPC API Latency p90",
			Query: fmt.Sprintf("sum(rate(grpc_server_handling_seconds_bucket{namespace=~\"nightly-ms|staging-ms|prod-ms\", pod=~\"account-provider.*\", grpc_type=\"unary\", le=\"0.5\"}[%s]))\n/\nsum(rate(grpc_server_handling_seconds_count{namespace=~\"nightly-ms|staging-ms|prod-ms\", pod=~\"account-provider.*\", grpc_type=\"unary\"}[%s])) >= bool 0.90", TimeRangePlaceholder, TimeRangePlaceholder),
		},
		{
			Name:  "GRPC API Latency p99",
			Query: fmt.Sprintf("sum(rate(grpc_server_handling_seconds_bucket{namespace=~\"nightly-ms|staging-ms|prod-ms\", pod=~\"account-provider.*\", grpc_type=\"unary\", le=\"5\"}[%s]))\n/\nsum(rate(grpc_server_handling_seconds_count{namespace=~\"nightly-ms|staging-ms|prod-ms\", pod=~\"account-provider.*\", grpc_type=\"unary\"}[%s])) >= bool 0.99", TimeRangePlaceholder, TimeRangePlaceholder),
		},
	},
	"AI Power": {
		{
			Name:  "API Availability",
			Query: fmt.Sprintf("sum(rate(request_latency_seconds_count{job=\"edaservice\", status_code!~\"5.*\"}[%s]))/sum(rate(request_latency_seconds_count{job=\"edaservice\"}[%s])) >= bool 0.995", TimeRangePlaceholder, TimeRangePlaceholder),
		},
		{
			Name:  "API Latency p90",
			Query: fmt.Sprintf("sum(rate(request_latency_seconds_bucket{job=\"edaservice\", le=\"2.5\"}[%s]))/sum(rate(request_latency_seconds_count{job=\"edaservice\"}[%s])) >= bool 0.90", TimeRangePlaceholder, TimeRangePlaceholder),
		},
		{
			Name:  "API Latency p99",
			Query: fmt.Sprintf("sum(rate(request_latency_seconds_bucket{job=\"edaservice\", le=\"5.0\"}[%s]))/sum(rate(request_latency_seconds_count{job=\"edaservice\"}[%s])) >= bool 0.99", TimeRangePlaceholder, TimeRangePlaceholder),
		},
		{
			Name:  "Worker Availability",
			Query: fmt.Sprintf("sum(rate(biz_latency_seconds_count{job=\"edaservice-worker\", status_code!~\"5.*\"}[%s]))/sum(rate(biz_latency_seconds_count{job=\"edaservice-worker\"}[%s])) >= bool 0.95", TimeRangePlaceholder, TimeRangePlaceholder),
		},
		{
			Name:  "Worker Latency p90 (Chat2DataGPT35Turbo)",
			Query: fmt.Sprintf("(sum(rate(biz_latency_seconds_bucket{job=\"edaservice-worker\",name=\"task_chat2data|task_chat2sql\",llm_model=\"gpt-3.5-turbo\",le=\"20.0\"}[%s]))/sum(rate(biz_latency_seconds_count{job=\"edaservice-worker\",name=\"task_chat2data|task_chat2sql\",llm_model=\"gpt-3.5-turbo\"}[%s]))) or on() vector(1) >= bool 0.90", TimeRangePlaceholder, TimeRangePlaceholder),
		},
		{
			Name:  "Worker Latency p99 (Chat2DataGPT35Turbo)",
			Query: fmt.Sprintf("(sum(rate(biz_latency_seconds_bucket{job=\"edaservice-worker\",name=\"task_chat2data|task_chat2sql\",llm_model=\"gpt-3.5-turbo\",le=\"60.0\"}[%s]))/sum(rate(biz_latency_seconds_count{job=\"edaservice-worker\",name=\"task_chat2data|task_chat2sql\",llm_model=\"gpt-3.5-turbo\"}[%s]))) or on() vector(1) >= bool 0.99", TimeRangePlaceholder, TimeRangePlaceholder),
		},
		{
			Name:  "Worker Latency p90 (Chat2DataGPT4)",
			Query: fmt.Sprintf("(sum(rate(biz_latency_seconds_bucket{job=\"edaservice-worker\",name=\"task_chat2data|task_chat2sql\",llm_model=\"gpt-4\",le=\"100.0\"}[%s]))/sum(rate(biz_latency_seconds_count{job=\"edaservice-worker\",name=\"task_chat2data|task_chat2sql\",llm_model=\"gpt-4\"}[%s]))) or on() vector(1) >= bool 0.90", TimeRangePlaceholder, TimeRangePlaceholder),
		},
		{
			Name:  "Worker Latency p99 (Chat2DataGPT4)",
			Query: fmt.Sprintf("(sum(rate(biz_latency_seconds_bucket{job=\"edaservice-worker\",name=\"task_chat2data|task_chat2sql\",llm_model=\"gpt-4\",le=\"300.0\"}[%s]))/sum(rate(biz_latency_seconds_count{job=\"edaservice-worker\",name=\"task_chat2data|task_chat2sql\",llm_model=\"gpt-4\"}[%s]))) or on() vector(1) >= bool 0.99", TimeRangePlaceholder, TimeRangePlaceholder),
		},
		{
			Name:  "Worker Latency p90 (UnderstandDBGPT35Turbo)",
			Query: fmt.Sprintf("(sum(rate(biz_latency_seconds_bucket{job=\"edaservice-worker\",name=\"task_understand_db\",llm_model=\"gpt-3.5-turbo\",le=\"60.0\"}[%s]))/sum(rate(biz_latency_seconds_count{job=\"edaservice-worker\",name=\"task_understand_db\",llm_model=\"gpt-3.5-turbo\"}[%s]))) or on() vector(1) >= bool 0.90", TimeRangePlaceholder, TimeRangePlaceholder),
		},
		{
			Name:  "Worker Latency p99 (UnderstandDBGPT35Turbo)",
			Query: fmt.Sprintf("(sum(rate(biz_latency_seconds_bucket{job=\"edaservice-worker\",name=\"task_understand_db\",llm_model=\"gpt-3.5-turbo\",le=\"300.0\"}[%s]))/sum(rate(biz_latency_seconds_count{job=\"edaservice-worker\",name=\"task_understand_db\",llm_model=\"gpt-3.5-turbo\"}[%s]))) or on() vector(1) >= bool 0.99", TimeRangePlaceholder, TimeRangePlaceholder),
		},
		{
			Name:  "Worker Latency p90 (UnderstandDBGPT4)",
			Query: fmt.Sprintf("(sum(rate(biz_latency_seconds_bucket{job=\"edaservice-worker\",name=\"task_understand_db\",llm_model=\"gpt-4\",le=\"300.0\"}[%s]))/sum(rate(biz_latency_seconds_count{job=\"edaservice-worker\",name=\"task_understand_db\",llm_model=\"gpt-4\"}[%s]))) or on() vector(1) >= bool 0.90", TimeRangePlaceholder, TimeRangePlaceholder),
		},
		{
			Name:  "Worker Latency p99 (UnderstandDBGPT4)",
			Query: fmt.Sprintf("(sum(rate(biz_latency_seconds_bucket{job=\"edaservice-worker\",name=\"task_understand_db\",llm_model=\"gpt-4\",le=\"900.0\"}[%s]))/sum(rate(biz_latency_seconds_count{job=\"edaservice-worker\",name=\"task_understand_db\",llm_model=\"gpt-4\"}[%s]))) or on() vector(1) >= bool 0.99", TimeRangePlaceholder, TimeRangePlaceholder),
		},
	},
	"Billing": {
		{
			Name:  "HTTP API Availability",
			Query: fmt.Sprintf("sum(rate(billing2_requests_total{service=\"billing2\",code!~\"5.*\"}[%s])) / sum(rate(billing2_requests_total{service=\"billing2\"}[%s])) >= bool 0.999", TimeRangePlaceholder, TimeRangePlaceholder),
		},
		{
			Name:  "OfficialWebsiteUserExperience HTTP API Availability",
			Query: fmt.Sprintf("sum(rate(billing2_requests_total{service=\"billing2\",code!~\"5.*\",url=~\"(.*)month||(.*)export||(.*)dryrun||(.*)payment_methods||(.*)invoices||(.*)setup_intent\"}[%s])) / sum(rate(billing2_requests_total{service=\"billing2\",url=~\"(.*)month||(.*)export||(.*)dryrun||(.*)payment_methods||(.*)invoices||(.*)setup_intent\"}[%s])) >= bool 0.9999", TimeRangePlaceholder, TimeRangePlaceholder),
		},
		{
			Name:  "HTTP API Latency p90",
			Query: fmt.Sprintf("sum(rate(billing2_request_duration_seconds_bucket{le=\"0.5\"}[%s]))\n/\nsum(rate(billing2_request_duration_seconds_count{}[%s])) >= bool 0.90", TimeRangePlaceholder, TimeRangePlaceholder),
		},
		{
			Name:  "HTTP API Latency p99",
			Query: fmt.Sprintf("sum(rate(billing2_request_duration_seconds_bucket{le=\"1\"}[%s]))\n/\nsum(rate(billing2_request_duration_seconds_count{}[%s])) >= bool 0.99", TimeRangePlaceholder, TimeRangePlaceholder),
		},
	},
	"Core Svc": {
		{
			Name:  "GRPC API Availability",
			Query: fmt.Sprintf("sum(rate(grpc_server_handled_total{namespace=~\"nightly-ms|staging-ms|prod-ms\", pod=~\"dbaas-core-svc-server.*\", grpc_type=\"unary\", grpc_code!=\"UNKNOWN|DEADLINE_EXCEEDED|INTERNAL|UNAVAILABLE|DATA_LOSS\",grpc_method!~\"UpsertProfileByYaml|CreateDBUser|DeleteDBUser|IsDBUserExisting|ChangeClusterPasswd\"}[%s]))/sum(rate(grpc_server_handled_total{namespace=~\"nightly-ms|staging-ms|prod-ms\", pod=~\"dbaas-core-svc-server.*\", grpc_type=\"unary\",grpc_method!~\"UpsertProfileByYaml|CreateDBUser|DeleteDBUser|IsDBUserExisting|ChangeClusterPasswd\"}[%s])) >= bool 0.999", TimeRangePlaceholder, TimeRangePlaceholder),
		},
		{
			Name:  "CreateOrDeleteCluster GRPC API Availability",
			Query: fmt.Sprintf("sum(rate(grpc_server_handled_total{namespace=~\"nightly-ms|staging-ms|prod-ms\", pod=~\"dbaas-core-portal-server.*\", grpc_type=\"unary\", grpc_code!=\"UNKNOWN|DEADLINE_EXCEEDED|INTERNAL|UNAVAILABLE|DATA_LOSS\",grpc_method=~\"CreateClusterV2|CreateCluster|DeleteCluster\"}[%s]))/sum(rate(grpc_server_handled_total{namespace=~\"nightly-ms|staging-ms|prod-ms\", pod=~\"dbaas-core-portal-server.*\", grpc_type=\"unary\",grpc_method=~\"CreateClusterV2|CreateCluster|DeleteCluster\"}[%s])) >= bool 0.999", TimeRangePlaceholder, TimeRangePlaceholder),
		},
		{
			Name:  "GRPC API Latency p90",
			Query: fmt.Sprintf("sum(rate(grpc_server_handling_seconds_bucket{namespace=~\"nightly-ms|staging-ms|prod-ms\", pod=~\"dbaas-core-svc-server.*\", grpc_type=\"unary\",grpc_method!~\"UpsertProfileByYaml|CreateDBUser|DeleteDBUser|IsDBUserExisting|ChangeClusterPasswd|CalcClusterFee|CreateCluster\",le=\"1\"}[%s]))\n/\nsum(rate(grpc_server_handling_seconds_count{namespace=~\"nightly-ms|staging-ms|prod-ms\", pod=~\"dbaas-core-svc-server.*\", grpc_type=\"unary\",grpc_method!~\"UpsertProfileByYaml|CreateDBUser|DeleteDBUser|IsDBUserExisting|ChangeClusterPasswd|CalcClusterFee|CreateCluster\"}[%s])) >= bool 0.90", TimeRangePlaceholder, TimeRangePlaceholder),
		},
		{
			Name:  "GRPC API Latency p99",
			Query: fmt.Sprintf("sum(rate(grpc_server_handling_seconds_bucket{namespace=~\"nightly-ms|staging-ms|prod-ms\", pod=~\"dbaas-core-svc-server.*\", grpc_type=\"unary\",grpc_method!~\"UpsertProfileByYaml|CreateDBUser|DeleteDBUser|IsDBUserExisting|ChangeClusterPasswd|CalcClusterFee|CreateCluster\", le=\"2.5\"}[%s]))\n/\nsum(rate(grpc_server_handling_seconds_count{namespace=~\"nightly-ms|staging-ms|prod-ms\", pod=~\"dbaas-core-svc-server.*\", grpc_type=\"unary\",grpc_method!~\"UpsertProfileByYaml|CreateDBUser|DeleteDBUser|IsDBUserExisting|ChangeClusterPasswd|CalcClusterFee|CreateCluster\"}[%s])) >= bool 0.99", TimeRangePlaceholder, TimeRangePlaceholder),
		},
		{
			Name:  "CreateOrDeleteCluster GRPC API Latency p90",
			Query: fmt.Sprintf("sum(rate(grpc_server_handling_seconds_bucket{namespace=~\"nightly-ms|staging-ms|prod-ms\", pod=~\"dbaas-core-portal-server.*\", grpc_type=\"unary\",grpc_method=~\"CreateClusterV2|CreateCluster|DeleteCluster\",le=\"2.5\"}[%s]))\n/\nsum(rate(grpc_server_handling_seconds_count{namespace=~\"nightly-ms|staging-ms|prod-ms\", pod=~\"dbaas-core-portal-server.*\", grpc_type=\"unary\",grpc_method=~\"CreateClusterV2|CreateCluster|DeleteCluster\"}[%s])) >= bool 0.90", TimeRangePlaceholder, TimeRangePlaceholder),
		},
		{
			Name:  "CreateOrDeleteCluster GRPC API Latency p99",
			Query: fmt.Sprintf("sum(rate(grpc_server_handling_seconds_bucket{namespace=~\"nightly-ms|staging-ms|prod-ms\", pod=~\"dbaas-core-portal-server.*\", grpc_type=\"unary\",grpc_method=~\"CreateClusterV2|CreateCluster|DeleteCluster\", le=\"5\"}[%s]))\n/\nsum(rate(grpc_server_handling_seconds_count{namespace=~\"nightly-ms|staging-ms|prod-ms\", pod=~\"dbaas-core-portal-server.*\", grpc_type=\"unary\",grpc_method=~\"CreateClusterV2|CreateCluster|DeleteCluster\"}[%s])) >= bool 0.99", TimeRangePlaceholder, TimeRangePlaceholder),
		},
	},
	"Dataflow": {
		{
			Name:  "HTTP API Availability",
			Query: fmt.Sprintf("sum(rate(kong_custom_http_status{kube_service=~\"prod-ms.dataflow-http.*\", path!~\".*/imports.*|.*/import-target-tables.*|.*/data-ingestion-profile.*|.*/dataflow.*\",code!~\"5.*\"}[%s])) / sum(rate(kong_custom_http_status{kube_service=~\"prod-ms.dataflow-http.*\", path!~\".*/imports.*|.*/import-target-tables.*|.*/data-ingestion-profile.*|.*/dataflow.*\"}[%s])) >= bool 0.999", TimeRangePlaceholder, TimeRangePlaceholder),
		},
		{
			Name:  "General HTTP API Latency p90",
			Query: fmt.Sprintf("sum(rate(kong_custom_request_latency_ms_bucket{kube_service=~\"prod-ms.dataflow-http.*\", path!~\".*/imports.*|.*/import-target-tables.*|.*/data-ingestion-profile.*|.*/dataflow.*|.*/backup.*\", le=\"1000\"}[%s]))\n/\nsum(rate(kong_custom_request_latency_ms_count{kube_service=~\"prod-ms.dataflow-http.*\", path!~\".*/imports.*|.*/import-target-tables.*|.*/data-ingestion-profile.*|.*/dataflow.*|.*/backup.*\"}[%s])) >= bool 0.90", TimeRangePlaceholder, TimeRangePlaceholder),
		},
		{
			Name:  "General HTTP API Latency p99",
			Query: fmt.Sprintf("sum(rate(kong_custom_request_latency_ms_bucket{kube_service=~\"prod-ms.dataflow-http.*\", path!~\".*/imports.*|.*/import-target-tables.*|.*/data-ingestion-profile.*|.*/dataflow.*|.*/backup.*\", le=\"2000\"}[%s]))\n/\nsum(rate(kong_custom_request_latency_ms_count{kube_service=~\"prod-ms.dataflow-http.*\", path!~\".*/imports.*|.*/import-target-tables.*|.*/data-ingestion-profile.*|.*/dataflow.*|.*/backup.*\"}[%s])) >= bool 0.99", TimeRangePlaceholder, TimeRangePlaceholder),
		},
		{
			Name:  "Restore HTTP API Latency p90",
			Query: fmt.Sprintf("sum(rate(kong_custom_request_latency_ms_bucket{kube_service=~\"prod-ms.dataflow-http.*\", path=~\".*/backup.*\", le=\"1000\"}[%s]))\n/\nsum(rate(kong_custom_request_latency_ms_count{kube_service=~\"prod-ms.dataflow-http.*\", path=~\".*/backup.*\"}[%s])) >= bool 0.90", TimeRangePlaceholder, TimeRangePlaceholder),
		},
		{
			Name:  "Restore HTTP API Latency p99",
			Query: fmt.Sprintf("sum(rate(kong_custom_request_latency_ms_bucket{kube_service=~\"prod-ms.dataflow-http.*\", path=~\".*/backup.*\", le=\"2000\"}[%s]))\n/\nsum(rate(kong_custom_request_latency_ms_count{kube_service=~\"prod-ms.dataflow-http.*\", path=~\".*/backup.*\"}[%s])) >= bool 0.99", TimeRangePlaceholder, TimeRangePlaceholder),
		},
		{
			Name:  "GRPC API Availability",
			Query: fmt.Sprintf("sum(rate(grpc_server_handled_total{namespace=~\"nightly-ms|staging-ms|prod-ms\", pod=~\"dataflow-grpc.*|dataflow-http.*\", grpc_type=\"unary\", grpc_code!=\"UNKNOWN|DEADLINE_EXCEEDED|INTERNAL|UNAVAILABLE|DATA_LOSS\", grpc_service!~\".*Import.*|^dm.Service$\"}[%s]))/sum(rate(grpc_server_handled_total{namespace=~\"nightly-ms|staging-ms|prod-ms\", pod=~\"dataflow-grpc.*|dataflow-http.*\", grpc_type=\"unary\", grpc_service!~\".*Import.*|^dm.Service$\"}[%s])) >= bool 0.999", TimeRangePlaceholder, TimeRangePlaceholder),
		},
		{
			Name:  "General GRPC API Latency p90",
			Query: fmt.Sprintf("sum(rate(grpc_server_handling_seconds_bucket{namespace=~\"nightly-ms|staging-ms|prod-ms\", pod=~\"dataflow-grpc.*|dataflow-http.*\", grpc_type=\"unary\", grpc_service!~\".*Import.*|^dm.Service$\", grpc_method!~\".*Restore.*\", le=\"1\"}[%s]))\n/\nsum(rate(grpc_server_handling_seconds_count{namespace=~\"nightly-ms|staging-ms|prod-ms\", pod=~\"dataflow-grpc.*|dataflow-http.*\", grpc_type=\"unary\", grpc_service!~\".*Import.*|^dm.Service$\", grpc_method!~\".*Restore.*\"}[%s])) >= bool 0.90", TimeRangePlaceholder, TimeRangePlaceholder),
		},
		{
			Name:  "General GRPC API Latency p99",
			Query: fmt.Sprintf("sum(rate(grpc_server_handling_seconds_bucket{namespace=~\"nightly-ms|staging-ms|prod-ms\", pod=~\"dataflow-grpc.*|dataflow-http.*\", grpc_type=\"unary\", grpc_service!~\".*Import.*|^dm.Service$\", grpc_method!~\".*Restore.*\", le=\"2.5\"}[%s]))\n/\nsum(rate(grpc_server_handling_seconds_count{namespace=~\"nightly-ms|staging-ms|prod-ms\", pod=~\"dataflow-grpc.*|dataflow-http.*\", grpc_type=\"unary\", grpc_service!~\".*Import.*|^dm.Service$\", grpc_method!~\".*Restore.*\"}[%s])) >= bool 0.99", TimeRangePlaceholder, TimeRangePlaceholder),
		},
		{
			Name:  "Restore GRPC API Latency p90",
			Query: fmt.Sprintf("sum(rate(grpc_server_handling_seconds_bucket{namespace=~\"nightly-ms|staging-ms|prod-ms\", pod=~\"dataflow-grpc.*|dataflow-http.*\", grpc_type=\"unary\", grpc_method=~\".*Restore.*\", le=\"5\"}[%s]))\n/\nsum(rate(grpc_server_handling_seconds_count{namespace=~\"nightly-ms|staging-ms|prod-ms\", pod=~\"dataflow-grpc.*|dataflow-http.*\", grpc_type=\"unary\", grpc_method=~\".*Restore.*\"}[%s])) >= bool 0.90", TimeRangePlaceholder, TimeRangePlaceholder),
		},
		{
			Name:  "Restore GRPC API Latency p99",
			Query: fmt.Sprintf("sum(rate(grpc_server_handling_seconds_bucket{namespace=~\"nightly-ms|staging-ms|prod-ms\", pod=~\"dataflow-grpc.*|dataflow-http.*\", grpc_type=\"unary\", grpc_method=~\".*Restore.*\", le=\"10\"}[%s]))\n/\nsum(rate(grpc_server_handling_seconds_count{namespace=~\"nightly-ms|staging-ms|prod-ms\", pod=~\"dataflow-grpc.*|dataflow-http.*\", grpc_type=\"unary\", grpc_method=~\".*Restore.*\"}[%s])) >= bool 0.99", TimeRangePlaceholder, TimeRangePlaceholder),
		},
	},
	"Data Service": {
		{
			Name:  "GRPC API Availability",
			Query: fmt.Sprintf("sum(rate(grpc_server_handled_total{namespace=~\"nightly-ms|staging-ms|prod-ms\", pod=~\"dataservice-dataapi.*\", grpc_type=\"unary\", grpc_code!=\"UNKNOWN|DEADLINE_EXCEEDED|INTERNAL|UNAVAILABLE|DATA_LOSS\"}[%s]))/sum(rate(grpc_server_handled_total{namespace=~\"nightly-ms|staging-ms|prod-ms\", pod=~\"dataservice-dataapi.*\", grpc_type=\"unary\"}[%s])) >= bool 0.999", TimeRangePlaceholder, TimeRangePlaceholder),
		},
		{
			Name:  "GRPC API Latency p90",
			Query: fmt.Sprintf("sum(rate(grpc_server_handling_seconds_bucket{namespace=~\"nightly-ms|staging-ms|prod-ms\", pod=~\"dataservice-dataapi.*\", grpc_type=\"unary\", le=\"1\"}[%s]))\n/\nsum(rate(grpc_server_handling_seconds_count{namespace=~\"nightly-ms|staging-ms|prod-ms\", pod=~\"dataservice-dataapi.*\", grpc_type=\"unary\"}[%s])) >= bool 0.90", TimeRangePlaceholder, TimeRangePlaceholder),
		},
		{
			Name:  "GRPC API Latency p99",
			Query: fmt.Sprintf("sum(rate(grpc_server_handling_seconds_bucket{namespace=~\"nightly-ms|staging-ms|prod-ms\", pod=~\"dataservice-dataapi.*\", grpc_type=\"unary\", le=\"5\"}[%s]))\n/\nsum(rate(grpc_server_handling_seconds_count{namespace=~\"nightly-ms|staging-ms|prod-ms\", pod=~\"dataservice-dataapi.*\", grpc_type=\"unary\"}[%s])) >= bool 0.99", TimeRangePlaceholder, TimeRangePlaceholder),
		},
	},
	"Serverless": {
		{
			Name:  "GRPC SetSpendLimit API Availability",
			Query: fmt.Sprintf("sum(rate(grpc_server_handled_total{namespace=~\"nightly-ms|staging-ms|prod-ms\", pod=~\"serverless-svc-grpc-server.*\", grpc_type=\"unary\", grpc_code!=\"UNKNOWN|DEADLINE_EXCEEDED|INTERNAL|UNAVAILABLE|DATA_LOSS\",grpc_method=\"SetSpendLimit\"}[%s]))/sum(rate(grpc_server_handled_total{namespace=~\"nightly-ms|staging-ms|prod-ms\", pod=~\"serverless-svc-grpc-server.*\", grpc_type=\"unary\",grpc_method=\"SetSpendLimit\"}[%s])) >= bool 0.999", TimeRangePlaceholder, TimeRangePlaceholder),
		},
	},
}