package cmd

import (
	"context"
	"encoding/json"
	"fmt"
	"net/url"
	"slices"
	"strconv"
	"strings"
	"time"

	"github.com/tidbcloud/controlplane9/internal/config"
	"github.com/tidbcloud/controlplane9/internal/host"
)

const (
	starterObservabilityNamespace  = "monitoring"
	starterPrometheusServiceProxy  = "/api/v1/namespaces/monitoring/services/http:prometheus:9090/proxy"
	starterGrafanaServiceProxy     = "/api/v1/namespaces/monitoring/services/http:grafana:3000/proxy"
	starterCanonicalTiDBCluster    = "tidb-serverless-serverless-cluster"
	starterGrafanaPrimaryUID       = "PB0C37C6A9EC17FDC"
	starterGrafanaKubernetesUID    = "CP9STARTERK8S"
	starterObservabilityDatasource = "http://prometheus.monitoring.svc:9090"
	starterDashboardFolder         = "Starter TiDB"
	starterDashboardRevision       = "89115a1ae6e7cb6d9f522330bd174fab329d8fa8"
	starterDashboardBundleSHA256   = "642df7473407bebd757c61c6176546abdf0f34e471727d3e3471880df56cc795"
)

var (
	starterGrafanaProvisioningTimeout      = 15 * time.Second
	starterGrafanaProvisioningPollInterval = 250 * time.Millisecond
)

var starterDashboardTitles = []string{
	"Test-Cluster-DM-Professional",
	"Test-Cluster-DM-Standard",
	"Test-Cluster-TiCDC-Summary",
	"Test-Cluster-Lightning",
	"Nodes-Info",
	"Test-Cluster-Overview",
	"Test-Cluster-PD",
	"Test-Cluster-Performance-Overview",
	"Test-Cluster-Pods-Info",
	"Test-Cluster-TiCDC",
	"Test-Cluster-TiCDC-New-Arch",
	"Test-Cluster-TiCI-Meta",
	"Test-Cluster-TiCI-Reader",
	"Test-Cluster-TiCI-Worker",
	"Test-Cluster-TiDB",
	"Test-Cluster-TiDB-Resource-Control",
	"Test-Cluster-TiDB-Runtime",
	"Test-Cluster-TiFlash-Proxy-Details",
	"Test-Cluster-TiFlash-Proxy-Summary",
	"Test-Cluster-TiFlash-Summary",
	"Test-Cluster-TiKV-CDC",
	"Test-Cluster-TiKV-Details",
	"Test-Cluster-TiKV-Summary",
	"Test-Cluster-TiKV-Trouble-Shooting",
	"Test-Cluster-TiProxy-Summary",
}

type starterPrometheusTargetExpectation struct {
	Name           string
	Selector       string
	Expected       int
	MinimumHealthy int
	Database       bool
}

type starterPrometheusSeriesExpectation struct {
	Name  string
	Query string
}

type starterPrometheusResponse struct {
	Status    string                        `json:"status"`
	ErrorType string                        `json:"errorType"`
	Error     string                        `json:"error"`
	Data      starterPrometheusResponseData `json:"data"`
}

type starterPrometheusResponseData struct {
	Result []starterPrometheusResult `json:"result"`
}

type starterPrometheusResult struct {
	Value []json.RawMessage `json:"value"`
}

type starterGrafanaDatasource struct {
	Name      string `json:"name"`
	UID       string `json:"uid"`
	URL       string `json:"url"`
	IsDefault bool   `json:"isDefault"`
}

type starterGrafanaDashboard struct {
	Title       string `json:"title"`
	FolderTitle string `json:"folderTitle"`
}

type starterKubeAccess struct {
	ExecEnv     []string
	Kubeconfig  string
	KubeContext string
	Remote      *host.Session
}

func (a starterKubeAccess) run(ctx context.Context, args ...string) ([]byte, error) {
	if a.Remote != nil {
		return host.RunSSH(ctx, a.Remote, nil, append([]string{"kubectl"}, args...)...)
	}
	return starterStatusCommandRunner(ctx, a.ExecEnv, "kubectl", args...)
}

type starterGrafanaDeploymentMetadata struct {
	Annotations map[string]string `json:"annotations"`
}

type starterGrafanaDeploymentTemplate struct {
	Metadata starterGrafanaDeploymentMetadata `json:"metadata"`
}

type starterGrafanaDeploymentSpec struct {
	Template starterGrafanaDeploymentTemplate `json:"template"`
}

type starterGrafanaDeployment struct {
	Spec starterGrafanaDeploymentSpec `json:"spec"`
}

func starterObservabilityStatusChecks(
	ctx context.Context,
	input starterLiveCheckInput,
	access starterKubeAccess,
) []starterStatusCheck {
	clusterName := ""
	if input.RemoteEKS != nil {
		clusterName = strings.TrimSpace(input.RemoteEKS.ClusterName)
	}
	if clusterName == "" {
		return []starterStatusCheck{{
			Name:    "starter-observability-targets",
			Status:  "not_ready",
			Message: "target_missing: resolved Starter EKS cluster name is empty",
		}}
	}

	checks := starterObservabilityRolloutChecks(ctx, access)
	selectedLive, err := starterSelectedLaneLivePodCount(ctx, access)
	if err != nil {
		checks = append(checks, starterStatusCheck{
			Name:    "starter-observability-selected-lane-discovery",
			Status:  "not_ready",
			Message: "target_missing: " + err.Error(),
		})
	}

	targets := starterDatabasePrometheusTargets(input.Session, selectedLive)
	targets = append(targets, starterPrometheusTargetExpectation{
		Name: "kube-state-metrics", Selector: `job="kube-state-metrics"`, Expected: 1,
	})
	readyNodes, err := starterReadyLinuxNodeCount(ctx, access)
	if err != nil {
		checks = append(checks, starterStatusCheck{
			Name:    "starter-observability-ready-linux-nodes",
			Status:  "not_ready",
			Message: "target_discovery_failed: " + err.Error(),
		})
	} else {
		checks = append(checks, starterStatusCheck{
			Name:    "starter-observability-ready-linux-nodes",
			Status:  "passed",
			Message: fmt.Sprintf("observed Ready Linux nodes=%d", readyNodes),
		})
		targets = append(targets,
			starterPrometheusTargetExpectation{Name: "node-exporter", Selector: `job="node-exporter"`, Expected: readyNodes},
			starterPrometheusTargetExpectation{Name: "kubelet", Selector: `job="kubelet",metrics_path="/metrics"`, Expected: readyNodes},
			starterPrometheusTargetExpectation{Name: "cadvisor", Selector: `job="kubelet",metrics_path="/metrics/cadvisor"`, Expected: readyNodes},
		)
	}
	for _, target := range targets {
		checks = append(checks, starterPrometheusTargetStatusCheck(
			ctx, access, clusterName, target,
		))
	}

	series := []starterPrometheusSeriesExpectation{
		{
			Name: "pd",
			Query: `pd_cluster_status{k8s_cluster=` + strconv.Quote(
				clusterName,
			) + `,tidb_cluster=` + strconv.Quote(
				starterCanonicalTiDBCluster,
			) + `}`,
		},
		{
			Name: "tidb",
			Query: `tidb_server_connections{k8s_cluster=` + strconv.Quote(
				clusterName,
			) + `,tidb_cluster=` + strconv.Quote(
				starterCanonicalTiDBCluster,
			) + `}`,
		},
		{
			Name: "tikv",
			Query: `tikv_engine_size_bytes{k8s_cluster=` + strconv.Quote(
				clusterName,
			) + `,tidb_cluster=` + strconv.Quote(
				starterCanonicalTiDBCluster,
			) + `}`,
		},
		{Name: "node", Query: `node_boot_time_seconds{job="node-exporter",k8s_cluster=` + strconv.Quote(clusterName) + `}`},
		{Name: "pods", Query: `kube_pod_info{cluster=` + strconv.Quote(clusterName) + `}`},
		{
			Name:  "cadvisor",
			Query: `container_cpu_usage_seconds_total{job="kubelet",metrics_path="/metrics/cadvisor",k8s_cluster=` + strconv.Quote(clusterName) + `}`,
		},
	}
	pd := config.ResolvedStarterPDTopology(input.Session)
	tsoJob := "pd"
	resourceManagerJob := "pd"
	if pd.Mode == config.StarterPDModePDMS {
		tsoJob = "tso"
		resourceManagerJob = "resource-manager"
	}
	series = append(series,
		starterPrometheusSeriesExpectation{
			Name: "pd-tso",
			Query: `pd_server_handle_tso_duration_seconds_count{job=` + strconv.Quote(tsoJob) + `,k8s_cluster=` + strconv.Quote(clusterName) +
				`,tidb_cluster=` + strconv.Quote(starterCanonicalTiDBCluster) + `}`,
		},
		starterPrometheusSeriesExpectation{
			Name: "pd-scheduler",
			Query: `pd_scheduler_status{job="pd",k8s_cluster=` + strconv.Quote(clusterName) +
				`,tidb_cluster=` + strconv.Quote(starterCanonicalTiDBCluster) + `}`,
		},
		starterPrometheusSeriesExpectation{
			Name: "pd-resource-manager",
			Query: `resource_manager_server_sync_load_group_counter{job=` + strconv.Quote(
				resourceManagerJob,
			) + `,k8s_cluster=` + strconv.Quote(
				clusterName,
			) +
				`,tidb_cluster=` + strconv.Quote(
				starterCanonicalTiDBCluster,
			) + `}`,
		},
	)
	for _, item := range series {
		value, err := starterPrometheusQueryNumber(ctx, access, "count("+item.Query+")")
		check := starterStatusCheck{
			Name:    "starter-observability-series-" + item.Name,
			Status:  "passed",
			Message: item.Name + " representative series is present",
		}
		if err != nil {
			check.Status = "not_ready"
			check.Message = "series_missing: " + err.Error()
		} else if value < 1 {
			check.Status = "not_ready"
			check.Message = "series_missing: " + item.Name + " representative series is empty"
		}
		checks = append(checks, check)
	}

	checks = append(checks, starterGrafanaStatusChecks(ctx, access)...)
	return checks
}

func starterDatabasePrometheusTargets(
	session *config.SessionConfig,
	selectedLive int,
) []starterPrometheusTargetExpectation {
	pd := config.ResolvedStarterPDTopology(session)
	targets := make([]starterPrometheusTargetExpectation, 0, 5)
	if pd.TSOReplicas > 0 {
		targets = append(
			targets,
			starterPrometheusTargetExpectation{Name: "tso", Selector: `job="tso"`, Expected: pd.TSOReplicas, Database: true},
			starterPrometheusTargetExpectation{
				Name:     "resource-manager",
				Selector: `job="resource-manager"`,
				Expected: pd.ResourceManagerReplicas,
				Database: true,
			},
		)
	}
	targets = append(targets,
		starterPrometheusTargetExpectation{Name: "pd", Selector: `job="pd"`, Expected: pd.RootPDReplicas, Database: true},
		starterPrometheusTargetExpectation{Name: "tikv", Selector: `job="tikv"`, Expected: 3, Database: true},
		starterPrometheusTargetExpectation{
			Name: "tidb", Selector: `job="tidb"`, Expected: 1 + selectedLive, MinimumHealthy: 1, Database: true,
		},
	)
	return targets
}

func starterObservabilityRolloutChecks(
	ctx context.Context,
	access starterKubeAccess,
) []starterStatusCheck {
	workloads := []struct {
		name     string
		resource string
	}{
		{name: "prometheus", resource: "deployment/prometheus"},
		{name: "grafana", resource: "deployment/grafana"},
		{name: "kube-state-metrics", resource: "deployment/kube-state-metrics"},
		{name: "node-exporter", resource: "daemonset/node-exporter"},
	}
	checks := make([]starterStatusCheck, 0, len(workloads))
	for _, workload := range workloads {
		output, err := access.run(ctx,
			"--kubeconfig", access.Kubeconfig,
			"--context", access.KubeContext,
			"--namespace", starterObservabilityNamespace,
			"rollout", "status", workload.resource,
			"--timeout=5s",
		)
		check := starterStatusCheck{
			Name:    "starter-observability-" + workload.name + "-rollout",
			Status:  "passed",
			Message: workload.resource + " rollout is available",
		}
		if err != nil {
			check.Status = "not_ready"
			check.Message = trimmedCommandError("check "+workload.resource+" rollout", output, err)
		}
		checks = append(checks, check)
	}
	return checks
}

func starterSelectedLaneLivePodCount(
	ctx context.Context,
	access starterKubeAccess,
) (int, error) {
	output, err := access.run(ctx,
		"--kubeconfig", access.Kubeconfig,
		"--context", access.KubeContext,
		"--namespace", starterSelectedServingLane,
		"get", "pods",
		"-l", "app.kubernetes.io/component=serverless-tidb",
		"-o", "json",
	)
	if err != nil {
		return 0, fmt.Errorf(
			"read selected %s serving lane Pods: %s",
			starterSelectedServingLane,
			trimmedCommandError("kubectl", output, err),
		)
	}
	var pods starterPodList
	if err := json.Unmarshal(output, &pods); err != nil {
		return 0, fmt.Errorf("parse selected %s serving lane Pods: %w", starterSelectedServingLane, err)
	}
	return len(pods.Items), nil
}

func starterReadyLinuxNodeCount(ctx context.Context, access starterKubeAccess) (int, error) {
	output, err := access.run(ctx,
		"--kubeconfig", access.Kubeconfig,
		"--context", access.KubeContext,
		"get", "nodes",
		"-o", "json",
	)
	if err != nil {
		return 0, fmt.Errorf("read Kubernetes Nodes: %s", trimmedCommandError("kubectl", output, err))
	}
	var nodes starterKubernetesNodeList
	if err := json.Unmarshal(output, &nodes); err != nil {
		return 0, fmt.Errorf("parse Kubernetes Nodes: %w", err)
	}
	ready := 0
	for _, node := range nodes.Items {
		if node.Metadata.Labels["kubernetes.io/os"] != "linux" {
			continue
		}
		for _, condition := range node.Status.Conditions {
			if condition.Type == "Ready" && condition.Status == "True" {
				ready++
				break
			}
		}
	}
	if ready == 0 {
		return 0, fmt.Errorf("observed zero Ready Linux nodes")
	}
	return ready, nil
}

func starterPrometheusTargetStatusCheck(
	ctx context.Context,
	access starterKubeAccess,
	clusterName string,
	target starterPrometheusTargetExpectation,
) starterStatusCheck {
	minimumHealthy := target.MinimumHealthy
	if minimumHealthy == 0 {
		minimumHealthy = target.Expected
	}
	check := starterStatusCheck{
		Name:    "starter-observability-target-" + target.Name,
		Status:  "passed",
		Message: fmt.Sprintf("%s targets healthy=%d expected=%d", target.Name, minimumHealthy, target.Expected),
	}
	base := target.Selector
	labeled := starterPrometheusSelectorWithLabels(base, map[string]string{
		"k8s_cluster": clusterName,
	})
	if target.Database {
		labeled = starterPrometheusSelectorWithLabels(labeled, map[string]string{
			"tidb_cluster": starterCanonicalTiDBCluster,
		})
	} else {
		labeled = starterPrometheusSelectorWithLabels(labeled, map[string]string{
			"cluster": clusterName,
		})
	}

	baseCount, err := starterPrometheusQueryNumber(ctx, access, "count(up{"+base+"})")
	if err != nil {
		check.Status = "not_ready"
		check.Message = "target_missing: " + err.Error()
		return check
	}
	if int(baseCount) != target.Expected {
		check.Status = "not_ready"
		check.Message = fmt.Sprintf("target_missing: %s targets observed=%g expected=%d", target.Name, baseCount, target.Expected)
		return check
	}
	labeledCount, err := starterPrometheusQueryNumber(ctx, access, "count(up{"+labeled+"})")
	if err != nil {
		check.Status = "not_ready"
		check.Message = "label_missing: " + err.Error()
		return check
	}
	if int(labeledCount) != target.Expected {
		check.Status = "not_ready"
		check.Message = fmt.Sprintf("label_missing: %s labeled targets observed=%g expected=%d", target.Name, labeledCount, target.Expected)
		return check
	}
	healthy, err := starterPrometheusQueryNumber(ctx, access, "count(up{"+labeled+"} == 1)")
	if err != nil {
		check.Status = "not_ready"
		check.Message = "target_down: " + err.Error()
		return check
	}
	check.Message = fmt.Sprintf(
		"%s targets observed=%g labeled=%g healthy=%g expected=%d minimum_healthy=%d",
		target.Name,
		baseCount,
		labeledCount,
		healthy,
		target.Expected,
		minimumHealthy,
	)
	if int(healthy) < minimumHealthy {
		check.Status = "not_ready"
		check.Message = fmt.Sprintf(
			"target_down: %s healthy targets observed=%g minimum=%d total_expected=%d",
			target.Name,
			healthy,
			minimumHealthy,
			target.Expected,
		)
	}
	return check
}

func starterPrometheusSelectorWithLabels(selector string, labels map[string]string) string {
	keys := make([]string, 0, len(labels))
	for key := range labels {
		keys = append(keys, key)
	}
	slices.Sort(keys)
	for _, key := range keys {
		if strings.TrimSpace(selector) != "" {
			selector += ","
		}
		selector += key + "=" + strconv.Quote(labels[key])
	}
	return selector
}

func starterPrometheusQueryNumber(
	ctx context.Context,
	access starterKubeAccess,
	query string,
) (float64, error) {
	path := starterPrometheusServiceProxy + "/api/v1/query?query=" + url.QueryEscape(query)
	output, err := access.run(ctx,
		"--kubeconfig", access.Kubeconfig,
		"--context", access.KubeContext,
		"get", "--raw="+path,
	)
	if err != nil {
		return 0, fmt.Errorf("query Prometheus: %s", trimmedCommandError("kubectl", output, err))
	}
	var response starterPrometheusResponse
	if err := json.Unmarshal(output, &response); err != nil {
		return 0, fmt.Errorf("parse Prometheus response: %w", err)
	}
	if response.Status != "success" {
		return 0, fmt.Errorf("prometheus query failed: %s %s", response.ErrorType, response.Error)
	}
	if len(response.Data.Result) == 0 {
		return 0, nil
	}
	value := response.Data.Result[0].Value
	if len(value) != 2 {
		return 0, fmt.Errorf("prometheus vector result has %d values, want 2", len(value))
	}
	var text string
	if err := json.Unmarshal(value[1], &text); err != nil {
		return 0, fmt.Errorf("parse Prometheus sample value: %w", err)
	}
	number, err := strconv.ParseFloat(text, 64)
	if err != nil {
		return 0, fmt.Errorf("parse Prometheus sample %q: %w", text, err)
	}
	return number, nil
}

func starterGrafanaStatusChecks(
	ctx context.Context,
	access starterKubeAccess,
) []starterStatusCheck {
	checks := make([]starterStatusCheck, 0, 6)
	checks = append(checks, starterGrafanaBundleStatusCheck(ctx, access))
	health, err := starterServiceProxyGet(ctx, access, starterGrafanaServiceProxy+"/api/health")
	var healthResponse struct {
		Database string `json:"database"`
	}
	if err == nil {
		err = json.Unmarshal(health, &healthResponse)
	}
	if err != nil || healthResponse.Database != "ok" {
		message := "Grafana health did not report database=ok"
		if err != nil {
			message = err.Error()
		}
		checks = append(
			checks,
			starterStatusCheck{Name: "starter-observability-grafana-health", Status: "not_ready", Message: "datasource_missing: " + message},
		)
	} else {
		checks = append(checks, starterStatusCheck{Name: "starter-observability-grafana-health", Status: "passed", Message: "Grafana API is healthy"})
	}

	provisioningCtx, cancel := context.WithTimeout(ctx, starterGrafanaProvisioningTimeout)
	defer cancel()
	var datasourceCheck starterStatusCheck
	var dashboardCheck starterStatusCheck

provisioningPoll:
	for {
		datasourceCheck, dashboardCheck = starterGrafanaProvisioningChecks(provisioningCtx, access)
		if datasourceCheck.Status == "passed" && dashboardCheck.Status == "passed" {
			break
		}
		timer := time.NewTimer(starterGrafanaProvisioningPollInterval)
		select {
		case <-provisioningCtx.Done():
			timer.Stop()
			break provisioningPoll
		case <-timer.C:
		}
	}
	checks = append(checks,
		datasourceCheck,
		dashboardCheck,
		starterStatusCheck{
			Name:    "starter-observability-dashboard-partial",
			Status:  "passed",
			Message: "dashboard_partial: Pods spans the core and both product-owned TiDB lane namespaces; obsolete kube-state-metrics panels remain known-empty",
		},
		starterStatusCheck{
			Name:    "starter-observability-unsupported-components",
			Status:  "passed",
			Message: "unsupported_component: 14 dashboards cover absent or conditional TiProxy, TiFlash, TiCDC, TiCI, Lightning, TiKV-CDC, and DM surfaces",
		},
	)
	return checks
}

func starterGrafanaProvisioningChecks(
	ctx context.Context,
	access starterKubeAccess,
) (datasourceCheck, dashboardCheck starterStatusCheck) {
	datasourceOutput, err := starterServiceProxyGet(ctx, access, starterGrafanaServiceProxy+"/api/datasources")
	datasourceCheck = starterStatusCheck{
		Name:    "starter-observability-datasources",
		Status:  "passed",
		Message: "tidb-cluster and k8s-cluster point to the session Prometheus",
	}
	if err != nil {
		datasourceCheck.Status = "not_ready"
		datasourceCheck.Message = "datasource_missing: " + err.Error()
	} else {
		var datasources []starterGrafanaDatasource
		if err := json.Unmarshal(datasourceOutput, &datasources); err != nil {
			datasourceCheck.Status = "not_ready"
			datasourceCheck.Message = "datasource_missing: parse Grafana datasources: " + err.Error()
		} else if !starterGrafanaDatasourcesMatch(datasources) {
			datasourceCheck.Status = "not_ready"
			datasourceCheck.Message = "datasource_missing: expected pinned tidb-cluster primary and k8s-cluster alias on one Prometheus"
		}
	}

	dashboardOutput, err := starterServiceProxyGet(
		ctx,
		access,
		starterGrafanaServiceProxy+"/api/search?type=dash-db&limit=1000",
	)
	dashboardCheck = starterStatusCheck{
		Name:    "starter-observability-dashboards",
		Status:  "passed",
		Message: "all 25 pinned dashboards are provisioned in folder " + starterDashboardFolder,
	}
	if err != nil {
		dashboardCheck.Status = "not_ready"
		dashboardCheck.Message = "dashboard_missing: " + err.Error()
	} else {
		var dashboards []starterGrafanaDashboard
		if err := json.Unmarshal(dashboardOutput, &dashboards); err != nil {
			dashboardCheck.Status = "not_ready"
			dashboardCheck.Message = "dashboard_missing: parse Grafana dashboard search: " + err.Error()
		} else {
			observed := make(map[string]string, len(dashboards))
			for _, dashboard := range dashboards {
				observed[dashboard.Title] = dashboard.FolderTitle
			}
			var missing []string
			for _, title := range starterDashboardTitles {
				folder, ok := observed[title]
				if !ok {
					missing = append(missing, title)
				} else if folder != starterDashboardFolder {
					missing = append(missing, fmt.Sprintf("%s (folder=%q)", title, folder))
				}
			}
			if len(missing) > 0 {
				dashboardCheck.Status = "not_ready"
				dashboardCheck.Message = "dashboard_missing: " + strings.Join(missing, ", ")
			}
		}
	}
	return datasourceCheck, dashboardCheck
}

func starterGrafanaBundleStatusCheck(ctx context.Context, access starterKubeAccess) starterStatusCheck {
	output, err := access.run(ctx,
		"--kubeconfig", access.Kubeconfig,
		"--context", access.KubeContext,
		"--namespace", starterObservabilityNamespace,
		"get", "deployment/grafana",
		"-o", "json",
	)
	if err != nil {
		return starterStatusCheck{
			Name:    "starter-observability-dashboard-bundle",
			Status:  "not_ready",
			Message: "dashboard_revision_mismatch: " + trimmedCommandError("read Grafana Deployment", output, err),
		}
	}
	var deployment starterGrafanaDeployment
	if err := json.Unmarshal(output, &deployment); err != nil {
		return starterStatusCheck{
			Name:    "starter-observability-dashboard-bundle",
			Status:  "not_ready",
			Message: "dashboard_revision_mismatch: parse Grafana Deployment: " + err.Error(),
		}
	}
	annotations := deployment.Spec.Template.Metadata.Annotations
	observedRevision := annotations["cp9.tidbcloud.com/dashboard-revision"]
	observedSHA256 := annotations["cp9.tidbcloud.com/dashboard-bundle-sha256"]
	message := fmt.Sprintf("dashboard revision=%s bundle_sha256=%s folder=%s", observedRevision, observedSHA256, starterDashboardFolder)
	if observedRevision != starterDashboardRevision || observedSHA256 != starterDashboardBundleSHA256 {
		return starterStatusCheck{
			Name:   "starter-observability-dashboard-bundle",
			Status: "not_ready",
			Message: fmt.Sprintf(
				"dashboard_revision_mismatch: observed revision=%q bundle_sha256=%q; expected revision=%q bundle_sha256=%q",
				observedRevision,
				observedSHA256,
				starterDashboardRevision,
				starterDashboardBundleSHA256,
			),
		}
	}
	return starterStatusCheck{Name: "starter-observability-dashboard-bundle", Status: "passed", Message: message}
}

func starterGrafanaDatasourcesMatch(datasources []starterGrafanaDatasource) bool {
	primary := false
	alias := false
	for _, datasource := range datasources {
		switch datasource.Name {
		case "tidb-cluster":
			primary = datasource.UID == starterGrafanaPrimaryUID &&
				datasource.URL == starterObservabilityDatasource && datasource.IsDefault
		case "k8s-cluster":
			alias = datasource.UID == starterGrafanaKubernetesUID &&
				datasource.URL == starterObservabilityDatasource && !datasource.IsDefault
		}
	}
	return primary && alias
}

func starterServiceProxyGet(
	ctx context.Context,
	access starterKubeAccess,
	path string,
) ([]byte, error) {
	output, err := access.run(ctx,
		"--kubeconfig", access.Kubeconfig,
		"--context", access.KubeContext,
		"get", "--raw="+path,
	)
	if err != nil {
		return nil, fmt.Errorf("query in-cluster service: %s", trimmedCommandError("kubectl", output, err))
	}
	return output, nil
}