import subprocess import sys import tempfile import textwrap import unittest from pathlib import Path import yaml REPO_ROOT = Path(__file__).resolve().parents[1] GENERATOR = REPO_ROOT / "hack" / "generate-byoc-pull-images.py" DBAAS = "380838443567.dkr.ecr.us-west-2.amazonaws.com" O11Y = "557537366020.dkr.ecr.us-west-2.amazonaws.com" def write(path: Path, content: str) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(textwrap.dedent(content).lstrip()) class GenerateByocPullImagesTest(unittest.TestCase): def run_generator(self, root: Path, *extra_args: str) -> subprocess.CompletedProcess: return subprocess.run( [ sys.executable, str(GENERATOR), "--repo-root", str(root / "docker-image-controller"), "--app-delivery-root", str(root / "app-delivery"), "--observability-root", str(root / "observability"), "--auto-deploy-root", str(root / "auto-deploy" / "terraform" / "aws" / "byoc"), *extra_args, ], text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) def seed_controller_repo(self, root: Path) -> Path: repo = root / "docker-image-controller" (root / "app-delivery" / "nextgen-regional-control-plane").mkdir(parents=True, exist_ok=True) (root / "app-delivery" / "dedicated-cluster-ng-service").mkdir(parents=True, exist_ok=True) (root / "observability" / "deployments").mkdir(parents=True, exist_ok=True) write( root / "auto-deploy" / "terraform" / "aws" / "byoc" / "shoot" / "vars.tf", ''' variable "regional_server_img_tag" { type = string default = "regional-tag" } ''', ) write( repo / "image-sync-prod.yaml", """ repositorySets: commonRepositories: [] targetRegistries: [] defaults: {} """, ) write( repo / "image-sync-byoc-prod.yaml", f""" repositorySets: tidbRepos: - name: tidbcloud/existing source: {DBAAS}/tidbcloud/existing tags: allow: - v0 - name: tidbcloud/worker source: {DBAAS}/tidbcloud/worker tags: allow: - old-worker - name: third-party/acme/new-service source: docker.io/acme/new-service tags: allow: - "v2.*" - name: tidbcloud/vector source: {DBAAS}/tidbcloud/vector tags: allow: - dbaas-old - name: tidbcloud/pd source: {DBAAS}/tidbcloud/pd tags: allow: - "^v8[.]5[.]4-nextgen.*$" - name: tidbcloud/dm source: {DBAAS}/tidbcloud/dm tags: allow: - old-dm o11yRepos: - name: observability-ops-server source: {O11Y}/observability-ops-server tags: allow: - old-o11y targetRegistries: [] defaults: {{}} """, ) write( repo / "byoc-pull-images-aws-prod.yaml", f""" kernelImages: - name: tidbcloud/pd source: {DBAAS}/tidbcloud/pd tags: - v8.5.4-nextgen.202606.1 - v8.5.4-nextgen.202606.2 dbaasImages: - name: tidbcloud/existing source: {DBAAS}/tidbcloud/existing tag: stale o11yImages: [] o11yCharts: [] """, ) write(repo / "byoc-pull-customers-aws-prod.yaml", "customers: []\n") return repo def test_generates_sets_from_byoc_allowlist_and_updates_sync_configs(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) repo = self.seed_controller_repo(root) write( root / "app-delivery" / "nextgen-regional-control-plane" / "component" / "values.yaml", """ existingBase: image: TMPL_IMAGE_REGISTRY/tidbcloud/existing:base-tag worker: image: TMPL_IMAGE_REGISTRY/tidbcloud/worker:v3 dbaasVector: image: TMPL_IMAGE_REGISTRY/tidbcloud/vector:dbaas-vector dm: image: TMPL_IMAGE_REGISTRY/tidbcloud/dm:dm-tag ignored: image: docker.io/not/in-byoc:v9 """, ) write( root / "app-delivery" / "nextgen-regional-control-plane" / "component" / "variants" / "common-values.yaml", """ existing: image: TMPL_IMAGE_REGISTRY/tidbcloud/existing:general-tag """, ) write( root / "app-delivery" / "nextgen-regional-control-plane" / "component" / "variants" / "prod-aws-byoc-values.yaml", """ existing: image: TMPL_IMAGE_REGISTRY/tidbcloud/existing:byoc-tag publicSource: registry: public.ecr.aws repository: docker.io/acme/new-service tag: v2 """, ) write( root / "app-delivery" / "nextgen-regional-control-plane" / "component" / "envs" / "prod-aws-us-west-2-byoc-fnu" / "values.yaml", """ staleEnv: image: TMPL_IMAGE_REGISTRY/tidbcloud/worker:stale-env """, ) write( root / "observability" / "deployments" / "infra-v2" / "ops-server" / "variants" / "byoc-aws-values-tmpl.yaml", """ image: repository: observability-ops-server tag: e5a584a """, ) write( root / "observability" / "deployments" / "infra-v2" / "vector" / "variants" / "aws-values-tmpl.yaml", """ vector: image: tidbcloud/vector tag: o11y-vector """, ) result = self.run_generator(root) self.assertEqual(result.returncode, 0, result.stderr + result.stdout) pull = yaml.safe_load((repo / "byoc-pull-images-aws-prod.yaml").read_text()) self.assertEqual( pull["kernelImages"], [ { "name": "tidbcloud/pd", "source": f"{DBAAS}/tidbcloud/pd", "tags": ["v8.5.4-nextgen.202606.1", "v8.5.4-nextgen.202606.2"], } ], ) self.assertEqual( sorted(pull["dbaasImages"], key=lambda image: image["name"]), [ { "name": "third-party/acme/new-service", "source": f"{DBAAS}/third-party/acme/new-service", "tag": "v2", }, { "name": "tidbcloud/dm", "source": f"{DBAAS}/tidbcloud/dm", "tag": "dm-tag", }, { "name": "tidbcloud/existing", "source": f"{DBAAS}/tidbcloud/existing", "tag": "byoc-tag", }, { "name": "tidbcloud/vector", "source": f"{DBAAS}/tidbcloud/vector", "tag": "dbaas-vector", }, { "name": "tidbcloud/worker", "source": f"{DBAAS}/tidbcloud/worker", "tag": "v3", }, ], ) self.assertEqual( sorted(pull["o11yImages"], key=lambda image: image["name"]), [ { "name": "observability-ops-server", "source": f"{O11Y}/observability-ops-server", "tag": "e5a584a", }, { "name": "tidbcloud/vector", "source": f"{O11Y}/tidbcloud/vector", "tag": "o11y-vector", }, ], ) prod = yaml.safe_load((repo / "image-sync-prod.yaml").read_text()) self.assertIn( { "name": "third-party/acme/new-service", "source": "docker.io/acme/new-service", "tags": {"allow": ["v2"]}, }, prod["repositorySets"]["commonRepositories"], ) self.assertNotIn( { "name": "not/in-byoc", "source": "docker.io/not/in-byoc", "tags": {"allow": ["v9"]}, }, prod["repositorySets"]["commonRepositories"], ) byoc = yaml.safe_load((repo / "image-sync-byoc-prod.yaml").read_text()) tidb_repos = byoc["repositorySets"]["tidbRepos"] self.assertIn( { "name": "third-party/acme/new-service", "source": f"{DBAAS}/third-party/acme/new-service", "tags": {"allow": ["v2"]}, }, tidb_repos, ) self.assertIn( { "name": "tidbcloud/existing", "source": f"{DBAAS}/tidbcloud/existing", "tags": {"allow": ["v0", "byoc-tag"]}, }, tidb_repos, ) self.assertIn( { "name": "tidbcloud/worker", "source": f"{DBAAS}/tidbcloud/worker", "tags": {"allow": ["old-worker", "v3"]}, }, tidb_repos, ) self.assertIn( { "name": "tidbcloud/dm", "source": f"{DBAAS}/tidbcloud/dm", "tags": {"allow": ["old-dm", "dm-tag"]}, }, tidb_repos, ) def test_fails_when_same_priority_resolves_multiple_tags_for_one_generated_image(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) repo = self.seed_controller_repo(root) byoc = yaml.safe_load((repo / "image-sync-byoc-prod.yaml").read_text()) byoc["repositorySets"]["tidbRepos"].append( { "name": "tidbcloud/api", "source": f"{DBAAS}/tidbcloud/api", "tags": {"allow": ["old"]}, } ) write(repo / "image-sync-byoc-prod.yaml", yaml.safe_dump(byoc, sort_keys=False)) variant_dir = ( root / "app-delivery" / "nextgen-regional-control-plane" / "component" / "variants" ) write(variant_dir / "common-a.yaml", "image: TMPL_IMAGE_REGISTRY/tidbcloud/api:v1\n") write(variant_dir / "common-b.yaml", "image: TMPL_IMAGE_REGISTRY/tidbcloud/api:v2\n") result = self.run_generator(root) self.assertNotEqual(result.returncode, 0) self.assertIn("resolves to multiple tags", result.stderr) self.assertIn("general variant", result.stderr) def test_check_report_only_does_not_fail_on_deployment_tag_conflict(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) repo = self.seed_controller_repo(root) byoc = yaml.safe_load((repo / "image-sync-byoc-prod.yaml").read_text()) byoc["repositorySets"]["tidbRepos"].append( { "name": "tidbcloud/api", "source": f"{DBAAS}/tidbcloud/api", "tags": {"allow": ["old"]}, } ) write(repo / "image-sync-byoc-prod.yaml", yaml.safe_dump(byoc, sort_keys=False)) variant_dir = ( root / "app-delivery" / "nextgen-regional-control-plane" / "component" / "variants" ) write(variant_dir / "common-a.yaml", "image: TMPL_IMAGE_REGISTRY/tidbcloud/api:v1\n") write(variant_dir / "common-b.yaml", "image: TMPL_IMAGE_REGISTRY/tidbcloud/api:v2\n") result = self.run_generator(root, "--check", "--report-only") self.assertEqual(result.returncode, 0, result.stderr + result.stdout) self.assertIn("resolved to multiple deployment tags", result.stderr) self.assertIn("tidbcloud/api resolves to multiple tags", result.stderr) self.assertIn("Generated BYOC pull image artifacts are out of date.", result.stderr) self.assertIn("report-only mode is enabled", result.stderr) pull = yaml.safe_load((repo / "byoc-pull-images-aws-prod.yaml").read_text()) self.assertEqual( pull["dbaasImages"], [ { "name": "tidbcloud/existing", "source": f"{DBAAS}/tidbcloud/existing", "tag": "stale", } ], ) def test_report_only_requires_check(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) self.seed_controller_repo(root) result = self.run_generator(root, "--report-only") self.assertNotEqual(result.returncode, 0) self.assertIn("--report-only requires --check", result.stderr) def test_reports_and_skips_mutable_deployment_tags(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) repo = self.seed_controller_repo(root) byoc = yaml.safe_load((repo / "image-sync-byoc-prod.yaml").read_text()) byoc["repositorySets"]["o11yRepos"].append( { "name": "o11ydiagnosis", "source": f"{O11Y}/o11ydiagnosis", "tags": {"allow": ["old"]}, } ) write(repo / "image-sync-byoc-prod.yaml", yaml.safe_dump(byoc, sort_keys=False)) write( root / "observability" / "deployments" / "charts" / "o11ydiagnosis-deltalake" / "Chart.yaml", """ apiVersion: v2 name: o11ydiagnosis-deltalake version: 0.1.0 """, ) write( root / "observability" / "deployments" / "charts" / "o11ydiagnosis-deltalake" / "values.yaml", """ image: repository: o11ydiagnosis tag: latest """, ) result = self.run_generator(root) self.assertEqual(result.returncode, 0, result.stderr + result.stdout) self.assertIn("resolved to mutable deployment tags", result.stderr) self.assertIn("o11yImages:o11ydiagnosis tag 'latest'", result.stderr) self.assertIn("o11ydiagnosis-deltalake", result.stderr) pull = yaml.safe_load((repo / "byoc-pull-images-aws-prod.yaml").read_text()) self.assertNotIn("o11ydiagnosis", [image["name"] for image in pull["o11yImages"]]) byoc = yaml.safe_load((repo / "image-sync-byoc-prod.yaml").read_text()) o11y_repos = byoc["repositorySets"]["o11yRepos"] self.assertIn( { "name": "o11ydiagnosis", "source": f"{O11Y}/o11ydiagnosis", "tags": {"allow": ["old"]}, }, o11y_repos, ) def test_resolves_observability_chart_artifacts(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) repo = self.seed_controller_repo(root) byoc = yaml.safe_load((repo / "image-sync-byoc-prod.yaml").read_text()) byoc["repositorySets"]["o11yRepos"].extend( [ { "name": "observability-charts/ops-server", "source": f"{O11Y}/observability-charts/ops-server", "tags": {"allow": ["old-chart"]}, }, { "name": "observability-charts/loki", "source": f"{O11Y}/observability-charts/loki", "tags": {"allow": ["old-loki"]}, }, ] ) write(repo / "image-sync-byoc-prod.yaml", yaml.safe_dump(byoc, sort_keys=False)) write( root / "observability" / "deployments" / "charts" / "ops-server" / "Chart.yaml", """ apiVersion: v2 name: ops-server version: 0.2.7 """, ) write(root / "observability" / "deployments" / "charts" / "ops-server" / "values.yaml", "{}\n") write( root / "observability" / "deployments" / "dataplane" / "deploy-with-ops.sh", 'LOKI_VERSION="${LOKI_CHART_VERSION:-6.29.0}"\n', ) result = self.run_generator(root) self.assertEqual(result.returncode, 0, result.stderr + result.stdout) pull = yaml.safe_load((repo / "byoc-pull-images-aws-prod.yaml").read_text()) self.assertIn( { "name": "observability-charts/ops-server", "source": f"{O11Y}/observability-charts/ops-server", "tag": "0.2.7", }, pull["o11yCharts"], ) self.assertIn( { "name": "observability-charts/loki", "source": f"{O11Y}/observability-charts/loki", "tag": "6.29.0", }, pull["o11yCharts"], ) self.assertNotIn( "observability-charts/ops-server", [image["name"] for image in pull["o11yImages"]], ) def test_resolves_drhouse_image_tag_overlays(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) repo = self.seed_controller_repo(root) byoc = yaml.safe_load((repo / "image-sync-byoc-prod.yaml").read_text()) byoc["repositorySets"]["o11yRepos"].extend( [ { "name": "diagnosis-query", "source": f"{O11Y}/diagnosis-query", "tags": {"allow": ["old-query"]}, }, { "name": "o11ydiagnosis", "source": f"{O11Y}/o11ydiagnosis", "tags": {"allow": ["old-diagnosis"]}, }, { "name": "observability-regional-task-collector", "source": f"{O11Y}/observability-regional-task-collector", "tags": {"allow": ["old-task"]}, }, ] ) write(repo / "image-sync-byoc-prod.yaml", yaml.safe_dump(byoc, sort_keys=False)) write( root / "observability" / "deployments" / "charts" / "o11ydiagnosis-deltalake" / "values.yaml", """ image: repository: o11ydiagnosis tag: latest taskCollector: image: repository: observability-regional-task-collector tag: latest """, ) write( root / "observability" / "deployments" / "drhouse" / "diagnosis-query" / "image-tag.yaml", """ image: tag: default-query """, ) write( root / "observability" / "deployments" / "drhouse" / "diagnosis-query" / "envs" / "prod-aws-byoc-pull-generator" / "image-tag.yaml", """ image: tag: env-query """, ) write( root / "observability" / "deployments" / "drhouse" / "o11ydiagnosis-deltalake" / "image-tag.yaml", """ image: tag: diagnosis-tag taskCollector: image: tag: task-tag """, ) result = self.run_generator(root) self.assertEqual(result.returncode, 0, result.stderr + result.stdout) self.assertNotIn("resolved to mutable deployment tags", result.stderr) pull = yaml.safe_load((repo / "byoc-pull-images-aws-prod.yaml").read_text()) self.assertIn( {"name": "diagnosis-query", "source": f"{O11Y}/diagnosis-query", "tag": "env-query"}, pull["o11yImages"], ) self.assertIn( {"name": "o11ydiagnosis", "source": f"{O11Y}/o11ydiagnosis", "tag": "diagnosis-tag"}, pull["o11yImages"], ) self.assertIn( { "name": "observability-regional-task-collector", "source": f"{O11Y}/observability-regional-task-collector", "tag": "task-tag", }, pull["o11yImages"], ) def test_resolves_regional_server_tags_from_auto_deploy(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) repo = self.seed_controller_repo(root) byoc = yaml.safe_load((repo / "image-sync-byoc-prod.yaml").read_text()) byoc["repositorySets"]["o11yRepos"].extend( [ { "name": "observability-regional-server", "source": f"{O11Y}/observability-regional-server", "tags": {"allow": ["old-server"]}, }, { "name": "observability-regional-data-server", "source": f"{O11Y}/observability-regional-data-server", "tags": {"allow": ["old-data"]}, }, ] ) write(repo / "image-sync-byoc-prod.yaml", yaml.safe_dump(byoc, sort_keys=False)) write( root / "auto-deploy" / "terraform" / "aws" / "byoc" / "shoot" / "vars.tf", ''' variable "regional_server_img_tag" { type = string default = "2026-02-11-regional" } ''', ) result = self.run_generator(root) self.assertEqual(result.returncode, 0, result.stderr + result.stdout) pull = yaml.safe_load((repo / "byoc-pull-images-aws-prod.yaml").read_text()) self.assertIn( { "name": "observability-regional-server", "source": f"{O11Y}/observability-regional-server", "tag": "2026-02-11-regional", }, pull["o11yImages"], ) self.assertIn( { "name": "observability-regional-data-server", "source": f"{O11Y}/observability-regional-data-server", "tag": "2026-02-11-regional", }, pull["o11yImages"], ) def test_static_o11y_scope_excludes_generated_but_undeployed_components(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) repo = self.seed_controller_repo(root) byoc = yaml.safe_load((repo / "image-sync-byoc-prod.yaml").read_text()) byoc["repositorySets"]["o11yRepos"].append( { "name": "deltalake-ops-server", "source": f"{O11Y}/deltalake-ops-server", "tags": {"allow": ["old"]}, } ) write(repo / "image-sync-byoc-prod.yaml", yaml.safe_dump(byoc, sort_keys=False)) write( root / "observability" / "deployments" / "drhouse" / "deltalake-ops-server" / "variants" / "aws" / "values-tmpl.yaml", """ image: repository: deltalake-ops-server tag: generated-only """, ) result = self.run_generator(root) self.assertEqual(result.returncode, 0, result.stderr + result.stdout) pull = yaml.safe_load((repo / "byoc-pull-images-aws-prod.yaml").read_text()) self.assertNotIn("deltalake-ops-server", [image["name"] for image in pull["o11yImages"]]) def test_rejects_customer_principal_from_another_account(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) repo = self.seed_controller_repo(root) write( repo / "byoc-pull-customers-aws-prod.yaml", """ customers: - name: wrong-account accountId: "123456789012" principals: - arn:aws:iam::210987654321:role/customer-node """, ) result = self.run_generator(root) self.assertNotEqual(result.returncode, 0) self.assertIn("does not match accountId", result.stderr) def test_validate_local_only_rejects_unknown_kernel_image(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) repo = self.seed_controller_repo(root) write( repo / "byoc-pull-images-aws-prod.yaml", f""" kernelImages: - name: tidbcloud/not-kernel source: {DBAAS}/tidbcloud/not-kernel tags: - v1 dbaasImages: [] o11yImages: [] o11yCharts: [] """, ) result = self.run_generator(root, "--validate-local-only") self.assertNotEqual(result.returncode, 0) self.assertIn("not an allowed kernel image", result.stderr) def test_validate_local_only_rejects_mutable_pull_image_tag(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) repo = self.seed_controller_repo(root) write( repo / "byoc-pull-images-aws-prod.yaml", f""" kernelImages: - name: tidbcloud/pd source: {DBAAS}/tidbcloud/pd tags: - v8.5.4-nextgen.202606.1 dbaasImages: - name: tidbcloud/existing source: {DBAAS}/tidbcloud/existing tag: latest o11yImages: [] o11yCharts: [] """, ) result = self.run_generator(root, "--validate-local-only") self.assertNotEqual(result.returncode, 0) self.assertIn("mutable tag", result.stderr) def test_validate_local_only_rejects_chart_in_o11y_images(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) repo = self.seed_controller_repo(root) write( repo / "byoc-pull-images-aws-prod.yaml", f""" kernelImages: - name: tidbcloud/pd source: {DBAAS}/tidbcloud/pd tags: - v8.5.4-nextgen.202606.1 dbaasImages: [] o11yImages: - name: observability-charts/ops-server source: {O11Y}/observability-charts/ops-server tag: 0.2.7 o11yCharts: [] """, ) result = self.run_generator(root, "--validate-local-only") self.assertNotEqual(result.returncode, 0) self.assertIn("must not be an observability chart", result.stderr) def test_validate_local_only_rejects_empty_kernel_tags(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) repo = self.seed_controller_repo(root) write( repo / "byoc-pull-images-aws-prod.yaml", f""" kernelImages: - name: tidbcloud/pd source: {DBAAS}/tidbcloud/pd tags: [] dbaasImages: [] o11yImages: [] o11yCharts: [] """, ) result = self.run_generator(root, "--validate-local-only") self.assertNotEqual(result.returncode, 0) self.assertIn("tags must be a non-empty list", result.stderr) def test_validate_local_only_rejects_duplicate_kernel_tags(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) repo = self.seed_controller_repo(root) write( repo / "byoc-pull-images-aws-prod.yaml", f""" kernelImages: - name: tidbcloud/pd source: {DBAAS}/tidbcloud/pd tags: - v8.5.4-nextgen.202606.1 - v8.5.4-nextgen.202606.1 dbaasImages: [] o11yImages: [] o11yCharts: [] """, ) result = self.run_generator(root, "--validate-local-only") self.assertNotEqual(result.returncode, 0) self.assertIn("has duplicate tag", result.stderr) def test_validate_local_only_rejects_mutable_kernel_tag(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) repo = self.seed_controller_repo(root) write( repo / "byoc-pull-images-aws-prod.yaml", f""" kernelImages: - name: tidbcloud/pd source: {DBAAS}/tidbcloud/pd tags: - latest dbaasImages: [] o11yImages: [] o11yCharts: [] """, ) result = self.run_generator(root, "--validate-local-only") self.assertNotEqual(result.returncode, 0) self.assertIn("mutable tag", result.stderr) if __name__ == "__main__": unittest.main()