"""Exercise canonical configuration and the real stop/recreate/readiness runner.""" from __future__ import annotations import base64 import io import json import os from pathlib import Path import shutil import socket import subprocess import sys import tarfile import tempfile import unittest import uuid sys.path.insert(0, str(Path(__file__).resolve().parent)) import manage_test2_integration as integration import common SOURCE = '05e2980f4b15f2f4d1d4c24ce294af4d6a470094' class ConfigurationFixture(unittest.TestCase): @classmethod def setUpClass(cls): cls.cert_dir = tempfile.TemporaryDirectory() cls.addClassCleanup(cls.cert_dir.cleanup) root = Path(cls.cert_dir.name) subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-keyout', str(root / 'key'), '-out', str(root / 'ca'), '-subj', '/CN=fixture.runtime.invalid', '-days', '1'], check=True, capture_output=True, timeout=20) cls.ca = (root / 'ca').read_bytes() def setUp(self): self.directory = tempfile.TemporaryDirectory() self.addCleanup(self.directory.cleanup) self.root = Path(self.directory.name) self.l0 = self.root / '.runtime/l0-standalone' self.source = { 'TIDB_HOST': 'fixture.invalid', 'TIDB_PORT': '4000', 'TIDB_USER': 'fixture', 'TIDB_PASSWORD': 'fixture-secret', 'TIDB_DATABASE': 'fixture', 'NEXUSCRM_NC_MASTER_KEY': base64.b64encode(bytes(32)).decode(), 'PUBLIC_BASE_URL': 'https://fixture.invalid', 'BACKEND_PORT': '3302', 'NEXUSCRM_CONNECTOR_VERIFIED_INGRESS_RECEIPTS': 'true', 'NEXUSCRM_CONNECTOR_RECEIPT_POLL_INTERVAL_MS': '100', } self.serving = { 'schema': 'nexuscrm.runtime-api-serving/v4', 'org_id': 'fixture-org', 'app_installation_id': 'fixture-installation', 'authorized_workload_ids': ['integration-service', 'sfcrm-app-service'], } self.identities = { 'schema': 'nexuscrm.runtime-workload-identities/v1', 'identities': [{'workloadId': 'integration-service', 'token': 'fixture-token'}, {'workloadId': 'sfcrm-app-service', 'token': 'must-not-be-mounted'}], } self.write_inputs() def write_inputs(self): for path in (self.root / 'env', self.l0 / 'config', self.l0 / 'secrets/tls'): path.mkdir(parents=True, exist_ok=True) (self.root / 'env/runtime.env').write_bytes(integration.render_env(self.source)) (self.l0 / 'config/serving.json').write_text(json.dumps(self.serving)) (self.l0 / 'secrets/workload-identities.json').write_text(json.dumps(self.identities)) (self.l0 / 'config/runtime-server-name').write_text('fixture.runtime.invalid\n') (self.l0 / 'secrets/tls/server.crt').write_bytes(self.ca) def configuration(self): return integration.configuration(self.root, '127.0.0.1:3403', '127.0.0.1:3312') class IntegrationContractTest(ConfigurationFixture): def test_public_webhook_route_is_scoped_idempotent_and_fail_closed(self): current = '''server { server_name sibling.invalid; location /webhooks/ { proxy_pass http://127.0.0.1:3999; } } server { server_name fixture.invalid; location /services/ { proxy_pass http://127.0.0.1:3302; } location /webhooks/ { proxy_pass http://127.0.0.1:3302; } } ''' rendered = integration.render_public_integration_route( current, 'fixture.invalid', '3302') self.assertIn('proxy_pass http://127.0.0.1:3403;', rendered) self.assertIn('proxy_pass http://127.0.0.1:3999;', rendered) self.assertEqual( rendered, integration.render_public_integration_route( rendered, 'fixture.invalid', '3302')) with self.assertRaisesRegex(RuntimeError, 'unexpected authority'): integration.render_public_integration_route( current.replace('127.0.0.1:3302;\n }\n}\n', '127.0.0.1:3555;\n }\n}\n'), 'fixture.invalid', '3302') def test_public_webhook_route_is_preflighted_before_activation(self): nginx = self.root / 'test2.conf' current = '''server { server_name fixture.invalid; location /webhooks/ { proxy_pass http://127.0.0.1:3302; } } ''' nginx.write_text(current) prepared = integration.preflight_public_integration_route(self.root, nginx) nginx.write_text(current.replace( ' location /webhooks/ {', ' location /unrelated-new-route/ { return 204; }\n location /webhooks/ {')) events = [] integration.activate_public_integration_route( prepared, activate=lambda path, content, **kwargs: events.append( ('activate', path, content, kwargs)), probe=lambda origin: events.append(('probe', origin)), ) self.assertEqual(['activate', 'probe'], [event[0] for event in events]) self.assertEqual(nginx, events[0][1]) self.assertIn('127.0.0.1:3403', events[0][2]) self.assertIn('location /unrelated-new-route/', events[0][2]) self.assertEqual( 'https://fixture.invalid/health', events[0][3]['healthcheck_url']) self.assertEqual('https://fixture.invalid', events[1][1]) def test_public_webhook_route_retry_reloads_when_file_already_matches(self): current = '''server { server_name fixture.invalid; location /webhooks/ { proxy_pass http://127.0.0.1:3403; } } ''' nginx = self.root / 'test2.conf' nginx.write_text(current) prepared = integration.preflight_public_integration_route(self.root, nginx) events = [] integration.activate_public_integration_route( prepared, activate=lambda path, content, **kwargs: events.append( ('activate', path, content, kwargs)), probe=lambda origin: events.append(('probe', origin)), ) self.assertEqual(['activate', 'probe'], [event[0] for event in events]) self.assertEqual(current, events[0][2]) def test_hubspot_secret_provisioning_is_owner_and_binding_scoped(self): envv = { 'EXP_HUBSPOT_ACCESS_TOKEN': 'provider-token', 'NEXUSCRM_NC_MASTER_KEY': base64.b64encode(bytes(32)).decode(), 'TIDB_DATABASE': 'fixture', } queries = [] statements = [] def query(_envv, sql, *, database): self.assertEqual('fixture', database) queries.append(sql) return 'owned-credential' if len(queries) == 1 else '' common.provision_hubspot_private_app_named_credential_token( envv, query=query, execute=lambda _envv, sql, *, database: statements.append((database, sql)), encrypt=lambda token, key: f'encrypted:{token}:{len(base64.b64decode(key))}', ) self.assertIn("OwnerPack='hubspot-crm-connector'", queries[0]) self.assertIn('_System_NamedCredentialSecret', queries[1]) self.assertEqual(1, len(statements)) self.assertIn("'owned-credential', 'Token'", statements[0][1]) self.assertIn('NamedCredentialId = VALUES(NamedCredentialId)', statements[0][1]) def test_hubspot_secret_provisioning_rejects_wrong_reserved_binding(self): envv = { 'EXP_HUBSPOT_ACCESS_TOKEN': 'provider-token', 'NEXUSCRM_NC_MASTER_KEY': base64.b64encode(bytes(32)).decode(), 'TIDB_DATABASE': 'fixture', } answers = iter(('owned-credential', 'other-credential\tToken')) with self.assertRaisesRegex(common.DeployError, 'reserved secret ID'): common.provision_hubspot_private_app_named_credential_token( envv, query=lambda *_args, **_kwargs: next(answers), execute=lambda *_args, **_kwargs: self.fail('unsafe binding must not be written'), encrypt=lambda *_args: self.fail('unsafe binding must fail before encryption'), ) def test_webhook_secret_provisioning_is_atomic_owner_scoped_and_verified(self): envv = { 'NEXUSCRM_NC_MASTER_KEY': base64.b64encode(bytes(32)).decode(), 'TIDB_DATABASE': 'fixture', } answers = iter(( '1', 'webhook-id', '1', 'workflow-action-id', '', '', '1', 'encrypted:canonical-secret', '1', 'encrypted:canonical-secret', )) queries = [] statements = [] def query(_envv, sql, *, database): self.assertEqual('fixture', database) queries.append(sql) return next(answers) verified = common.provision_hubspot_webhook_secret_values( envv, 'canonical-secret', query=query, execute=lambda _envv, sql, *, database: statements.append((database, sql)), encrypt=lambda plaintext, _key: f'encrypted:{plaintext}', decrypt=lambda ciphertext, _key: ciphertext.removeprefix('encrypted:'), ) self.assertEqual(2, verified) self.assertEqual(1, len(statements)) self.assertTrue(statements[0][1].startswith('START TRANSACTION;')) self.assertTrue(statements[0][1].endswith('COMMIT;')) self.assertEqual(2, statements[0][1].count('INSERT INTO _System_WebhookSecretValue')) self.assertTrue(all("OwnerPack = 'hubspot-crm-connector'" in sql for sql in queries[:4])) def test_webhook_secret_provisioning_rejects_wrong_reserved_binding(self): envv = { 'NEXUSCRM_NC_MASTER_KEY': base64.b64encode(bytes(32)).decode(), 'TIDB_DATABASE': 'fixture', } answers = iter(('1', 'webhook-id', '1', 'workflow-action-id', 'other-id')) with self.assertRaisesRegex(common.DeployError, 'reserved value ID'): common.provision_hubspot_webhook_secret_values( envv, 'canonical-secret', query=lambda *_args, **_kwargs: next(answers), execute=lambda *_args, **_kwargs: self.fail('unsafe binding must not be written'), encrypt=lambda *_args: self.fail('unsafe binding must fail before encryption'), decrypt=lambda *_args: self.fail('unsafe binding must fail before decryption'), ) def test_credential_is_reconciled_and_verified_without_persisting_plaintext(self): self.source['ENV_NAME'] = 'test-test2' self.write_inputs() original = (self.root / 'env/runtime.env').read_bytes() events = [] def identify(token): events.append(('identify', token)) return integration.expected_nonprod_hubspot_identity('test-test2') def provision(envv): events.append(('provision', envv.copy())) def provision_webhooks(envv, secret): events.append(('provision_webhooks', envv.copy(), secret)) def admit(envv): events.append(('admit', envv.copy())) integration.reconcile_hubspot_credential( self.root, 'test2-provider-token', 'test2-client-secret', identify=identify, provision=provision, provision_webhooks=provision_webhooks, admit=admit) self.assertEqual( ['identify', 'provision', 'provision_webhooks', 'admit'], [event[0] for event in events], ) self.assertEqual('test2-provider-token', events[1][1]['EXP_HUBSPOT_ACCESS_TOKEN']) self.assertEqual('test2-client-secret', events[2][2]) self.assertEqual(original, (self.root / 'env/runtime.env').read_bytes()) def test_credential_reconciliation_rejects_bad_inputs_before_database_access(self): cases = ( ('', 'test2-client-secret', 'TEST2_HUBSPOT_ACCESS_TOKEN'), (' leading', 'test2-client-secret', 'TEST2_HUBSPOT_ACCESS_TOKEN'), ('trailing ', 'test2-client-secret', 'TEST2_HUBSPOT_ACCESS_TOKEN'), ('line\nbreak', 'test2-client-secret', 'TEST2_HUBSPOT_ACCESS_TOKEN'), ('test2-provider-token', '', 'TEST2_HUBSPOT_CLIENT_SECRET'), ('test2-provider-token', 'too-short', 'TEST2_HUBSPOT_CLIENT_SECRET'), ('test2-provider-token', 'line\nbreak-secret', 'TEST2_HUBSPOT_CLIENT_SECRET'), ) for token, secret, error in cases: with self.subTest(token=token, secret=secret), self.assertRaisesRegex(RuntimeError, error): integration.reconcile_hubspot_credential( self.root, token, secret, identify=lambda _: self.fail('identity lookup must not run'), provision=lambda _: self.fail('database write must not run'), provision_webhooks=lambda *_: self.fail('database write must not run')) def test_production_admits_only_exact_owner_repository_images(self): for image in ('local-fixture:new', 'sha256:' + 'a' * 64, 'ghcr.io/pingcap-inc/nexuscrm-integration-service:latest', 'ghcr.io/pingcap-inc/another-service@sha256:' + 'a' * 64): with self.subTest(image=image), self.assertRaisesRegex(RuntimeError, 'immutable'): integration.require_image(image) integration.require_image('ghcr.io/pingcap-inc/nexuscrm-integration-service@sha256:' + 'a' * 64) def test_configuration_uses_canonical_authority_and_preserves_pool_and_tuning(self): env, token, ca = self.configuration() values = dict(row.split('=', 1) for row in env.decode().splitlines()) self.assertEqual(self.serving['app_installation_id'], values['NEXUSCRM_INTEGRATION_APP_INSTALLATION_ID']) self.assertEqual(self.source['TIDB_PASSWORD'], values['TIDB_PASSWORD']) self.assertEqual('100', values['NEXUSCRM_CONNECTOR_RECEIPT_POLL_INTERVAL_MS']) self.assertEqual(('8', '4'), (values['NEXUSCRM_DB_MAX_OPEN_CONNS'], values['NEXUSCRM_DB_MAX_IDLE_CONNS'])) self.assertEqual(b'fixture-token\n', token) self.assertEqual(self.ca, ca.read_bytes()) self.assertNotIn(b'must-not-be-mounted', env + token) def test_missing_ambiguous_or_disabled_authority_fails(self): for change in ('missing-key', 'duplicate-token', 'missing-workload', 'disabled-receipts', 'connected-mail'): with self.subTest(change=change): if change == 'missing-key': self.source['NEXUSCRM_NC_MASTER_KEY'] = '' elif change == 'duplicate-token': self.identities['identities'].append(self.identities['identities'][0].copy()) elif change == 'missing-workload': self.serving['authorized_workload_ids'] = [] elif change == 'disabled-receipts': self.source['NEXUSCRM_CONNECTOR_VERIFIED_INGRESS_RECEIPTS'] = 'false' else: self.source['NEXUSCRM_INTEGRATION_CONNECTED_MAIL_DEFINITION_FILE'] = '/unowned/provider.json' self.write_inputs() with self.assertRaises(RuntimeError): self.configuration() # Restore all authorities for the next independently meaningful case. self.source['NEXUSCRM_NC_MASTER_KEY'] = base64.b64encode(bytes(32)).decode() self.source['NEXUSCRM_CONNECTOR_VERIFIED_INGRESS_RECEIPTS'] = 'true' self.source.pop('NEXUSCRM_INTEGRATION_CONNECTED_MAIL_DEFINITION_FILE', None) self.identities['identities'] = self.identities['identities'][:2] self.serving['authorized_workload_ids'] = ['integration-service'] def test_public_origin_is_validated_before_rendering(self): for origin in ('http://fixture.invalid', '/', 'https://', 'https://fixture.invalid/path', 'https://user:secret@fixture.invalid', 'https://fixture.invalid?query', 'https://fixture.invalid#fragment', 'https://fixture.invalid:bad', 'https://fixture.invalid:65536', 'https://fixture.invalid:0', 'https://fixture.invalid:', 'https://bad host.invalid', 'https://fixture.invalid?'): with self.subTest(origin=origin): self.source['PUBLIC_BASE_URL'] = origin self.write_inputs() with self.assertRaisesRegex(RuntimeError, 'PUBLIC_BASE_URL'): self.configuration() for origin, expected in (('https://fixture.invalid/', 'https://fixture.invalid'), ('https://fixture.invalid:8443', 'https://fixture.invalid:8443')): self.source['PUBLIC_BASE_URL'] = origin self.write_inputs() env, _, _ = self.configuration() values = dict(row.split('=', 1) for row in env.decode().splitlines()) self.assertEqual(expected, values['WEBHOOK_BASE_URL']) def test_cli_rejects_retired_operations_and_bad_images_before_host_access(self): script = Path(integration.__file__) for args in (['start'], ['upgrade'], ['deploy', '--image', 'latest'], ['stop', '--image', 'latest']): result = subprocess.run([sys.executable, str(script), *args], capture_output=True, text=True, timeout=10) self.assertNotEqual(0, result.returncode) self.assertNotIn('Docker operation failed', result.stderr) @unittest.skipUnless(os.environ.get('STANDALONE_DOCKER_TESTS') == '1', 'requires real Docker') class IntegrationDockerTest(ConfigurationFixture): BASE = 'busybox@sha256:dc2d74b28e4cf8984fa52af1f39bc7c3d9c73760b41a74d629f5d11b1ab28616' @classmethod def setUpClass(cls): super().setUpClass() cls.images = [] cls.addClassCleanup(cls.remove_images) try: architecture = integration.docker('image', 'inspect', cls.BASE, '--format', '{{.Architecture}}') except subprocess.CalledProcessError: architecture = '' if architecture != 'amd64': integration.docker('pull', '--platform', 'linux/amd64', cls.BASE) # Reuse the public base already used by StandaloneDockerTest. The fixture # serves a real HTTP endpoint; it is not a simulated Integration runtime. # No private registry login or application build is needed by PR CI. with tempfile.TemporaryDirectory() as raw: archive = Path(raw) / 'rootfs.tar' carrier = integration.docker('create', '--platform', 'linux/amd64', cls.BASE, 'true') try: integration.docker('export', '--output', str(archive), carrier) finally: integration.docker('rm', carrier) with tarfile.open(archive, 'a') as stream: files = { 'usr/local/bin/integration-service': b'#!/bin/sh\ntest -r "$NEXUSCRM_INTEGRATION_WORKLOAD_TOKEN_FILE" || exit 2\n' b'test -s "$NEXUSCRM_INTEGRATION_RUNTIME_CA_FILE" || exit 3\n' b'trap \'kill "$child"; exit 0\' TERM INT\n' b'/bin/httpd -f -p "$NEXUSCRM_INTEGRATION_HTTP_ADDR" -h /www &\nchild=$!\nwait "$child"\n', 'www/readyz': b'ready\n', } for name, value in files.items(): entry = tarfile.TarInfo(name) entry.size, entry.mode = len(value), 0o755 if name.endswith('integration-service') else 0o644 stream.addfile(entry, io.BytesIO(value)) for command in ('ENTRYPOINT ["/usr/local/bin/integration-service"]', 'ENTRYPOINT ["/bin/false"]'): cls.images.append(integration.docker( 'import', '--platform', 'linux/amd64', '--change', command, '--change', 'LABEL org.opencontainers.image.revision=' + SOURCE, str(archive))) cls.image, cls.bad_image = cls.images @classmethod def remove_images(cls): for image in cls.images: integration.docker('image', 'rm', image) def setUp(self): super().setUp() self.name = 'test2-integration-case-' + uuid.uuid4().hex self.names = [self.name] self.addCleanup(self.remove_containers) with socket.socket() as listener: listener.bind(('127.0.0.1', 0)) self.port = listener.getsockname()[1] self.url = f'http://127.0.0.1:{self.port}/readyz' def remove_containers(self): for name in self.names: item = integration.inspect(name) if item: integration.docker('rm', '-f', item['Id']) def deploy(self, image=None, readiness_url=None): return integration.deploy_container( image or self.image, self.root, name=self.name, address='0.0.0.0:8080', network=('--network', 'bridge', '-p', f'127.0.0.1:{self.port}:8080'), readiness_url=readiness_url or self.url, timeout=5) def test_absent_container_and_interrupted_removal_use_same_runner(self): first = self.deploy() integration.stop_container(integration.inspect(self.name)) integration.docker('rm', first) shutil.rmtree(self.root / '.runtime/integration') self.serving['app_installation_id'] = 'current-installation' self.identities['identities'][0]['token'] = 'current-token' self.write_inputs() second = self.deploy() current = integration.inspect(self.name) self.assertNotEqual(first, second) self.assertTrue(current['State']['Running']) values = dict(row.split('=', 1) for row in current['Config']['Env']) self.assertEqual('current-installation', values['NEXUSCRM_INTEGRATION_APP_INSTALLATION_ID']) token = self.root / '.runtime/integration/integration.token' self.assertEqual('current-token\n', token.read_text()) self.assertEqual(0o400, token.stat().st_mode & 0o777) self.assertTrue(all('/serviceification-d3/' not in m['Source'] for m in current['Mounts'])) self.assertIsNone(integration.inspect(self.name + '-upgrade')) def test_redeploy_uses_authority_and_leaves_sibling_running(self): first = self.deploy() sibling = self.name + '-sibling' self.names.append(sibling) sibling_id = integration.docker('run', '-d', '--name', sibling, '--network', 'none', '--entrypoint', '/bin/sleep', self.image, '300') self.source['NEXUSCRM_CONNECTOR_RECEIPT_POLL_INTERVAL_MS'] = '250' self.write_inputs() second = self.deploy() self.assertNotEqual(first, second) self.assertTrue(integration.inspect(sibling)['State']['Running']) self.assertEqual(sibling_id, integration.inspect(sibling)['Id']) values = dict(row.split('=', 1) for row in integration.inspect(self.name)['Config']['Env']) self.assertEqual('250', values['NEXUSCRM_CONNECTOR_RECEIPT_POLL_INTERVAL_MS']) def test_preflight_errors_preserve_running_service(self): cid = self.deploy() started_at = integration.inspect(self.name)['State']['StartedAt'] with self.assertRaisesRegex(RuntimeError, 'process command'): self.deploy(image=self.bad_image) for origin in ('http://fixture.invalid', '/', 'https://'): with self.subTest(origin=origin): self.source['PUBLIC_BASE_URL'] = origin self.write_inputs() with self.assertRaisesRegex(RuntimeError, 'PUBLIC_BASE_URL'): self.deploy() current = integration.inspect(self.name) self.assertEqual(cid, current['Id']) self.assertTrue(current['State']['Running']) self.assertEqual(started_at, current['State']['StartedAt']) with integration.urllib.request.urlopen(self.url, timeout=3) as response: self.assertEqual(200, response.status) self.source['PUBLIC_BASE_URL'] = 'https://fixture.invalid' self.source['NEXUSCRM_NC_MASTER_KEY'] = '' self.write_inputs() with self.assertRaisesRegex(RuntimeError, 'input'): self.deploy() self.assertEqual(cid, integration.inspect(self.name)['Id']) self.assertTrue(integration.inspect(self.name)['State']['Running']) def test_readiness_failure_leaves_stopped_process_and_retry_recovers(self): with self.assertRaisesRegex(RuntimeError, 'did not become ready'): self.deploy(readiness_url=f'http://127.0.0.1:{self.port}/missing') self.assertFalse(integration.inspect(self.name)['State']['Running']) self.deploy() self.assertTrue(integration.inspect(self.name)['State']['Running']) def test_unowned_container_is_not_stopped(self): cid = integration.docker('run', '-d', '--name', self.name, '--network', 'none', '--entrypoint', '/bin/sleep', self.image, '300') with self.assertRaisesRegex(RuntimeError, 'not an owned'): self.deploy() self.assertEqual(cid, integration.inspect(self.name)['Id']) self.assertTrue(integration.inspect(self.name)['State']['Running']) if __name__ == '__main__': unittest.main()