//go:build functional

package spectests_test

import (
	"bytes"
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"regexp"
	"slices"
	"strconv"
	"strings"
	"syscall"
	"testing"
	"time"

	"github.com/tidbcloud/controlplane9/internal/config"
	"github.com/tidbcloud/controlplane9/internal/tilt"
)

const (
	githubAppFunctionalEnabled = "CP9_GITHUB_APP_ENABLED"
	githubAppRetryStageEnv     = "CP9_GITHUB_APP_RETRY_STAGE"
	githubAppDestroyEnv        = "CP9_GITHUB_APP_DESTROY_SESSION"
)

var (
	githubAppUserCode         = regexp.MustCompile(`^[A-Z0-9-]+$`)
	githubAppUserID           = regexp.MustCompile(`^[0-9]+$`)
	errGitHubAppActionReceipt = errors.New("invalid GitHub App action receipt")
)

type githubAppRun struct {
	t         *testing.T
	repoRoot  string
	root      string
	journal   githubAppJournal
	commandID int
	resume    bool
	retry     string
}

type githubAppJournal struct {
	Version          int                                `json:"version"`
	Case             string                             `json:"case"`
	CP9Head          string                             `json:"cp9_head"`
	Hostname         string                             `json:"hostname"`
	Username         string                             `json:"username"`
	UID              int                                `json:"uid"`
	Owner            string                             `json:"owner,omitempty"`
	ClientID         string                             `json:"client_id,omitempty"`
	CachePreexisted  *bool                              `json:"cache_preexisted,omitempty"`
	HumanActionCount int                                `json:"human_action_count"`
	Stages           map[string]githubAppStageReceipt   `json:"stages"`
	Attempts         map[string][]githubAppStageReceipt `json:"attempts,omitempty"`
	Actions          map[string]githubAppActionReceipt  `json:"actions"`
	Outputs          map[string]githubAppCommandOutput  `json:"outputs"`
}

type githubAppActionReceipt struct {
	Status    string   `json:"status"`
	Head      string   `json:"head"`
	Directory string   `json:"directory"`
	Command   []string `json:"command"`
	ExitCode  int      `json:"exit_code"`
}

type githubAppCommandOutput struct {
	Stdout string `json:"stdout"`
	Stderr string `json:"stderr"`
}

type githubAppStageReceipt struct {
	Status   string `json:"status"`
	Product  string `json:"product,omitempty"`
	Harness  string `json:"harness,omitempty"`
	Security string `json:"security,omitempty"`
	Cleanup  string `json:"cleanup,omitempty"`
	Message  string `json:"message,omitempty"`
}

type githubAppSession struct {
	Stage      string
	Label      string
	Service    string
	Evidence   string
	CP9Root    string
	Config     string
	Name       string
	SourceRoot string
}

type githubAppSessionMetadata struct {
	Account  string `json:"account"`
	Region   string `json:"region"`
	Session  string `json:"session"`
	Owner    string `json:"owner"`
	TTL      string `json:"ttl"`
	CP9Root  string `json:"cp9_root"`
	Config   string `json:"config"`
	WorkRoot string `json:"work_root"`
	Head     string `json:"head"`
}

type githubAppTokenCache struct {
	Version          int    `json:"version"`
	AccessToken      string `json:"access_token"`
	ExpiresAt        int64  `json:"expires_at"`
	ClientID         string `json:"client_id"`
	GitHubUserID     string `json:"github_user_id"`
	GitHubLogin      string `json:"github_login"`
	RefreshToken     string `json:"refresh_token,omitempty"`
	RefreshExpiresAt int64  `json:"refresh_expires_at,omitempty"`
}

type githubAppHumanAction struct {
	Version          int    `json:"version"`
	Type             string `json:"type"`
	VerificationURI  string `json:"verification_uri"`
	UserCode         string `json:"user_code"`
	ExpiresInSeconds int    `json:"expires_in_seconds"`
	Continuation     string `json:"continuation"`
}

type githubAppCommandReceipt struct {
	Command    []string `json:"command"`
	StartedAt  string   `json:"started_at"`
	FinishedAt string   `json:"finished_at"`
	DurationMS int64    `json:"duration_ms"`
	ExitCode   int      `json:"exit_code"`
	Stdout     string   `json:"stdout"`
	Stderr     string   `json:"stderr"`
	Error      string   `json:"error,omitempty"`
}

// # Source build reads private modules through the user's GitHub App authorization
//
// This case proves one user journey: a clean remote source build needs private
// GitHub modules, the current developer authorizes once through GitHub App Device
// Flow, and later builds and sessions on the same machine reuse or refresh that
// authorization without another human step.
//
// It covers issue [#1000](https://github.com/tidbcloud/controlplane9/issues/1000)
// and issues [#1197](https://github.com/tidbcloud/controlplane9/issues/1197) and
// [#1199](https://github.com/tidbcloud/controlplane9/issues/1199), plus PR
// [#999](https://github.com/tidbcloud/controlplane9/pull/999).
//
// ## Agent execution
//
// **Mode: single agent with human authorization.** Spawn no sub-agents. The
// parent keeps the authorization prompt, sanitized local credential metadata,
// product verdict, and cleanup inside one credential boundary. The human completes
// Device Flow in the browser only when no valid or refreshable local authorization
// exists.
//
// When authorization is required, CP9 emits exactly one delimited
// `CP9_HUMAN_ACTION` JSON event with the verification URI, user code, expiry, and
// `"continuation":"automatic"`. The event is public operator input, not a GitHub
// token. The original CP9 command keeps polling and continues after approval; the
// operator never issues a second command to resume it.
//
// Go functional entrypoint:
//
// ```text
// CP9_GITHUB_APP_ENABLED=1 \
// CP9_UI_API_AWS_KEY_PAIR='<existing-key-pair>' \
// CP9_UI_API_SSH_PRIVATE_KEY='<path-to-matching-private-key>' \
// CP9_GITHUB_APP_SECURITY_RECEIPT='<security-run>/remote-security-receipt.json' \
// GOTOOLCHAIN=go1.26.1 GO_FUNCTIONAL_TEST_PACKAGES=./spec-tests \
// GO_FUNCTIONAL_TEST_RUN='^TestFunctional_SourceBuildGitHubApp$$' make test-functional
// ```
//
// ## Default run
//
// - AWS account `804837308162`, region `us-west-2`, source profile
//   `dev-offline-afterworld`, and role `cp9-developer`.
// - `services.github_app_client_id` is prefilled with the approved CP9 GitHub App
//   in this case's generated `cp9.yaml`; no separate App input is required.
// - Two sequential fresh 10-hour `remote-ec2-k3d` sessions with a caller-owned
//   existing EC2 key pair plus matching private key supplied through
//   `CP9_UI_API_AWS_KEY_PAIR` and `CP9_UI_API_SSH_PRIVATE_KEY`, isolated exact-head
//   CP9 worktrees, and one clean shallow `tidbcloud/tidb-management-service`
//   checkout per session. Session A selects and source-builds only `tempo-proxy`;
//   after its normal destroy, session B selects and source-builds only
//   `tempo-wrapper` as the same local user.
// - The instruction to execute this case authorizes both sequential session
//   setups, both deploys, sanitized shared-credential inspection, and both
//   exact-session cleanups.
//   Do not ask again.
//
// The human completes GitHub Device Flow in a browser only if CP9 has no reusable
// local grant. No PAT, App private key, client secret, access token, or refresh
// token may be copied into session state, EC2, logs, or evidence. The machine
// needs `gh`, `jq`, `python3`, `ssh`, and AWS CLI.
//
// ## Execute
//
// The Go test captures the raw one-time credential prompt and sanitized cache
// metadata, journals each named phase outside session cleanup, and keeps exact
// cloud cleanup explicit. `make test-functional` runs the Go test verbosely, so a
// Device Flow event is streamed through the active test output even though the
// child test process does not own a terminal. Relay that exact public event to the
// named human without stopping or rerunning the active `cp9 up`.
//
// ### 1. Freeze and prepare the run
//
// Use the Go entrypoint above. Freeze the selected CP9 and both
// `tidb-management-service` revisions. Before creating session A, run the
// focused integration contract
// `TestIntegration_GitHubAppAuthorizationIsSharedRefreshedAndRevoked`. It uses
// fake GitHub endpoints and must prove forced-expiry refresh and rotation,
// concurrent access through one owner-only lock/cache, invalid or revoked-token
// fallback to one new Device Flow, and no token in argv, logs, repositories, or
// remote-session inputs. This is Harness/Security contract evidence, not a
// substitute for the live Product journey below.
//
// Then create session A. Before mutation:
//
// - run setup, doctor, and a non-mutating plan;
// - require the declared account, region, owner, session, and TTL;
// - record cleanup authority;
// - read the approved GitHub App client ID from the generated CP9 config and
//   require it to be present;
// - retain exact-session cleanup ownership, but preserve the current session on
//   the first unexpected red. Do not create session B until session A has a
//   passing destroy receipt.
// - rewrite each isolated Session Tiltfile to select only that session's named
//   service, so the two-session proof performs two private source builds rather
//   than rebuilding both services twice.
//
// On first red, keep the run directory and current session. After diagnosis, set
// `CP9_GITHUB_APP_RESUME=1`, `CP9_GITHUB_APP_RUN_DIR` to that exact directory,
// and `CP9_GITHUB_APP_RETRY_STAGE` to the failed stage, then select only that
// stage with `GO_FUNCTIONAL_TEST_RUN`. A completed `up` or deploy action is reused
// for evidence-only continuation; a started action with an unknown outcome fails
// closed and is never replayed. Earlier failed Product, Harness, or Security
// evidence remains visible in the aggregate verdict after a repaired attempt.
//
// For example, if `up` completes but its follow-up evidence fails, retry that
// stage without repeating `up`:
//
// ```text
// CP9_GITHUB_APP_ENABLED=1 CP9_GITHUB_APP_RESUME=1 \
// CP9_GITHUB_APP_RUN_DIR='<preserved-run-directory>' \
// CP9_GITHUB_APP_RETRY_STAGE='session-a-up-and-source-build' \
// GOTOOLCHAIN=go1.26.1 GO_FUNCTIONAL_TEST_PACKAGES=./spec-tests \
// GO_FUNCTIONAL_TEST_RUN='^TestFunctional_SourceBuildGitHubApp$$/^session-a-up-and-source-build$$' \
// make test-functional
// ```
//
// ### 2. Reuse or complete one user authorization
//
// Run `cp9 up`. If CP9 shows a GitHub Device Flow instruction, record it exactly
// as shown on the live terminal and let the current user complete it. Require a
// prominent `CP9 HUMAN ACTION REQUIRED` banner followed by one machine-readable
// `CP9_HUMAN_ACTION` JSON line containing schema version `1`, type
// `github_device_flow`, the exact GitHub verification URI, a nonempty user code,
// a positive expiry, and automatic continuation. Do not rerun `cp9 up`: the same
// process must become ready after approval. If the same App and GitHub identity
// already have a valid or refreshable authorization on this machine, no event
// should appear and the command should continue without a human step.
//
// After the session becomes ready, inspect only sanitized metadata for the local
// shared GitHub App authorization. Require:
//
// - file mode `0600`;
// - ownership by the current local user;
// - the expected client ID and a GitHub user ID/login;
// - an access token with either no expiry or more than five minutes remaining;
// - a refresh token and expiry when GitHub provides one;
// - no `github-app-user-token.json` or other credential in session state.
//
// Retain only client ID, GitHub identity, remaining lifetime, refresh availability,
// and file mode. Never copy token values into the run directory or evidence.
//
// ### 3. Prove a later session reuses the authorization
//
// Deploy `tempo-proxy` in session A, destroy that exact session, and require the
// owner-only shared authorization to remain. Then create session B from the same
// exact CP9 head and App client ID and deploy `tempo-wrapper`. Record the complete
// stdout and stderr for both commands without changing or masking any value. Both
// private-module builds and deploys must succeed.
//
// Across both `up` commands and both deploys, at most one `CP9_HUMAN_ACTION` event
// may appear. It may appear only in session A's `up`; session B and both deploys
// must add none. A second browser flow, a failed private-module download, a failed
// deploy, or a need to rerun an original command after approval is `FAIL`.
//
// The product behavior ends after both builds reuse the authorization. Remote
// credential residue is a separate security check described in
// [`tools/github-credential-residue/README.md`](../tools/github-credential-residue/README.md);
// do not rerun this product case merely to validate that harness.
//
// ### 4. Clean up
//
// Destroy only each recorded session and isolated worktree, and record separate
// cleanup receipts. Require the shared local GitHub authorization to remain after
// each destroy.
// The caller-owned key pair is outside CP9 cleanup; no key-pair ownership or
// deletion receipt is expected. A completed run requires passing destroy and
// either a passing or unnecessary worktree receipt.
//
// If a failed stage preserves a session, explicitly clean only that recorded
// session after disposition. Select its destroy stage and name the same stage as
// the retry only when its prior cleanup attempt failed:
//
// ```text
// CP9_GITHUB_APP_ENABLED=1 CP9_GITHUB_APP_RESUME=1 \
// CP9_GITHUB_APP_RUN_DIR='<preserved-run-directory>' \
// CP9_GITHUB_APP_DESTROY_SESSION=1 \
// CP9_GITHUB_APP_RETRY_STAGE='session-a-destroy' \
// GOTOOLCHAIN=go1.26.1 GO_FUNCTIONAL_TEST_PACKAGES=./spec-tests \
// GO_FUNCTIONAL_TEST_RUN='^TestFunctional_SourceBuildGitHubApp$$/^session-a-destroy$$' \
// make test-functional
// ```
//
// Omit `CP9_GITHUB_APP_RETRY_STAGE` when the selected destroy stage has no failed
// attempt yet. A retry reuses a previously passing cleanup receipt for remaining
// evidence checks; a failed cleanup command may rerun because the exact-session
// helper is idempotent.
//
// ## Verdicts
//
// ### Product
//
// Product `PASS` requires both sessions' doctor and build success using the same
// client ID in `cp9.yaml`, no more than one current-user authorization, a
// prominent structured human-action event when authorization is required,
// automatic continuation of the same command, successful private module download
// and deploy in each session, and safe cross-session local cache reuse. Duplicate
// Device Flows or inability to download and deploy the private modules is product
// `FAIL`.
//
// Unavailable human authorization, expired `dev-offline-afterworld` login,
// repository access, or required tools is `NOT RUN`; do not replace Device Flow
// with a PAT. The public client ID and normal session inputs are defaults, not
// questions.
//
// ### Security
//
// Security `PASS` requires a separate receipt showing zero remote credential
// residue, an owner-only local shared store, and no token in argv, logs, session
// state, repositories, or evidence. A refresh token retained only in that
// owner-only shared store is expected. An unavailable or unreadable residue probe
// is security `NOT RUN` and a follow-up, and does not rewrite direct product
// evidence.
//
// Revoking the user's shared GitHub grant is outside this case's mutation
// authority and is `NOT RUN` as a live action; exercise it live only with a
// separately authorized throwaway GitHub identity. The focused fake-endpoint
// contract must still cover revoked/invalid fallback without changing the user's
// grant. Remote credential residue is reported separately by the security case
// above; its `pass`, `fail`, or `not_run` result does not replace this product
// verdict.
//
// ### Harness
//
// Harness `PASS` requires the focused refresh/concurrency/revocation contract,
// event count, JSON schema, banner, sanitized cache, and build observations to
// complete correctly. A locator, parser, test, or evidence-capture defect is
// harness `FAIL`; preserve direct product and security observations and the
// current session, and mark only unobserved product assertions `NOT RUN`. Harness
// is `NOT RUN` only when a prerequisite stops the case before any probe begins.
//
// ### Cleanup
//
// Cleanup `PASS` requires both exact sessions and isolated worktrees removed with
// their receipts recorded. The caller-owned key pair is not a run-owned resource
// and must not be deleted. Known residue or a missing receipt is cleanup `FAIL`,
// not product or security `FAIL`. Cleanup is `NOT RUN` only when the case created
// no run-owned state.

func TestFunctional_SourceBuildGitHubApp(t *testing.T) {
	if !enabled(os.Getenv(githubAppFunctionalEnabled)) {
		t.Skipf("set %s=1 to run the private-module GitHub App journey", githubAppFunctionalEnabled)
	}

	run := newGitHubAppRun(t)
	if enabled(os.Getenv(githubAppDestroyEnv)) {
		t.Run("session-a-destroy", func(t *testing.T) {
			run.runStage(t, "session-a-destroy", func(t *testing.T) { run.destroy(t, "session-a") })
		})
		t.Run("session-b-destroy", func(t *testing.T) {
			run.runStage(t, "session-b-destroy", func(t *testing.T) { run.destroy(t, "session-b") })
		})
		return
	}
	t.Run("authorization-prerequisites", func(t *testing.T) {
		run.runStage(t, "authorization-prerequisites", run.authorizationPrerequisites)
	})
	t.Run("session-a-up-and-source-build", func(t *testing.T) {
		run.runStage(t, "session-a-up-and-source-build", func(t *testing.T) {
			run.requirePassed(t, "authorization-prerequisites")
			run.prepareAndUp(t, &githubAppSession{Stage: "session-a-up-and-source-build", Label: "session-a", Service: "tempo-proxy"})
		})
	})
	t.Run("session-a-deploy", func(t *testing.T) {
		run.runStage(t, "session-a-deploy", func(t *testing.T) {
			run.requirePassed(t, "session-a-up-and-source-build")
			run.deploy(t, run.loadSession(t, "session-a-deploy", "session-a", "tempo-proxy"))
		})
	})
	t.Run("session-a-destroy", func(t *testing.T) {
		run.runStage(t, "session-a-destroy", func(t *testing.T) {
			run.requirePassed(t, "session-a-deploy")
			run.destroy(t, "session-a")
		})
	})
	t.Run("session-b-reuse-and-source-build", func(t *testing.T) {
		run.runStage(t, "session-b-reuse-and-source-build", func(t *testing.T) {
			run.requirePassed(t, "session-a-destroy")
			run.prepareAndUp(t, &githubAppSession{Stage: "session-b-reuse-and-source-build", Label: "session-b", Service: "tempo-wrapper"})
		})
	})
	t.Run("session-b-deploy", func(t *testing.T) {
		run.runStage(t, "session-b-deploy", func(t *testing.T) {
			run.requirePassed(t, "session-b-reuse-and-source-build")
			run.deploy(t, run.loadSession(t, "session-b-deploy", "session-b", "tempo-wrapper"))
		})
	})
	t.Run("session-b-destroy", func(t *testing.T) {
		run.runStage(t, "session-b-destroy", func(t *testing.T) {
			run.requirePassed(t, "session-b-deploy")
			run.destroy(t, "session-b")
		})
	})
	t.Run("security-receipt", func(t *testing.T) {
		run.runStage(t, "security-receipt", run.validateSecurityReceipt)
	})
}

func newGitHubAppRun(t *testing.T) *githubAppRun {
	t.Helper()
	repoRoot := repositoryRoot(t)
	head := currentCommit(t, repoRoot)
	resume := enabled(os.Getenv("CP9_GITHUB_APP_RESUME"))
	retry := strings.TrimSpace(os.Getenv(githubAppRetryStageEnv))
	if !resume && (retry != "" || enabled(os.Getenv(githubAppDestroyEnv))) {
		t.Fatal("GitHub App retry and destroy modes require CP9_GITHUB_APP_RESUME=1")
	}
	root := strings.TrimSpace(os.Getenv("CP9_GITHUB_APP_RUN_DIR"))
	if resume && root == "" {
		t.Fatal("CP9_GITHUB_APP_RUN_DIR is required with CP9_GITHUB_APP_RESUME=1")
	}
	if !resume && root != "" {
		t.Fatal("CP9_GITHUB_APP_RUN_DIR is only accepted with CP9_GITHUB_APP_RESUME=1")
	}
	if root == "" {
		root = filepath.Join(repoRoot, "artifacts", "spec-tests", time.Now().UTC().Format("20060102T150405Z")+"-private-module-device-flow")
		if err := os.MkdirAll(root, 0o700); err != nil {
			t.Fatalf("create GitHub App evidence directory: %v", err)
		}
	}
	absRoot, err := filepath.Abs(root)
	if err != nil {
		t.Fatalf("resolve GitHub App evidence directory: %v", err)
	}
	hostname, err := os.Hostname()
	if err != nil {
		t.Fatalf("resolve hostname: %v", err)
	}
	currentUser, err := user.Current()
	if err != nil {
		t.Fatalf("resolve local user: %v", err)
	}
	username := currentUser.Username
	run := &githubAppRun{
		t: t, repoRoot: repoRoot, root: absRoot, resume: resume,
		retry: retry, commandID: githubAppPriorCommandID(absRoot),
	}
	if resume {
		run.readJSON(t, "journal.json", &run.journal)
		if run.journal.Version != 1 || run.journal.Case != "source-build-private-repo-with-github-app" ||
			run.journal.CP9Head != head || run.journal.Hostname != hostname ||
			run.journal.Username != username || run.journal.UID != os.Geteuid() {
			t.Fatal("resume journal does not match this case, CP9 head, machine, or local user")
		}
	} else {
		run.journal = githubAppJournal{
			Version: 1, Case: "source-build-private-repo-with-github-app", CP9Head: head,
			Hostname: hostname, Username: username, UID: os.Geteuid(),
			Stages: map[string]githubAppStageReceipt{}, Attempts: map[string][]githubAppStageReceipt{},
			Actions: map[string]githubAppActionReceipt{}, Outputs: map[string]githubAppCommandOutput{},
		}
		run.persistJournal(t)
	}
	if run.journal.Stages == nil {
		t.Fatal("resume journal has no stages")
	}
	if run.journal.Attempts == nil {
		run.journal.Attempts = map[string][]githubAppStageReceipt{}
	}
	if run.journal.Actions == nil {
		run.journal.Actions = map[string]githubAppActionReceipt{}
	}
	if run.journal.Outputs == nil {
		run.journal.Outputs = map[string]githubAppCommandOutput{}
	}
	for _, label := range []string{"session-a", "session-b"} {
		if pathExists(filepath.Join(run.root, label, "default-804-session.json")) {
			run.markCleanupPending(t, label)
		}
	}
	t.Cleanup(func() { run.writeSummary(t) })
	return run
}

func (r *githubAppRun) runStage(t *testing.T, name string, body func(*testing.T)) {
	t.Helper()
	previous := r.journal.Stages[name]
	if previous.Status == "pass" {
		t.Skip("stage already passed in the matching resume journal")
	}
	if previous.Status == "running" || previous.Status == "fail" {
		if !r.resume || r.retry != name {
			t.Fatalf("stage is %s in the preserved journal; set %s=%s only after diagnosing a safe evidence-only continuation", previous.Status, githubAppRetryStageEnv, name)
		}
		r.journal.Attempts[name] = append(r.journal.Attempts[name], previous)
	}
	r.journal.Stages[name] = githubAppStageReceipt{Status: "running", Cleanup: previous.Cleanup}
	r.persistJournal(t)
	t.Cleanup(func() {
		receipt := r.journal.Stages[name]
		if receipt.Status == "running" {
			if t.Failed() {
				receipt.Status = "fail"
				if receipt.Harness == "" {
					receipt.Harness = "fail"
				}
			} else if t.Skipped() {
				receipt.Status = "not_run"
			} else {
				receipt.Status = "pass"
			}
			r.journal.Stages[name] = receipt
			r.persistJournal(t)
		}
	})
	body(t)
	receipt := r.journal.Stages[name]
	receipt.Status = "pass"
	r.journal.Stages[name] = receipt
	r.persistJournal(t)
}

func (r *githubAppRun) authorizationPrerequisites(t *testing.T) {
	t.Helper()
	for _, tool := range []string{"aws", "gh", "git", "go", "jq", "make", "python3", "ssh"} {
		if _, err := exec.LookPath(tool); err != nil {
			r.fail(t, "authorization-prerequisites", "harness", "required tool %s is unavailable: %v", tool, err)
		}
	}
	if os.Getenv("CP9_UI_API_AWS_KEY_PAIR") == "" || os.Getenv("CP9_UI_API_SSH_PRIVATE_KEY") == "" {
		r.fail(t, "authorization-prerequisites", "harness", "CP9_UI_API_AWS_KEY_PAIR and CP9_UI_API_SSH_PRIVATE_KEY are required")
	}
	err := r.command(10*time.Minute, r.repoRoot, "authorization-contract", false,
		"go", "test", "-count=1", "-tags=integration", "-run", "^TestIntegration_GitHubAppAuthorizationIsSharedRefreshedAndRevoked$", "./internal/tilt")
	if err != nil {
		r.fail(t, "authorization-prerequisites", "harness", "focused GitHub App authorization contract: %v", err)
	}
	r.setLanes(t, "authorization-prerequisites", "not_run", "pass", "not_run", "")
}

func (r *githubAppRun) prepareAndUp(t *testing.T, session *githubAppSession) {
	t.Helper()
	session.Evidence = filepath.Join(r.root, session.Label)
	if err := os.MkdirAll(session.Evidence, 0o700); err != nil {
		r.fail(t, r.stageName(session), "harness", "create session evidence directory: %v", err)
	}
	metadataPath := filepath.Join(session.Evidence, "default-804-session.json")
	if !pathExists(metadataPath) {
		prepare := filepath.Join(r.repoRoot, "spec-tests", "scripts", "default-804-session.sh")
		err := r.command(20*time.Minute, r.repoRoot, session.Label+"-prepare", false,
			prepare, "prepare", "private", session.Evidence)
		if err != nil {
			if pathExists(metadataPath) {
				r.markCleanupPending(t, session.Label)
			}
			r.fail(t, r.stageName(session), "harness", "prepare %s: %v", session.Label, err)
		}
	}
	r.markCleanupPending(t, session.Label)
	metadata := githubAppSessionMetadata{}
	r.readJSONPath(t, metadataPath, &metadata)
	if metadata.Account != "804837308162" || metadata.Region != "us-west-2" || metadata.Session == "" ||
		metadata.Owner == "" || metadata.TTL == "" || metadata.Head != r.journal.CP9Head {
		r.fail(t, r.stageName(session), "harness", "%s metadata does not match the frozen account, region, session, owner, TTL, and CP9 head", session.Label)
	}
	if r.journal.Owner == "" {
		r.journal.Owner = metadata.Owner
	} else if metadata.Owner != r.journal.Owner {
		r.fail(t, r.stageName(session), "product", "%s owner differs from session A", session.Label)
	}
	session.CP9Root = metadata.CP9Root
	session.Config = metadata.Config
	session.Name = metadata.Session
	session.SourceRoot = filepath.Join(metadata.WorkRoot, "tidb-management-service")
	r.writeText(t, filepath.Join(session.Label, "cp9-head.txt"), metadata.Head+"\n")
	cleanupAuthority := strings.TrimSpace(os.Getenv("CP9_GITHUB_APP_CLEANUP_AUTHORITY"))
	if cleanupAuthority == "" {
		cleanupAuthority = "selected execution instruction for this functional journey"
	}
	r.writeText(t, filepath.Join(session.Label, "cleanup-authority.txt"), cleanupAuthority+"\n")
	sourceHead := runLocalGit(t, session.SourceRoot, "rev-parse", "HEAD")
	sourceHeadPath := filepath.Join(session.Evidence, "tidb-management-service-head.txt")
	if frozen, err := os.ReadFile(sourceHeadPath); err == nil && strings.TrimSpace(string(frozen)) != sourceHead {
		r.fail(t, r.stageName(session), "harness", "%s source head changed after it was frozen", session.Label)
	} else if err != nil && !os.IsNotExist(err) {
		r.fail(t, r.stageName(session), "harness", "read %s frozen source head: %v", session.Label, err)
	} else if os.IsNotExist(err) {
		r.writeText(t, filepath.Join(session.Label, "tidb-management-service-head.txt"), sourceHead+"\n")
	}

	cfg, err := config.LoadSession(session.Config)
	if err != nil || cfg.Services.GitHubAppClientID == "" || cfg.Tilt.File == "" {
		r.fail(t, r.stageName(session), "harness", "%s config lacks the GitHub App client ID or Tiltfile", session.Label)
	}
	if r.journal.ClientID == "" {
		r.journal.ClientID = cfg.Services.GitHubAppClientID
	} else if cfg.Services.GitHubAppClientID != r.journal.ClientID {
		r.fail(t, r.stageName(session), "product", "%s client ID differs from session A", session.Label)
	}
	r.selectSourceBuild(t, session, cfg.Tilt.File)
	cachePath := githubAppCachePath(t, r.journal.ClientID)
	if session.Label == "session-a" && r.journal.CachePreexisted == nil {
		preexisted := pathExists(cachePath)
		r.journal.CachePreexisted = &preexisted
	}
	r.persistJournal(t)

	cp9 := filepath.Join(session.CP9Root, "bin", "cp9")
	upLabel := session.Label + "-up"
	if !r.actionExists(upLabel) {
		if err := r.command(30*time.Minute, session.CP9Root, session.Label+"-build", false, "make", "build"); err != nil {
			r.fail(t, r.stageName(session), "harness", "build CP9 for %s: %v", session.Label, err)
		}
		if err := r.command(30*time.Minute, session.CP9Root, session.Label+"-setup", false,
			cp9, "setup", "checkout=false", "config="+session.Config); err != nil {
			r.fail(t, r.stageName(session), "harness", "setup %s: %v", session.Label, err)
		}
		if err := r.command(5*time.Minute, session.CP9Root, session.Label+"-dry-run", false,
			cp9, "{sys.output.format=json}", "up", "dry-run=true", "config="+session.Config); err != nil {
			r.fail(t, r.stageName(session), "harness", "dry-run %s: %v", session.Label, err)
		}
		r.validateDryRun(t, session)
		if err := r.command(30*time.Minute, session.CP9Root, session.Label+"-doctor", false,
			cp9, "doctor", "config="+session.Config); err != nil {
			r.fail(t, r.stageName(session), "harness", "doctor %s: %v", session.Label, err)
		}
	}
	if err := r.action(3*time.Hour, session.CP9Root, upLabel, true,
		cp9, "up", "config="+session.Config); err != nil {
		lane := "product"
		if errors.Is(err, errGitHubAppActionReceipt) {
			lane = "harness"
		}
		r.fail(t, r.stageName(session), lane, "cp9 up %s: %v", session.Label, err)
	}
	r.validateHumanActions(t, session)
	r.validateCacheAndResidue(t, session, "after-up")
	if err := r.command(5*time.Minute, session.CP9Root, session.Label+"-status", false,
		cp9, "{sys.output.format=json}", "status", "config="+session.Config); err != nil {
		r.fail(t, r.stageName(session), "harness", "status %s: %v", session.Label, err)
	}
	r.validateReady(t, session)
	r.setLanes(t, session.Stage, "pass", "pass", "pass", "")
}

func (r *githubAppRun) deploy(t *testing.T, session *githubAppSession) {
	t.Helper()
	cp9 := filepath.Join(session.CP9Root, "bin", "cp9")
	if err := r.action(3*time.Hour, session.CP9Root, session.Label+"-deploy-"+session.Service, true,
		cp9, "deploy", "service="+session.Service, "config="+session.Config); err != nil {
		lane := "product"
		if errors.Is(err, errGitHubAppActionReceipt) {
			lane = "harness"
		}
		r.fail(t, r.stageName(session), lane, "deploy %s in %s: %v", session.Service, session.Label, err)
	}
	r.setLanes(t, r.stageName(session), "pass", "pass", "", "")
	r.validateHumanActions(t, session)
	r.validateCacheAndResidue(t, session, "after-deploy")
	r.setLanes(t, r.stageName(session), "pass", "pass", "pass", "")
}

func (r *githubAppRun) loadSession(t *testing.T, stage, label, service string) *githubAppSession {
	t.Helper()
	session := &githubAppSession{Stage: stage, Label: label, Service: service, Evidence: filepath.Join(r.root, label)}
	metadata := githubAppSessionMetadata{}
	r.readJSONPath(t, filepath.Join(session.Evidence, "default-804-session.json"), &metadata)
	if metadata.Head != r.journal.CP9Head || metadata.Account != "804837308162" || metadata.Region != "us-west-2" || metadata.Owner != r.journal.Owner {
		r.fail(t, stage, "harness", "%s resume metadata does not match the frozen CP9 head, account, region, or owner", label)
	}
	session.CP9Root = metadata.CP9Root
	session.Config = metadata.Config
	session.Name = metadata.Session
	session.SourceRoot = filepath.Join(metadata.WorkRoot, "tidb-management-service")
	cfg, err := config.LoadSession(session.Config)
	if err != nil {
		r.fail(t, stage, "harness", "load %s resume config: %v", label, err)
	}
	if cfg.Services.GitHubAppClientID != r.journal.ClientID {
		r.fail(t, stage, "product", "%s resume config uses a different GitHub App client ID", label)
	}
	return session
}

func (r *githubAppRun) destroy(t *testing.T, label string) {
	t.Helper()
	evidence := filepath.Join(r.root, label)
	metadataPath := filepath.Join(evidence, "default-804-session.json")
	if !pathExists(metadataPath) {
		if enabled(os.Getenv(githubAppDestroyEnv)) && !r.actionExists(label+"-up") {
			r.setLanes(t, label+"-destroy", "not_run", "pass", "not_run", "not_run")
			return
		}
		r.fail(t, label+"-destroy", "cleanup", "%s session metadata is unavailable", label)
	}
	r.setLanes(t, label+"-destroy", "not_run", "", "", "fail")
	metadata := githubAppSessionMetadata{}
	r.readJSONPath(t, metadataPath, &metadata)
	if !r.cleanupPreviouslyPassed(label) {
		cleanup := filepath.Join(r.repoRoot, "spec-tests", "scripts", "default-804-session.sh")
		if previous := r.latestCleanupActionLabel(label); previous != "" && !r.cleanupAttemptArchived(label, previous) {
			r.captureCleanupAttempt(t, label, previous)
			if !r.cleanupAttemptArchived(label, previous) {
				r.fail(t, label+"-destroy", "harness", "cleanup evidence from %s is incomplete", previous)
			}
		}
		actionLabel := r.cleanupActionLabel(label)
		err := r.action(3*time.Hour, r.repoRoot, actionLabel, false, cleanup, "cleanup", evidence)
		r.captureCleanupAttempt(t, label, actionLabel)
		if err != nil {
			r.fail(t, label+"-destroy", "cleanup", "cleanup %s: %v", label, err)
		}
	}
	destroyed, worktree, handoff, err := r.aggregateCleanupAttempts(label)
	if err != nil {
		r.fail(t, label+"-destroy", "harness", "read %s cleanup attempts: %v", label, err)
	}
	if !destroyed || !worktree || !handoff {
		r.fail(t, label+"-destroy", "cleanup", "%s cleanup attempts are incomplete: destroy=%t worktree=%t handoff=%t", label, destroyed, worktree, handoff)
	}
	r.setLanes(t, label+"-destroy", "not_run", "pass", "", "pass")
	if r.journal.ClientID == "" {
		r.setLanes(t, label+"-destroy", "not_run", "pass", "not_run", "pass")
		return
	}
	if !pathExists(githubAppCachePath(t, r.journal.ClientID)) {
		r.fail(t, label+"-destroy", "security", "shared GitHub authorization disappeared after %s destroy", label)
	}
	service := "tempo-proxy"
	if label == "session-b" {
		service = "tempo-wrapper"
	}
	r.validateCacheAndResidue(t, &githubAppSession{
		Stage: label + "-destroy", Label: label, Service: service, Evidence: evidence, CP9Root: metadata.CP9Root,
		Name: metadata.Session, SourceRoot: filepath.Join(metadata.WorkRoot, "tidb-management-service"),
	}, "after-destroy")
	r.setLanes(t, label+"-destroy", "not_run", "pass", "pass", "pass")
}

func (r *githubAppRun) cleanupPreviouslyPassed(label string) bool {
	for _, attempt := range r.journal.Attempts[label+"-destroy"] {
		if attempt.Cleanup == "pass" {
			return true
		}
	}
	return false
}

func (r *githubAppRun) latestCleanupActionLabel(label string) string {
	base, latest := label+"-cleanup", ""
	for attempt := 1; ; attempt++ {
		name := base
		if attempt > 1 {
			name = fmt.Sprintf("%s-%d", base, attempt)
		}
		action, ok := r.journal.Actions[name]
		if !ok {
			return latest
		}
		if action.Status != "completed" {
			return ""
		}
		latest = name
		if action.ExitCode == 0 {
			return latest
		}
	}
}

func (r *githubAppRun) cleanupAttemptArchived(label, action string) bool {
	directory := filepath.Join(r.root, label, "cleanup-attempts", action)
	data, err := os.ReadFile(filepath.Join(directory, "cleanup-receipt.json"))
	if err != nil {
		return false
	}
	var receipt struct {
		Destroy string `json:"destroy"`
	}
	if json.Unmarshal(data, &receipt) != nil {
		return false
	}
	if receipt.Destroy != "pass" {
		return true
	}
	log, err := os.ReadFile(filepath.Join(directory, "destroy.log"))
	return err == nil && bytes.Contains(log, []byte("cleanup_workflow_run_id="))
}

func (r *githubAppRun) cleanupActionLabel(label string) string {
	base := label + "-cleanup"
	for attempt := 1; ; attempt++ {
		name := base
		if attempt > 1 {
			name = fmt.Sprintf("%s-%d", base, attempt)
		}
		action, ok := r.journal.Actions[name]
		if !ok || action.Status != "completed" || action.ExitCode == 0 {
			return name
		}
	}
}

func (r *githubAppRun) captureCleanupAttempt(t *testing.T, label, actionLabel string) {
	t.Helper()
	for _, name := range []string{"cleanup-receipt.json", "destroy.log", "remove-worktree.log", "remote-tilt-up.log", "remote-tilt-up.stderr"} {
		data, err := os.ReadFile(filepath.Join(r.root, label, name))
		if os.IsNotExist(err) {
			continue
		}
		if err != nil {
			r.fail(t, label+"-destroy", "harness", "preserve %s from %s: %v", name, actionLabel, err)
		}
		r.writeText(t, filepath.Join(label, "cleanup-attempts", actionLabel, name), string(data))
	}
}

func (r *githubAppRun) aggregateCleanupAttempts(label string) (bool, bool, bool, error) {
	entries, err := os.ReadDir(filepath.Join(r.root, label, "cleanup-attempts"))
	if err != nil {
		return false, false, false, err
	}
	destroyed, worktree, handoff := false, false, !r.actionExists(label+"-up")
	for _, entry := range entries {
		if !entry.IsDir() {
			continue
		}
		directory := filepath.Join(r.root, label, "cleanup-attempts", entry.Name())
		data, err := os.ReadFile(filepath.Join(directory, "cleanup-receipt.json"))
		if os.IsNotExist(err) {
			continue
		}
		if err != nil {
			return false, false, false, err
		}
		var receipt struct {
			Destroy  string `json:"destroy"`
			Worktree string `json:"worktree"`
		}
		if err := json.Unmarshal(data, &receipt); err != nil {
			return false, false, false, err
		}
		if receipt.Destroy == "pass" || receipt.Destroy == "not_needed" && !r.actionExists(label+"-up") {
			destroyed = true
		}
		if receipt.Worktree == "pass" || receipt.Worktree == "not_needed" {
			worktree = true
		}
		if receipt.Destroy == "pass" {
			log, err := os.ReadFile(filepath.Join(directory, "destroy.log"))
			if err == nil && bytes.Contains(log, []byte("cleanup_workflow_run_id=")) {
				handoff = true
			}
		}
	}
	return destroyed, worktree, handoff, nil
}

func (r *githubAppRun) validateSecurityReceipt(t *testing.T) {
	t.Helper()
	path := strings.TrimSpace(os.Getenv("CP9_GITHUB_APP_SECURITY_RECEIPT"))
	if path == "" {
		r.setLanes(t, "security-receipt", "not_run", "not_run", "not_run", "")
		t.Skip("CP9_GITHUB_APP_SECURITY_RECEIPT is unset; independent remote residue Security is NOT RUN")
	}
	var receipt struct {
		Status                string `json:"status"`
		SuccessHelperRC       int    `json:"success_helper_rc"`
		FailureHelperRC       string `json:"failure_helper_rc"`
		TemporaryPaths        int    `json:"temporary_paths"`
		TokenCacheFiles       int    `json:"token_cache_files"`
		AskpassFiles          int    `json:"askpass_files"`
		CredentialEnvironment int    `json:"credential_env_names"`
		ArgvTokenMatches      int    `json:"argv_token_matches"`
		LogTokenMatches       int    `json:"log_token_matches"`
	}
	data, err := os.ReadFile(path)
	if err != nil {
		r.setLanes(t, "security-receipt", "not_run", "pass", "not_run", "")
		t.Skipf("independent remote credential-residue receipt is unavailable; Security is NOT RUN: %v", err)
	}
	fields := map[string]json.RawMessage{}
	if err := json.Unmarshal(data, &fields); err != nil {
		r.fail(t, "security-receipt", "harness", "parse independent Security receipt: %v", err)
	}
	if err := json.Unmarshal(data, &receipt); err != nil {
		r.fail(t, "security-receipt", "harness", "decode independent Security receipt: %v", err)
	}
	keys := make([]string, 0, len(fields))
	for key := range fields {
		keys = append(keys, key)
	}
	slices.Sort(keys)
	if !slices.Equal(keys, []string{
		"argv_token_matches", "askpass_files", "credential_env_names", "failure_helper_rc", "log_token_matches",
		"status", "success_helper_rc", "temporary_paths", "token_cache_files",
	}) {
		r.fail(t, "security-receipt", "harness", "independent Security receipt has unexpected fields")
	}
	if receipt.Status != "pass" || receipt.SuccessHelperRC != 0 || receipt.FailureHelperRC != "nonzero" ||
		receipt.TemporaryPaths != 0 || receipt.TokenCacheFiles != 0 || receipt.AskpassFiles != 0 ||
		receipt.CredentialEnvironment != 0 || receipt.ArgvTokenMatches != 0 || receipt.LogTokenMatches != 0 {
		r.fail(t, "security-receipt", "security", "independent remote credential-residue receipt is not a complete pass")
	}
	head, err := os.ReadFile(filepath.Join(filepath.Dir(path), "cp9-head.txt"))
	if err != nil || strings.TrimSpace(string(head)) != r.journal.CP9Head {
		r.fail(t, "security-receipt", "harness", "independent Security receipt is not bound to CP9 head %s", r.journal.CP9Head)
	}
	var verdict struct {
		Status   string `json:"status"`
		RemoteRC int    `json:"remote_rc"`
	}
	verdictData, err := os.ReadFile(filepath.Join(filepath.Dir(path), "security-verdict.json"))
	if err != nil || json.Unmarshal(verdictData, &verdict) != nil || verdict.Status != "pass" || verdict.RemoteRC != 0 {
		r.fail(t, "security-receipt", "harness", "independent Security verdict is missing or not pass")
	}
	if err := os.WriteFile(filepath.Join(r.root, "remote-security-receipt.json"), data, 0o600); err != nil {
		r.fail(t, "security-receipt", "harness", "retain independent Security receipt: %v", err)
	}
	r.setLanes(t, "security-receipt", "not_run", "pass", "pass", "")
}

func (r *githubAppRun) validateDryRun(t *testing.T, session *githubAppSession) {
	t.Helper()
	path := r.commandStdout(session.Label + "-dry-run")
	var value struct {
		NoMutation bool   `json:"no_mutation"`
		Session    string `json:"session"`
		Owner      string `json:"owner"`
		TTL        string `json:"ttl"`
		AWS        struct {
			Account string `json:"account"`
			Region  string `json:"region"`
		} `json:"aws"`
	}
	r.readJSONPath(t, path, &value)
	if !value.NoMutation || value.Session != session.Name || value.Owner == "" || value.TTL == "" ||
		value.AWS.Account != "804837308162" || value.AWS.Region != "us-west-2" {
		r.fail(t, r.stageName(session), "harness", "%s dry-run does not match the declared no-mutation scope", session.Label)
	}
	r.writeJSON(t, filepath.Join(session.Label, "run-scope.json"), map[string]string{
		"session": value.Session, "owner": value.Owner, "account": value.AWS.Account,
		"region": value.AWS.Region, "ttl": value.TTL,
	})
}

func (r *githubAppRun) validateReady(t *testing.T, session *githubAppSession) {
	t.Helper()
	var value struct {
		Status   string `json:"status"`
		StateDir string `json:"state_dir"`
	}
	r.readJSONPath(t, r.commandStdout(session.Label+"-status"), &value)
	if value.Status != "ready" || value.StateDir == "" {
		r.fail(t, r.stageName(session), "product", "%s status=%q state_dir=%q, want ready with state", session.Label, value.Status, value.StateDir)
	}
	if pathExists(filepath.Join(value.StateDir, "github-app-user-token.json")) {
		r.fail(t, r.stageName(session), "security", "%s state contains github-app-user-token.json", session.Label)
	}
}

func (r *githubAppRun) validateHumanActions(t *testing.T, session *githubAppSession) {
	t.Helper()
	type locatedEvent struct {
		Path  string
		Event githubAppHumanAction
	}
	events := []locatedEvent{}
	for _, label := range []string{
		"session-a-up", "session-a-deploy-tempo-proxy",
		"session-b-up", "session-b-deploy-tempo-wrapper",
	} {
		path := r.commandStderr(label)
		data, err := os.ReadFile(path)
		if os.IsNotExist(err) {
			continue
		}
		if err != nil {
			r.fail(t, r.stageName(session), "harness", "read %s stderr: %v", label, err)
		}
		for _, line := range strings.Split(string(data), "\n") {
			const prefix = "CP9_HUMAN_ACTION "
			if !strings.HasPrefix(line, prefix) {
				continue
			}
			raw := []byte(strings.TrimPrefix(line, prefix))
			fields := map[string]json.RawMessage{}
			event := githubAppHumanAction{}
			if err := json.Unmarshal(raw, &fields); err != nil || json.Unmarshal(raw, &event) != nil {
				r.fail(t, r.stageName(session), "harness", "parse human-action event in %s", label)
			}
			keys := make([]string, 0, len(fields))
			for key := range fields {
				keys = append(keys, key)
			}
			slices.Sort(keys)
			if !slices.Equal(keys, []string{"continuation", "expires_in_seconds", "type", "user_code", "verification_uri", "version"}) ||
				event.Version != 1 || event.Type != "github_device_flow" ||
				event.VerificationURI != "https://github.com/login/device" || !githubAppUserCode.MatchString(event.UserCode) ||
				event.ExpiresInSeconds <= 0 || event.Continuation != "automatic" {
				r.fail(t, r.stageName(session), "product", "invalid human-action event in %s", label)
			}
			if !strings.Contains(string(data), "========== CP9 HUMAN ACTION REQUIRED ==========") ||
				!strings.Contains(string(data), "CP9 is waiting and will continue automatically after authorization.") {
				r.fail(t, r.stageName(session), "product", "human-action event in %s lacks its banner or automatic-continuation notice", label)
			}
			events = append(events, locatedEvent{Path: label, Event: event})
		}
	}
	if len(events) > 1 || (len(events) == 1 && events[0].Path != "session-a-up") {
		r.fail(t, r.stageName(session), "product", "human-action events occurred %d times at invalid locations", len(events))
	}
	if session.Label == "session-a" {
		upData, err := os.ReadFile(r.commandStderr("session-a-up"))
		if err != nil {
			r.fail(t, r.stageName(session), "harness", "read session A up stderr: %v", err)
		}
		flowStarted := r.journal.CachePreexisted != nil && !*r.journal.CachePreexisted ||
			bytes.Contains(upData, []byte("starting a new Device Flow")) ||
			bytes.Contains(upData, []byte("========== CP9 HUMAN ACTION REQUIRED =========="))
		want := 0
		if flowStarted {
			want = 1
		}
		if len(events) != want {
			r.fail(t, r.stageName(session), "product", "session A human-action events=%d, want %d", len(events), want)
		}
		r.journal.HumanActionCount = len(events)
	} else if len(events) != r.journal.HumanActionCount {
		r.fail(t, r.stageName(session), "product", "session B added a human-action event")
	}
	r.persistJournal(t)
}

func (r *githubAppRun) validateCacheAndResidue(t *testing.T, session *githubAppSession, checkpoint string) {
	t.Helper()
	path := githubAppCachePath(t, r.journal.ClientID)
	info, err := os.Lstat(path)
	if err != nil {
		r.fail(t, r.stageName(session), "security", "inspect shared GitHub authorization: %v", err)
	}
	parentInfo, err := os.Lstat(filepath.Dir(path))
	if err != nil {
		r.fail(t, r.stageName(session), "security", "inspect shared GitHub authorization directory: %v", err)
	}
	fileStat, fileOK := info.Sys().(*syscall.Stat_t)
	parentStat, parentOK := parentInfo.Sys().(*syscall.Stat_t)
	if !info.Mode().IsRegular() || info.Mode().Perm() != 0o600 || !fileOK || int(fileStat.Uid) != os.Geteuid() ||
		!parentInfo.IsDir() || parentInfo.Mode().Perm() != 0o700 || !parentOK || int(parentStat.Uid) != os.Geteuid() {
		r.fail(t, r.stageName(session), "security", "shared GitHub authorization is not an owner-only regular file and directory")
	}
	data, err := os.ReadFile(path)
	if err != nil {
		r.fail(t, r.stageName(session), "security", "read shared GitHub authorization: %v", err)
	}
	fields := map[string]json.RawMessage{}
	cache := githubAppTokenCache{}
	if err := json.Unmarshal(data, &fields); err != nil || json.Unmarshal(data, &cache) != nil {
		r.fail(t, r.stageName(session), "security", "shared GitHub authorization is invalid JSON")
	}
	keys := make([]string, 0, len(fields))
	for key := range fields {
		keys = append(keys, key)
	}
	slices.Sort(keys)
	withoutRefresh := []string{"access_token", "client_id", "expires_at", "github_login", "github_user_id", "version"}
	withRefresh := []string{"access_token", "client_id", "expires_at", "github_login", "github_user_id", "refresh_expires_at", "refresh_token", "version"}
	if !slices.Equal(keys, withoutRefresh) && !slices.Equal(keys, withRefresh) {
		r.fail(t, r.stageName(session), "security", "shared GitHub authorization has unexpected fields")
	}
	now := time.Now().Unix()
	if cache.Version != 1 || cache.ClientID != r.journal.ClientID || !githubAppUserID.MatchString(cache.GitHubUserID) || cache.GitHubLogin == "" ||
		!strings.HasPrefix(cache.AccessToken, "ghu_") || (cache.ExpiresAt != 0 && (cache.ExpiresAt <= now+300 || cache.ExpiresAt > now+28800)) {
		r.fail(t, r.stageName(session), "security", "shared GitHub authorization metadata is invalid or too near expiry")
	}
	refreshAvailable := cache.RefreshToken != ""
	if refreshAvailable && (!strings.HasPrefix(cache.RefreshToken, "ghr_") || cache.RefreshExpiresAt <= now+300) {
		r.fail(t, r.stageName(session), "security", "shared GitHub refresh authorization is invalid or too near expiry")
	}
	var accessTTL any
	if cache.ExpiresAt != 0 {
		accessTTL = cache.ExpiresAt - now
	}
	r.writeJSON(t, filepath.Join(session.Label, "token-cache-"+checkpoint+".json"), map[string]any{
		"client_id": cache.ClientID, "github_user_id": cache.GitHubUserID, "github_login": cache.GitHubLogin,
		"access_ttl_seconds": accessTTL, "refresh_available": refreshAvailable, "mode": "0600",
	})
	secrets := [][]byte{[]byte(cache.AccessToken)}
	if cache.RefreshToken != "" {
		secrets = append(secrets, []byte(cache.RefreshToken))
	}
	for _, root := range []string{r.root, filepath.Join(session.CP9Root, ".cp9", session.Name), session.SourceRoot} {
		if found, err := findSecret(root, secrets); err != nil {
			r.fail(t, r.stageName(session), "harness", "scan %s for credential residue: %v", root, err)
		} else if found != "" {
			r.fail(t, r.stageName(session), "security", "GitHub credential residue found under a forbidden evidence, session, or repository path")
		}
	}
}

func (r *githubAppRun) selectSourceBuild(t *testing.T, session *githubAppSession, path string) {
	t.Helper()
	data, err := os.ReadFile(path)
	if err != nil {
		r.fail(t, r.stageName(session), "harness", "read Session Tiltfile: %v", err)
	}
	marker := []byte("source_build_services = [")
	if bytes.Count(data, marker) != 1 {
		t.Fatalf("Session Tiltfile must contain one source_build_services assignment")
	}
	start := bytes.Index(data, marker)
	end := bytes.Index(data[start:], []byte("\n]"))
	if end < 0 {
		t.Fatal("Session Tiltfile source_build_services assignment is unterminated")
	}
	end += start + len("\n]")
	replacement := []byte("source_build_services = [\n    \"" + session.Service + "\",\n]")
	data = append(append(append([]byte(nil), data[:start]...), replacement...), data[end:]...)
	if err := os.WriteFile(path, data, 0o600); err != nil {
		t.Fatalf("write Session Tiltfile: %v", err)
	}
	selected, err := tilt.SelectedSourceBuildServices(path)
	if err != nil || !slices.Equal(selected, []string{session.Service}) {
		t.Fatalf("Session Tiltfile source-build selection=%v, err=%v, want [%s]", selected, err, session.Service)
	}
}

func (r *githubAppRun) actionExists(label string) bool {
	return r.journal.Actions[label].Status != ""
}

func (r *githubAppRun) action(timeout time.Duration, dir, label string, streamStderr bool, argv ...string) error {
	saved := r.journal.Actions[label]
	if saved.Status != "" {
		if !r.resume || saved.Head != r.journal.CP9Head || saved.Directory != dir || !slices.Equal(saved.Command, argv) {
			return fmt.Errorf("%w: %s does not match this resume", errGitHubAppActionReceipt, label)
		}
		if saved.Status == "started" {
			return fmt.Errorf("%w: %s has unknown outcome; refusing to replay", errGitHubAppActionReceipt, label)
		}
		if saved.Status != "completed" {
			return fmt.Errorf("%w: %s has status %q", errGitHubAppActionReceipt, label, saved.Status)
		}
		output, ok := r.journal.Outputs[label]
		prefix, validPrefix := strings.CutSuffix(output.Stdout, ".stdout.log")
		if !ok || !validPrefix || !fs.ValidPath(output.Stdout) || output.Stderr != prefix+".stderr.log" ||
			!pathExists(filepath.Join(r.root, filepath.FromSlash(output.Stdout))) ||
			!pathExists(filepath.Join(r.root, filepath.FromSlash(output.Stderr))) ||
			!pathExists(filepath.Join(r.root, filepath.FromSlash(prefix+".command.json"))) {
			return fmt.Errorf("%w: %s output is unavailable", errGitHubAppActionReceipt, label)
		}
		if saved.ExitCode != 0 {
			return fmt.Errorf("saved action %s exited %d", label, saved.ExitCode)
		}
		return nil
	}
	r.journal.Actions[label] = githubAppActionReceipt{
		Status: "started", Head: r.journal.CP9Head, Directory: dir, Command: append([]string(nil), argv...), ExitCode: -1,
	}
	r.persistJournal(r.t)
	err := r.command(timeout, dir, label, streamStderr, argv...)
	saved = r.journal.Actions[label]
	saved.Status = "completed"
	saved.ExitCode = githubAppExitCode(err)
	r.journal.Actions[label] = saved
	r.persistJournal(r.t)
	return err
}

func (r *githubAppRun) command(timeout time.Duration, dir, label string, streamStderr bool, argv ...string) error {
	r.t.Helper()
	if len(argv) == 0 {
		return errors.New("empty command")
	}
	r.commandID++
	if err := os.MkdirAll(filepath.Join(r.root, "commands"), 0o700); err != nil {
		return err
	}
	prefix := fmt.Sprintf("%03d-%s", r.commandID, label)
	stdoutPath := filepath.Join(r.root, "commands", prefix+".stdout.log")
	stderrPath := filepath.Join(r.root, "commands", prefix+".stderr.log")
	r.journal.Outputs[label] = githubAppCommandOutput{
		Stdout: filepath.ToSlash(strings.TrimPrefix(stdoutPath, r.root+string(filepath.Separator))),
		Stderr: filepath.ToSlash(strings.TrimPrefix(stderrPath, r.root+string(filepath.Separator))),
	}
	r.persistJournal(r.t)
	stdout, err := os.OpenFile(stdoutPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
	if err != nil {
		return err
	}
	defer stdout.Close()
	stderr, err := os.OpenFile(stderrPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
	if err != nil {
		return err
	}
	defer stderr.Close()
	ctx, cancel := context.WithTimeout(context.Background(), timeout)
	defer cancel()
	cmd := exec.Command(argv[0], argv[1:]...)
	cmd.Dir = dir
	cmd.Env = append(os.Environ(), "GOTOOLCHAIN=go1.26.1")
	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
	cmd.Stdout = stdout
	cmd.Stderr = stderr
	if streamStderr {
		cmd.Stderr = io.MultiWriter(os.Stderr, stderr)
	}
	started := time.Now()
	err = cmd.Start()
	if err == nil {
		done := make(chan error, 1)
		go func() { done <- cmd.Wait() }()
		select {
		case err = <-done:
		case <-ctx.Done():
			killErr := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
			waitErr := <-done
			if killErr != nil && !errors.Is(killErr, syscall.ESRCH) {
				err = fmt.Errorf("command timed out and process-group kill failed: %w", killErr)
			} else if waitErr != nil {
				err = fmt.Errorf("command timed out: %w (%v)", ctx.Err(), waitErr)
			} else {
				err = fmt.Errorf("command timed out: %w", ctx.Err())
			}
		}
	}
	finished := time.Now()
	receipt := githubAppCommandReceipt{
		Command: argv, StartedAt: started.UTC().Format(time.RFC3339Nano), FinishedAt: finished.UTC().Format(time.RFC3339Nano),
		DurationMS: finished.Sub(started).Milliseconds(), ExitCode: githubAppExitCode(err),
		Stdout: filepath.ToSlash(strings.TrimPrefix(stdoutPath, r.root+string(filepath.Separator))),
		Stderr: filepath.ToSlash(strings.TrimPrefix(stderrPath, r.root+string(filepath.Separator))),
	}
	if err != nil {
		receipt.Error = err.Error()
	}
	r.writeJSON(r.t, filepath.Join("commands", prefix+".command.json"), receipt)
	return err
}

func githubAppExitCode(err error) int {
	if err == nil {
		return 0
	}
	var exitErr *exec.ExitError
	if errors.As(err, &exitErr) {
		return exitErr.ExitCode()
	}
	return -1
}

func (r *githubAppRun) fail(t *testing.T, stage, lane, format string, args ...any) {
	t.Helper()
	receipt := r.journal.Stages[stage]
	receipt.Status = "fail"
	receipt.Message = fmt.Sprintf(format, args...)
	switch lane {
	case "product":
		receipt.Product = "fail"
	case "harness":
		receipt.Harness = "fail"
	case "security":
		receipt.Security = "fail"
	case "cleanup":
		receipt.Cleanup = "fail"
	}
	r.journal.Stages[stage] = receipt
	r.persistJournal(t)
	t.Fatal(receipt.Message)
}

func (r *githubAppRun) setLanes(t *testing.T, stage, product, harness, security, cleanup string) {
	t.Helper()
	receipt := r.journal.Stages[stage]
	receipt.Product = product
	receipt.Harness = harness
	receipt.Security = security
	receipt.Cleanup = cleanup
	r.journal.Stages[stage] = receipt
	r.persistJournal(t)
}

func (r *githubAppRun) requirePassed(t *testing.T, stage string) {
	t.Helper()
	if r.journal.Stages[stage].Status != "pass" {
		t.Skipf("requires %s=pass in this invocation or matching resume journal", stage)
	}
}

func (r *githubAppRun) persistJournal(t *testing.T) {
	t.Helper()
	r.writeJSON(t, "journal.json", r.journal)
}

func (r *githubAppRun) markCleanupPending(t *testing.T, label string) {
	t.Helper()
	name := label + "-destroy"
	receipt := r.journal.Stages[name]
	if receipt.Cleanup == "pass" {
		return
	}
	receipt.Cleanup = "fail"
	if receipt.Message == "" {
		receipt.Message = label + " has run-owned state pending exact cleanup"
	}
	r.journal.Stages[name] = receipt
	r.persistJournal(t)
}

func (r *githubAppRun) writeSummary(t *testing.T) {
	t.Helper()
	product := aggregateGitHubAppLane(r.journal, "product", "session-a-up-and-source-build", "session-a-deploy", "session-b-reuse-and-source-build", "session-b-deploy")
	harness := aggregateGitHubAppLane(r.journal, "harness", "authorization-prerequisites", "session-a-up-and-source-build", "session-a-deploy", "session-a-destroy", "session-b-reuse-and-source-build", "session-b-deploy", "session-b-destroy")
	security := aggregateGitHubAppLane(r.journal, "security", "session-a-up-and-source-build", "session-a-deploy", "session-a-destroy", "session-b-reuse-and-source-build", "session-b-deploy", "session-b-destroy", "security-receipt")
	cleanup := aggregateGitHubAppLane(r.journal, "cleanup", "session-a-destroy", "session-b-destroy")
	summary := fmt.Sprintf("# GitHub App private source-build result\n\n- CP9: `%s`\n- Product: `%s`\n- Harness: `%s`\n- Security: `%s`\n- Cleanup: `%s`\n", r.journal.CP9Head, product, harness, security, cleanup)
	if err := os.WriteFile(filepath.Join(r.root, "summary.md"), []byte(summary), 0o600); err != nil {
		t.Errorf("write GitHub App summary: %v", err)
	}
}

func aggregateGitHubAppLane(journal githubAppJournal, lane string, names ...string) string {
	allPass := true
	observed := false
	for _, name := range names {
		if lane != "cleanup" {
			for _, attempt := range journal.Attempts[name] {
				if githubAppLaneValue(attempt, lane) == "fail" {
					return "fail"
				}
			}
		}
		value := githubAppLaneValue(journal.Stages[name], lane)
		if lane == "cleanup" && (value == "" || value == "not_run") {
			continue
		}
		observed = true
		if value == "fail" {
			return "fail"
		}
		if value != "pass" {
			allPass = false
		}
	}
	if allPass && observed {
		return "pass"
	}
	return "not_run"
}

func githubAppLaneValue(receipt githubAppStageReceipt, lane string) string {
	switch lane {
	case "product":
		return receipt.Product
	case "harness":
		return receipt.Harness
	case "security":
		return receipt.Security
	case "cleanup":
		return receipt.Cleanup
	default:
		return ""
	}
}

func (r *githubAppRun) readJSON(t *testing.T, relative string, value any) {
	t.Helper()
	r.readJSONPath(t, filepath.Join(r.root, relative), value)
}

func (r *githubAppRun) readJSONPath(t *testing.T, path string, value any) {
	t.Helper()
	data, err := os.ReadFile(path)
	if err != nil {
		t.Fatalf("read %s: %v", path, err)
	}
	if err := json.Unmarshal(data, value); err != nil {
		t.Fatalf("parse %s: %v", path, err)
	}
}

func (r *githubAppRun) writeJSON(t *testing.T, relative string, value any) {
	t.Helper()
	path := filepath.Join(r.root, relative)
	if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
		t.Fatalf("create evidence directory for %s: %v", relative, err)
	}
	data, err := json.MarshalIndent(value, "", "  ")
	if err != nil {
		t.Fatalf("encode %s: %v", relative, err)
	}
	temporary := path + ".tmp"
	if err := os.WriteFile(temporary, append(data, '\n'), 0o600); err != nil {
		t.Fatalf("write %s: %v", relative, err)
	}
	if err := os.Rename(temporary, path); err != nil {
		t.Fatalf("publish %s: %v", relative, err)
	}
}

func (r *githubAppRun) writeText(t *testing.T, relative, value string) {
	t.Helper()
	path := filepath.Join(r.root, relative)
	if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
		t.Fatalf("create evidence directory for %s: %v", relative, err)
	}
	if err := os.WriteFile(path, []byte(value), 0o600); err != nil {
		t.Fatalf("write %s: %v", relative, err)
	}
}

func (r *githubAppRun) commandStdout(label string) string {
	if output, ok := r.journal.Outputs[label]; ok && fs.ValidPath(output.Stdout) && strings.HasSuffix(output.Stdout, ".stdout.log") {
		return filepath.Join(r.root, filepath.FromSlash(output.Stdout))
	}
	return filepath.Join(r.root, "commands", label+".stdout.log")
}

func (r *githubAppRun) commandStderr(label string) string {
	if output, ok := r.journal.Outputs[label]; ok && fs.ValidPath(output.Stderr) && strings.HasSuffix(output.Stderr, ".stderr.log") {
		return filepath.Join(r.root, filepath.FromSlash(output.Stderr))
	}
	return filepath.Join(r.root, "commands", label+".stderr.log")
}

func (r *githubAppRun) stageName(session *githubAppSession) string {
	return session.Stage
}

func githubAppCachePath(t *testing.T, clientID string) string {
	t.Helper()
	home, err := os.UserHomeDir()
	if err != nil {
		t.Fatalf("resolve home directory: %v", err)
	}
	digest := sha256.Sum256([]byte(clientID))
	return filepath.Join(home, ".cp9", "data", "shared", "github-app", hex.EncodeToString(digest[:])+".json")
}

func pathExists(path string) bool {
	_, err := os.Stat(path)
	return err == nil
}

func githubAppPriorCommandID(root string) int {
	entries, err := os.ReadDir(filepath.Join(root, "commands"))
	if err != nil {
		return 0
	}
	maxID := 0
	for _, entry := range entries {
		prefix, _, ok := strings.Cut(entry.Name(), "-")
		id, err := strconv.Atoi(prefix)
		if ok && err == nil && id > maxID {
			maxID = id
		}
	}
	return maxID
}

func findSecret(root string, secrets [][]byte) (string, error) {
	if _, err := os.Stat(root); os.IsNotExist(err) {
		return "", nil
	} else if err != nil {
		return "", err
	}
	var found string
	err := filepath.WalkDir(root, func(path string, entry os.DirEntry, walkErr error) error {
		if walkErr != nil {
			return walkErr
		}
		if entry.IsDir() && entry.Name() == ".git" {
			return filepath.SkipDir
		}
		if !entry.Type().IsRegular() {
			return nil
		}
		file, err := os.Open(path)
		if err != nil {
			return err
		}
		defer file.Close()
		data, err := io.ReadAll(file)
		if err != nil {
			return err
		}
		for _, secret := range secrets {
			if len(secret) > 0 && bytes.Contains(data, secret) {
				found = path
				return errors.New("credential residue")
			}
		}
		return nil
	})
	if found != "" {
		return found, nil
	}
	return "", err
}