package automation

import (
	"context"
	"errors"
	"fmt"
	"sync"
	"testing"
	"time"

	"github.com/stretchr/testify/require"
)

func TestAutomationQueue(t *testing.T) {
	ctx := context.Background()
	store := newAutomationStoreForTest(t)
	now := time.Date(2026, 6, 2, 15, 0, 0, 0, time.UTC)
	_, err := store.CreateRun(ctx, AutomationRun{ID: "run-1", WorkspaceID: "ws-1", Name: "Queue", Status: AutomationRunStatusRunning, CreatedAt: now, UpdatedAt: now})
	require.NoError(t, err)
	require.NoError(t, store.CreateJobRuns(ctx, []AutomationJobRun{{ID: "job-plan", WorkspaceID: "ws-1", RunID: "run-1", JobID: "plan", JobType: "github_actions_job", Status: AutomationJobRunStatusQueued, ScheduledAt: now, CreatedAt: now, UpdatedAt: now}}))

	job, err := store.EnqueueJob(ctx, AutomationJob{WorkspaceID: "ws-1", RunID: "run-1", JobRunID: "job-plan", QueueName: "default", Status: AutomationJobStatusReady, Priority: 10, ScheduledAt: now, Attempt: 0, MaxAttempts: 3, IdempotencyKey: "run-1:job-plan", ShardBucket: 0, CreatedAt: now, UpdatedAt: now})
	require.NoError(t, err)
	require.NotZero(t, job.ID)

	again, err := store.EnqueueJob(ctx, AutomationJob{WorkspaceID: "ws-1", RunID: "run-1", JobRunID: "job-plan", QueueName: "default", Status: AutomationJobStatusReady, Priority: 10, ScheduledAt: now, Attempt: 0, MaxAttempts: 3, IdempotencyKey: "run-1:job-plan", ShardBucket: 0, CreatedAt: now, UpdatedAt: now})
	require.NoError(t, err)
	require.Equal(t, job.ID, again.ID)

	claimed, err := store.ClaimJobs(ctx, ClaimJobsRequest{WorkspaceID: "ws-1", QueueName: "default", ShardBucket: 0, LeaseOwner: "worker-1", LeaseDuration: time.Minute, Limit: 2, Now: now.Add(time.Second)})
	require.NoError(t, err)
	require.Len(t, claimed, 1)
	require.Equal(t, AutomationJobStatusLeased, claimed[0].Status)
	require.Equal(t, 1, claimed[0].Attempt)
	require.NotNil(t, claimed[0].LeaseUntil)

	jobRun, err := store.GetJobRun(ctx, "ws-1", "job-plan")
	require.NoError(t, err)
	completedAt := now.Add(2 * time.Second)
	jobRun.Status = AutomationJobRunStatusSucceeded
	jobRun.Attempt = claimed[0].Attempt
	jobRun.CompletedAt = &completedAt
	jobRun.UpdatedAt = completedAt
	require.NoError(t, store.CompleteJobAssignmentSuccess(ctx, claimed[0].ID, "worker-1", jobRun, automationEvent("job_succeeded", jobRun, "Automation job succeeded", "{}", completedAt), completedAt))

	recovered, err := store.RecoverExpiredJobs(ctx, now.Add(2*time.Minute), 10)
	require.NoError(t, err)
	require.Zero(t, recovered)
}

func TestMarkJobAssignmentRunningAcceptsFreshlyClaimedJob(t *testing.T) {
	ctx := context.Background()
	store := newAutomationStoreForTest(t)
	now := time.Date(2026, 6, 2, 15, 30, 0, 0, time.UTC)
	_, err := store.CreateRun(ctx, AutomationRun{ID: "run-fresh", WorkspaceID: "ws-1", Name: "Queue", Status: AutomationRunStatusRunning, CreatedAt: now, UpdatedAt: now})
	require.NoError(t, err)
	require.NoError(t, store.CreateJobRuns(ctx, []AutomationJobRun{{ID: "job-fresh", WorkspaceID: "ws-1", RunID: "run-fresh", JobID: "build", JobType: "github_actions_job", Status: AutomationJobRunStatusQueued, ScheduledAt: now, CreatedAt: now, UpdatedAt: now}}))
	_, err = store.EnqueueJob(ctx, AutomationJob{WorkspaceID: "ws-1", RunID: "run-fresh", JobRunID: "job-fresh", QueueName: "default", Status: AutomationJobStatusReady, ScheduledAt: now, MaxAttempts: 3, IdempotencyKey: "run-fresh:build", ShardBucket: 0, CreatedAt: now, UpdatedAt: now})
	require.NoError(t, err)
	claimed, err := store.ClaimJobs(ctx, ClaimJobsRequest{WorkspaceID: "ws-1", QueueName: "default", ShardBucket: 0, LeaseOwner: "worker-1", LeaseDuration: time.Minute, Limit: 1, Now: now})
	require.NoError(t, err)
	require.Len(t, claimed, 1)

	jobRun, err := store.GetJobRun(ctx, "ws-1", "job-fresh")
	require.NoError(t, err)
	jobRun.Status = AutomationJobRunStatusRunning
	jobRun.Attempt = claimed[0].Attempt
	jobRun.LeaseOwner = "worker-1"
	jobRun.LeaseUntil = claimed[0].LeaseUntil
	jobRun.StartedAt = &now
	jobRun.UpdatedAt = now
	require.NoError(t, store.MarkJobAssignmentRunning(ctx, claimed[0].ID, "worker-1", jobRun, now))

	stored, err := store.GetJobRun(ctx, "ws-1", "job-fresh")
	require.NoError(t, err)
	require.Equal(t, AutomationJobRunStatusRunning, stored.Status)
	require.Equal(t, "worker-1", stored.LeaseOwner)
}

func TestRenewLeasedJobAcceptsFreshlyClaimedJob(t *testing.T) {
	ctx := context.Background()
	store := newAutomationStoreForTest(t)
	now := time.Date(2026, 6, 2, 15, 35, 0, 0, time.UTC)
	_, err := store.CreateRun(ctx, AutomationRun{ID: "run-renew-fresh", WorkspaceID: "ws-1", Name: "Queue", Status: AutomationRunStatusRunning, CreatedAt: now, UpdatedAt: now})
	require.NoError(t, err)
	require.NoError(t, store.CreateJobRuns(ctx, []AutomationJobRun{{ID: "job-renew-fresh", WorkspaceID: "ws-1", RunID: "run-renew-fresh", JobID: "build", JobType: "github_actions_job", Status: AutomationJobRunStatusQueued, ScheduledAt: now, CreatedAt: now, UpdatedAt: now}}))
	_, err = store.EnqueueJob(ctx, AutomationJob{WorkspaceID: "ws-1", RunID: "run-renew-fresh", JobRunID: "job-renew-fresh", QueueName: "default", Status: AutomationJobStatusReady, ScheduledAt: now, MaxAttempts: 3, IdempotencyKey: "run-renew-fresh:build", ShardBucket: 0, CreatedAt: now, UpdatedAt: now})
	require.NoError(t, err)

	leaseUntil := now.Add(5 * time.Minute)
	claimed, err := store.ClaimJobs(ctx, ClaimJobsRequest{WorkspaceID: "ws-1", QueueName: "default", ShardBucket: 0, LeaseOwner: "worker-1", LeaseDuration: 5 * time.Minute, Limit: 1, Now: now})
	require.NoError(t, err)
	require.Len(t, claimed, 1)
	require.Equal(t, leaseUntil, *claimed[0].LeaseUntil)

	renewed, err := store.RenewLeasedJob(ctx, "ws-1", claimed[0].ID, "worker-1", now, leaseUntil)
	require.NoError(t, err)
	require.NotNil(t, renewed.LeaseUntil)
	require.Equal(t, leaseUntil, *renewed.LeaseUntil)
}

func TestListReadyWorkspacesIsBoundedAndOrdersByOldestReadyJob(t *testing.T) {
	ctx := context.Background()
	store := newAutomationStoreForTest(t)
	now := time.Date(2026, 6, 12, 9, 0, 0, 0, time.UTC)
	jobs := []AutomationJob{
		{WorkspaceID: "ws-later", RunID: "run-later", JobRunID: "job-later", QueueName: "default", Status: AutomationJobStatusReady, ScheduledAt: now.Add(2 * time.Minute), IdempotencyKey: "run-later:job-later", ShardBucket: 0, CreatedAt: now, UpdatedAt: now},
		{WorkspaceID: "ws-first", RunID: "run-first", JobRunID: "job-first", QueueName: "default", Status: AutomationJobStatusReady, ScheduledAt: now.Add(-time.Minute), IdempotencyKey: "run-first:job-first", ShardBucket: 0, CreatedAt: now, UpdatedAt: now},
		{WorkspaceID: "ws-first", RunID: "run-first-2", JobRunID: "job-first-2", QueueName: "default", Status: AutomationJobStatusReady, ScheduledAt: now.Add(-30 * time.Second), IdempotencyKey: "run-first:job-first-2", ShardBucket: 0, CreatedAt: now, UpdatedAt: now},
		{WorkspaceID: "ws-other-queue", RunID: "run-other-queue", JobRunID: "job-other-queue", QueueName: "slow", Status: AutomationJobStatusReady, ScheduledAt: now.Add(-2 * time.Minute), IdempotencyKey: "run-other-queue:job", ShardBucket: 0, CreatedAt: now, UpdatedAt: now},
		{WorkspaceID: "ws-leased", RunID: "run-leased", JobRunID: "job-leased", QueueName: "default", Status: AutomationJobStatusLeased, ScheduledAt: now.Add(-3 * time.Minute), IdempotencyKey: "run-leased:job", ShardBucket: 0, CreatedAt: now, UpdatedAt: now},
		{WorkspaceID: "ws-future", RunID: "run-future", JobRunID: "job-future", QueueName: "default", Status: AutomationJobStatusReady, ScheduledAt: now.Add(time.Minute), IdempotencyKey: "run-future:job", ShardBucket: 0, CreatedAt: now, UpdatedAt: now},
	}
	require.NoError(t, store.db.Create(&jobs).Error)

	workspaces, err := store.ListReadyWorkspaces(ctx, "default", now, 1)
	require.NoError(t, err)
	require.Equal(t, []string{"ws-first"}, workspaces)

	workspaces, err = store.ListReadyWorkspaces(ctx, "default", now.Add(3*time.Minute), 10)
	require.NoError(t, err)
	require.Equal(t, []string{"ws-first", "ws-future", "ws-later"}, workspaces)
}

func TestListReadyWorkspacesIsWorkspaceFair(t *testing.T) {
	ctx := context.Background()
	store := newAutomationStoreForTest(t)
	now := time.Date(2026, 6, 12, 9, 30, 0, 0, time.UTC)
	var jobs []AutomationJob
	for idx := 0; idx < 20; idx++ {
		jobs = append(jobs, AutomationJob{
			WorkspaceID:    "ws-noisy",
			RunID:          "run-noisy",
			JobRunID:       "job-noisy",
			QueueName:      "default",
			Status:         AutomationJobStatusReady,
			ScheduledAt:    now.Add(-time.Duration(20-idx) * time.Minute),
			IdempotencyKey: "run-noisy:job-" + string(rune('a'+idx)),
			ShardBucket:    0,
			CreatedAt:      now,
			UpdatedAt:      now,
		})
	}
	jobs = append(jobs,
		AutomationJob{WorkspaceID: "ws-second", RunID: "run-second", JobRunID: "job-second", QueueName: "default", Status: AutomationJobStatusReady, ScheduledAt: now.Add(-10 * time.Minute), IdempotencyKey: "run-second:job", ShardBucket: 0, CreatedAt: now, UpdatedAt: now},
		AutomationJob{WorkspaceID: "ws-third", RunID: "run-third", JobRunID: "job-third", QueueName: "default", Status: AutomationJobStatusReady, ScheduledAt: now.Add(-5 * time.Minute), IdempotencyKey: "run-third:job", ShardBucket: 0, CreatedAt: now, UpdatedAt: now},
	)
	require.NoError(t, store.db.Create(&jobs).Error)

	workspaces, err := store.ListReadyWorkspaces(ctx, "default", now, 3)
	require.NoError(t, err)
	require.Equal(t, []string{"ws-noisy", "ws-second", "ws-third"}, workspaces)
}

func TestListReadyWorkspacesAppliesWorkspaceScopeBeforeLimit(t *testing.T) {
	ctx := context.Background()
	store := newAutomationStoreForTest(t)
	now := time.Date(2026, 6, 12, 9, 45, 0, 0, time.UTC)
	require.NoError(t, store.db.Create(&[]AutomationJob{
		{WorkspaceID: "ws-foreign", RunID: "run-foreign", JobRunID: "job-foreign", QueueName: "default", Status: AutomationJobStatusReady, ScheduledAt: now.Add(-time.Minute), IdempotencyKey: "run-foreign:job", ShardBucket: 0, CreatedAt: now, UpdatedAt: now},
		{WorkspaceID: "ws-local", RunID: "run-local", JobRunID: "job-local", QueueName: "default", Status: AutomationJobStatusReady, ScheduledAt: now, IdempotencyKey: "run-local:job", ShardBucket: 0, CreatedAt: now, UpdatedAt: now},
	}).Error)

	workspaces, err := store.ListReadyWorkspaces(ctx, "default", now, 1, []string{"ws-local"})
	require.NoError(t, err)
	require.Equal(t, []string{"ws-local"}, workspaces)

	workspaces, err = store.ListReadyWorkspaces(ctx, "default", now, 1, []string{})
	require.NoError(t, err)
	require.Empty(t, workspaces)
}

func TestAutomationWaitAndRunnerMetricsAreWorkspaceScoped(t *testing.T) {
	ctx := context.Background()
	store := newAutomationStoreForTest(t)
	now := time.Date(2026, 6, 7, 10, 0, 0, 0, time.UTC)
	revokedAt := now.Add(-time.Minute)
	require.NoError(t, store.db.Create([]AutomationWait{
		{ID: "wait-human", WorkspaceID: "ws-1", RunID: "run-1", JobRunID: "job-1", WaitKind: AutomationWaitKindAgentTask, Status: AutomationWaitStatusActive, CreatedAt: now, UpdatedAt: now},
		{ID: "wait-agent", WorkspaceID: "ws-1", RunID: "run-1", JobRunID: "job-2", WaitKind: AutomationWaitKindAgentTask, Status: AutomationWaitStatusActive, CreatedAt: now, UpdatedAt: now},
		{ID: "wait-event", WorkspaceID: "ws-1", RunID: "run-1", JobRunID: "job-3", WaitKind: AutomationWaitKindWaitEvent, Status: AutomationWaitStatusActive, CreatedAt: now, UpdatedAt: now},
		{ID: "wait-resolved", WorkspaceID: "ws-1", RunID: "run-1", JobRunID: "job-4", WaitKind: AutomationWaitKindAgentTask, Status: AutomationWaitStatusResolved, CreatedAt: now, UpdatedAt: now},
		{ID: "wait-canceled", WorkspaceID: "ws-1", RunID: "run-1", JobRunID: "job-5", WaitKind: AutomationWaitKindAgentTask, Status: AutomationWaitStatusCanceled, CreatedAt: now, UpdatedAt: now},
		{ID: "wait-other-workspace", WorkspaceID: "ws-2", RunID: "run-2", JobRunID: "job-6", WaitKind: AutomationWaitKindAgentTask, Status: AutomationWaitStatusActive, CreatedAt: now, UpdatedAt: now},
	}).Error)
	require.NoError(t, store.db.Create([]AutomationRunner{
		{WorkspaceID: "ws-1", RunnerID: "runner-active", Name: "active", LabelsJSON: `[]`, Status: AutomationRunnerStatusActive, RegisteredAt: now, CreatedAt: now, UpdatedAt: now},
		{WorkspaceID: "ws-1", RunnerID: "runner-revoked-status", Name: "revoked status", LabelsJSON: `[]`, Status: AutomationRunnerStatusRevoked, RegisteredAt: now, CreatedAt: now, UpdatedAt: now},
		{WorkspaceID: "ws-1", RunnerID: "runner-revoked-at", Name: "revoked at", LabelsJSON: `[]`, Status: AutomationRunnerStatusActive, RevokedAt: &revokedAt, RegisteredAt: now, CreatedAt: now, UpdatedAt: now},
		{WorkspaceID: "ws-2", RunnerID: "runner-other-workspace", Name: "other", LabelsJSON: `[]`, Status: AutomationRunnerStatusActive, RegisteredAt: now, CreatedAt: now, UpdatedAt: now},
	}).Error)
	require.NoError(t, store.db.Create([]AutomationRunnerSession{
		{WorkspaceID: "ws-1", RunnerID: "runner-active", SessionID: "session-active", LeaseOwner: "lease-active", LabelsJSON: `[]`, Status: AutomationRunnerSessionStatusActive, HeartbeatAt: now, ExpiresAt: now.Add(time.Minute), CreatedAt: now, UpdatedAt: now},
		{WorkspaceID: "ws-1", RunnerID: "runner-active", SessionID: "session-expired", LeaseOwner: "lease-expired", LabelsJSON: `[]`, Status: AutomationRunnerSessionStatusActive, HeartbeatAt: now.Add(-time.Hour), ExpiresAt: now.Add(-time.Minute), CreatedAt: now, UpdatedAt: now},
		{WorkspaceID: "ws-1", RunnerID: "runner-revoked-status", SessionID: "session-revoked-status", LeaseOwner: "lease-revoked-status", LabelsJSON: `[]`, Status: AutomationRunnerSessionStatusRevoked, HeartbeatAt: now, ExpiresAt: now.Add(time.Minute), CreatedAt: now, UpdatedAt: now},
		{WorkspaceID: "ws-1", RunnerID: "runner-revoked-at", SessionID: "session-revoked-at", LeaseOwner: "lease-revoked-at", LabelsJSON: `[]`, Status: AutomationRunnerSessionStatusActive, RevokedAt: &revokedAt, HeartbeatAt: now, ExpiresAt: now.Add(time.Minute), CreatedAt: now, UpdatedAt: now},
		{WorkspaceID: "ws-2", RunnerID: "runner-other-workspace", SessionID: "session-other-workspace", LeaseOwner: "lease-other", LabelsJSON: `[]`, Status: AutomationRunnerSessionStatusActive, HeartbeatAt: now, ExpiresAt: now.Add(time.Minute), CreatedAt: now, UpdatedAt: now},
	}).Error)

	waitMetrics, err := store.WaitMetrics(ctx, "ws-1")
	require.NoError(t, err)
	require.EqualValues(t, 3, waitMetrics.Active)
	require.EqualValues(t, 1, waitMetrics.Resolved)
	require.EqualValues(t, 1, waitMetrics.Canceled)
	require.EqualValues(t, 2, waitMetrics.ActiveAgentTasks)
	require.EqualValues(t, 1, waitMetrics.ActiveEventWaits)

	runnerMetrics, err := store.RunnerMetrics(ctx, "ws-1", now)
	require.NoError(t, err)
	require.EqualValues(t, 3, runnerMetrics.Registered)
	require.EqualValues(t, 1, runnerMetrics.Active)
	require.EqualValues(t, 2, runnerMetrics.Revoked)
	require.EqualValues(t, 1, runnerMetrics.ActiveSessions)
	require.EqualValues(t, 1, runnerMetrics.ExpiredSessions)
	require.EqualValues(t, 2, runnerMetrics.RevokedSessions)
}

func TestAutomationActiveRunSlotsEnforceCapacityAndRelease(t *testing.T) {
	ctx := context.Background()
	store := newAutomationStoreForTest(t)
	now := time.Date(2026, 6, 5, 15, 20, 0, 0, time.UTC)
	run1 := AutomationRun{ID: "run-slot-1", WorkspaceID: "ws-1", Name: "Slots", Status: AutomationRunStatusRunning, CreatedAt: now, UpdatedAt: now}
	_, err := store.CreateRunWithRuntimeWithCapacity(ctx, 1, run1, nil, nil, nil)
	require.NoError(t, err)

	run2 := AutomationRun{ID: "run-slot-2", WorkspaceID: "ws-1", Name: "Slots", Status: AutomationRunStatusRunning, CreatedAt: now, UpdatedAt: now}
	_, err = store.CreateRunWithRuntimeWithCapacity(ctx, 1, run2, nil, nil, nil)
	require.ErrorIs(t, err, ErrActiveRunCapacity)

	var slots int64
	require.NoError(t, store.db.Model(&AutomationRunSlot{}).Where("workspace_id = ?", "ws-1").Count(&slots).Error)
	require.EqualValues(t, 1, slots)

	require.NoError(t, store.CancelRunRuntime(ctx, "ws-1", "run-slot-1", "user-1", "release", now.Add(time.Minute)))
	require.NoError(t, store.db.Model(&AutomationRunSlot{}).Where("workspace_id = ?", "ws-1").Count(&slots).Error)
	require.Zero(t, slots)

	_, err = store.CreateRunWithRuntimeWithCapacity(ctx, 1, run2, nil, nil, nil)
	require.NoError(t, err)
}

func TestAutomationActiveRunSlotsUseDBUniqueGate(t *testing.T) {
	ctx := context.Background()
	store := newAutomationStoreForTest(t)
	now := time.Date(2026, 6, 5, 15, 25, 0, 0, time.UTC)
	require.NoError(t, store.db.Create(&AutomationRunSlot{WorkspaceID: "ws-1", SlotIndex: 0, RunID: "run-existing", CreatedAt: now, UpdatedAt: now}).Error)

	run := AutomationRun{ID: "run-slot-blocked", WorkspaceID: "ws-1", Name: "Slots", Status: AutomationRunStatusRunning, CreatedAt: now, UpdatedAt: now}
	_, err := store.CreateRunWithRuntimeWithCapacity(ctx, 1, run, nil, nil, nil)
	require.True(t, errors.Is(err, ErrActiveRunCapacity), "CreateRunWithRuntimeWithCapacity error = %v", err)
}

func TestAutomationActiveRunSlotsCountOutOfRangeSlots(t *testing.T) {
	ctx := context.Background()
	store := newAutomationStoreForTest(t)
	now := time.Date(2026, 6, 6, 16, 15, 0, 0, time.UTC)
	require.NoError(t, store.db.Create(&AutomationRunSlot{WorkspaceID: "ws-1", SlotIndex: 3, RunID: "run-existing", CreatedAt: now, UpdatedAt: now}).Error)

	run := AutomationRun{ID: "run-slot-blocked-by-out-of-range", WorkspaceID: "ws-1", Name: "Slots", Status: AutomationRunStatusRunning, CreatedAt: now, UpdatedAt: now}
	_, err := store.CreateRunWithRuntimeWithCapacity(ctx, 1, run, nil, nil, nil)
	require.True(t, errors.Is(err, ErrActiveRunCapacity), "CreateRunWithRuntimeWithCapacity error = %v", err)
}

func TestAutomationActiveRunSlotMetricsAndRepair(t *testing.T) {
	ctx := context.Background()
	store := newAutomationStoreForTest(t)
	now := time.Date(2026, 6, 6, 16, 30, 0, 0, time.UTC)
	completedAt := now.Add(-time.Minute)
	require.NoError(t, store.db.Create([]AutomationRun{
		{ID: "run-active", WorkspaceID: "ws-1", Name: "Active", Status: AutomationRunStatusRunning, CreatedAt: now, UpdatedAt: now},
		{ID: "run-terminal", WorkspaceID: "ws-1", Name: "Terminal", Status: AutomationRunStatusSucceeded, CompletedAt: &completedAt, CreatedAt: now, UpdatedAt: now},
	}).Error)
	require.NoError(t, store.db.Create([]AutomationRunSlot{
		{WorkspaceID: "ws-1", SlotIndex: 0, RunID: "run-active", CreatedAt: now, UpdatedAt: now},
		{WorkspaceID: "ws-1", SlotIndex: 1, RunID: "run-terminal", CreatedAt: now, UpdatedAt: now},
		{WorkspaceID: "ws-1", SlotIndex: 2, RunID: "run-missing", CreatedAt: now, UpdatedAt: now},
	}).Error)

	metrics, err := store.RunMetrics(ctx, "ws-1")
	require.NoError(t, err)
	require.EqualValues(t, 3, metrics.ActiveSlots)
	require.EqualValues(t, 2, metrics.LeakedActiveSlots)

	repaired, err := store.RepairLeakedActiveRunSlots(ctx, 10)
	require.NoError(t, err)
	require.EqualValues(t, 2, repaired)

	metrics, err = store.RunMetrics(ctx, "ws-1")
	require.NoError(t, err)
	require.EqualValues(t, 1, metrics.ActiveSlots)
	require.Zero(t, metrics.LeakedActiveSlots)
	var slot AutomationRunSlot
	require.NoError(t, store.db.Where("workspace_id = ? AND slot_index = ?", "ws-1", 0).First(&slot).Error)
	require.Equal(t, "run-active", slot.RunID)
}

func TestAutomationQueueMatchesRunnerLabels(t *testing.T) {
	ctx := context.Background()
	store := newAutomationStoreForTest(t)
	now := time.Date(2026, 6, 2, 15, 30, 0, 0, time.UTC)
	_, err := store.CreateRun(ctx, AutomationRun{ID: "run-labels", WorkspaceID: "ws-1", Name: "Queue", Status: AutomationRunStatusRunning, CreatedAt: now, UpdatedAt: now})
	require.NoError(t, err)
	require.NoError(t, store.CreateJobRuns(ctx, []AutomationJobRun{
		{ID: "job-macos", WorkspaceID: "ws-1", RunID: "run-labels", JobID: "macos", JobType: "github_actions_job", Status: AutomationJobRunStatusQueued, RunsOnJSON: `["self-hosted","macos"]`, ScheduledAt: now, CreatedAt: now, UpdatedAt: now},
		{ID: "job-linux", WorkspaceID: "ws-1", RunID: "run-labels", JobID: "linux", JobType: "github_actions_job", Status: AutomationJobRunStatusQueued, RunsOnJSON: `"ubuntu-latest"`, ScheduledAt: now, CreatedAt: now, UpdatedAt: now},
	}))
	_, err = store.EnqueueJob(ctx, AutomationJob{WorkspaceID: "ws-1", RunID: "run-labels", JobRunID: "job-macos", QueueName: "default", Status: AutomationJobStatusReady, Priority: 10, ScheduledAt: now, MaxAttempts: 3, IdempotencyKey: "run-labels:macos", RunnerGroup: "trusted", LabelsJSON: `["self-hosted","macos"]`, ShardBucket: 0, CreatedAt: now, UpdatedAt: now})
	require.NoError(t, err)
	linuxJob, err := store.EnqueueJob(ctx, AutomationJob{WorkspaceID: "ws-1", RunID: "run-labels", JobRunID: "job-linux", QueueName: "default", Status: AutomationJobStatusReady, Priority: 20, ScheduledAt: now, MaxAttempts: 3, IdempotencyKey: "run-labels:linux", LabelsJSON: `["ubuntu-latest"]`, ShardBucket: 0, CreatedAt: now, UpdatedAt: now})
	require.NoError(t, err)
	require.Equal(t, "ubuntu-latest", linuxJob.LabelKey)
	require.NotEmpty(t, linuxJob.LabelKeyHash)
	require.Equal(t, 1, linuxJob.LabelCount)

	claimedLinux, err := store.ClaimJobs(ctx, ClaimJobsRequest{WorkspaceID: "ws-1", QueueName: "default", ShardBucket: 0, LeaseOwner: "linux-runner", LeaseDuration: time.Minute, Limit: 1, Labels: []string{"ubuntu-latest"}, Now: now.Add(time.Second)})
	require.NoError(t, err)
	require.Len(t, claimedLinux, 1)
	require.Equal(t, "job-linux", claimedLinux[0].JobRunID)

	claimedWrongGroup, err := store.ClaimJobs(ctx, ClaimJobsRequest{WorkspaceID: "ws-1", QueueName: "default", ShardBucket: 0, LeaseOwner: "mac-runner", LeaseDuration: time.Minute, Limit: 1, Labels: []string{"self-hosted", "macos"}, Now: now.Add(2 * time.Second)})
	require.NoError(t, err)
	require.Empty(t, claimedWrongGroup)

	claimedMac, err := store.ClaimJobs(ctx, ClaimJobsRequest{WorkspaceID: "ws-1", QueueName: "default", ShardBucket: 0, LeaseOwner: "mac-runner", LeaseDuration: time.Minute, Limit: 1, RunnerGroup: "trusted", Labels: []string{"self-hosted", "macos", "x64"}, Now: now.Add(3 * time.Second)})
	require.NoError(t, err)
	require.Len(t, claimedMac, 1)
	require.Equal(t, "job-macos", claimedMac[0].JobRunID)
}

func TestAutomationQueueClaimsAcrossShardBuckets(t *testing.T) {
	ctx := context.Background()
	store := newAutomationStoreForTest(t)
	now := time.Date(2026, 6, 6, 18, 0, 0, 0, time.UTC)
	_, err := store.CreateRun(ctx, AutomationRun{ID: "run-shards", WorkspaceID: "ws-1", Name: "Queue", Status: AutomationRunStatusRunning, CreatedAt: now, UpdatedAt: now})
	require.NoError(t, err)
	require.NoError(t, store.CreateJobRuns(ctx, []AutomationJobRun{
		{ID: "job-shard-1", WorkspaceID: "ws-1", RunID: "run-shards", JobID: "shard-1", JobType: "github_actions_job", Status: AutomationJobRunStatusQueued, ScheduledAt: now, CreatedAt: now, UpdatedAt: now},
		{ID: "job-shard-2", WorkspaceID: "ws-1", RunID: "run-shards", JobID: "shard-2", JobType: "github_actions_job", Status: AutomationJobRunStatusQueued, ScheduledAt: now, CreatedAt: now, UpdatedAt: now},
	}))
	_, err = store.EnqueueJob(ctx, AutomationJob{WorkspaceID: "ws-1", RunID: "run-shards", JobRunID: "job-shard-1", QueueName: "default", Status: AutomationJobStatusReady, Priority: 10, ScheduledAt: now, MaxAttempts: 3, IdempotencyKey: "run-shards:shard-1", ShardBucket: 1, CreatedAt: now, UpdatedAt: now})
	require.NoError(t, err)
	_, err = store.EnqueueJob(ctx, AutomationJob{WorkspaceID: "ws-1", RunID: "run-shards", JobRunID: "job-shard-2", QueueName: "default", Status: AutomationJobStatusReady, Priority: 20, ScheduledAt: now, MaxAttempts: 3, IdempotencyKey: "run-shards:shard-2", ShardBucket: 2, CreatedAt: now, UpdatedAt: now})
	require.NoError(t, err)

	claimed, err := store.ClaimJobs(ctx, ClaimJobsRequest{WorkspaceID: "ws-1", QueueName: "default", ShardBuckets: []int{2, 1, 2}, LeaseOwner: "runner", LeaseDuration: time.Minute, Limit: 2, Now: now.Add(time.Second)})
	require.NoError(t, err)
	require.Len(t, claimed, 2)
	require.Equal(t, "job-shard-1", claimed[0].JobRunID)
	require.Equal(t, "job-shard-2", claimed[1].JobRunID)

	_, err = store.ClaimJobs(ctx, ClaimJobsRequest{WorkspaceID: "ws-1", QueueName: "default", ShardBuckets: []int{AutomationClaimShardCount}, LeaseOwner: "runner", Limit: 1, Now: now.Add(time.Second)})
	require.ErrorContains(t, err, "automation claim shard_bucket is invalid")
}

func TestAutomationQueueIndexedClaimSkipsUnmatchedEarlierJobs(t *testing.T) {
	ctx := context.Background()
	store := newAutomationStoreForTest(t)
	now := time.Date(2026, 6, 2, 15, 35, 0, 0, time.UTC)
	_, err := store.CreateRun(ctx, AutomationRun{ID: "run-indexed-claim", WorkspaceID: "ws-1", Name: "Queue", Status: AutomationRunStatusRunning, CreatedAt: now, UpdatedAt: now})
	require.NoError(t, err)

	var jobRuns []AutomationJobRun
	for idx := 0; idx < 12; idx++ {
		jobRuns = append(jobRuns, AutomationJobRun{ID: "job-macos-blocker-" + string(rune('a'+idx)), WorkspaceID: "ws-1", RunID: "run-indexed-claim", JobID: "macos", JobType: "github_actions_job", Status: AutomationJobRunStatusQueued, RunsOnJSON: `["self-hosted","macos"]`, ScheduledAt: now, CreatedAt: now, UpdatedAt: now})
	}
	jobRuns = append(jobRuns, AutomationJobRun{ID: "job-linux-match", WorkspaceID: "ws-1", RunID: "run-indexed-claim", JobID: "linux", JobType: "github_actions_job", Status: AutomationJobRunStatusQueued, RunsOnJSON: `"ubuntu-latest"`, ScheduledAt: now, CreatedAt: now, UpdatedAt: now})
	require.NoError(t, store.CreateJobRuns(ctx, jobRuns))

	for idx := 0; idx < 12; idx++ {
		jobID := "job-macos-blocker-" + string(rune('a'+idx))
		_, err = store.EnqueueJob(ctx, AutomationJob{WorkspaceID: "ws-1", RunID: "run-indexed-claim", JobRunID: jobID, QueueName: "default", Status: AutomationJobStatusReady, Priority: idx, ScheduledAt: now, MaxAttempts: 3, IdempotencyKey: "run-indexed-claim:" + jobID, RunnerGroup: "trusted", LabelsJSON: `["self-hosted","macos"]`, ShardBucket: 0, CreatedAt: now, UpdatedAt: now})
		require.NoError(t, err)
	}
	_, err = store.EnqueueJob(ctx, AutomationJob{WorkspaceID: "ws-1", RunID: "run-indexed-claim", JobRunID: "job-linux-match", QueueName: "default", Status: AutomationJobStatusReady, Priority: 100, ScheduledAt: now, MaxAttempts: 3, IdempotencyKey: "run-indexed-claim:linux", LabelsJSON: `["ubuntu-latest"]`, ShardBucket: 0, CreatedAt: now, UpdatedAt: now})
	require.NoError(t, err)

	claimed, err := store.ClaimJobs(ctx, ClaimJobsRequest{WorkspaceID: "ws-1", QueueName: "default", ShardBucket: 0, LeaseOwner: "linux-runner", LeaseDuration: time.Minute, Limit: 1, Labels: []string{"ubuntu-latest"}, Now: now.Add(time.Second)})
	require.NoError(t, err)
	require.Len(t, claimed, 1)
	require.Equal(t, "job-linux-match", claimed[0].JobRunID)
}

func TestAutomationQueueClaimOrdersByScheduleBeforePriority(t *testing.T) {
	ctx := context.Background()
	store := newAutomationStoreForTest(t)
	now := time.Date(2026, 6, 5, 15, 10, 0, 0, time.UTC)
	_, err := store.CreateRun(ctx, AutomationRun{ID: "run-claim-order", WorkspaceID: "ws-1", Name: "Queue", Status: AutomationRunStatusRunning, CreatedAt: now, UpdatedAt: now})
	require.NoError(t, err)
	require.NoError(t, store.CreateJobRuns(ctx, []AutomationJobRun{
		{ID: "job-newer-priority", WorkspaceID: "ws-1", RunID: "run-claim-order", JobID: "newer", JobType: "github_actions_job", Status: AutomationJobRunStatusQueued, ScheduledAt: now.Add(-time.Minute), CreatedAt: now, UpdatedAt: now},
		{ID: "job-older-ready", WorkspaceID: "ws-1", RunID: "run-claim-order", JobID: "older", JobType: "github_actions_job", Status: AutomationJobRunStatusQueued, ScheduledAt: now.Add(-2 * time.Minute), CreatedAt: now, UpdatedAt: now},
	}))
	_, err = store.EnqueueJob(ctx, AutomationJob{WorkspaceID: "ws-1", RunID: "run-claim-order", JobRunID: "job-newer-priority", QueueName: "default", Status: AutomationJobStatusReady, Priority: 1, ScheduledAt: now.Add(-time.Minute), MaxAttempts: 3, IdempotencyKey: "run-claim-order:newer", ShardBucket: 0, CreatedAt: now, UpdatedAt: now})
	require.NoError(t, err)
	_, err = store.EnqueueJob(ctx, AutomationJob{WorkspaceID: "ws-1", RunID: "run-claim-order", JobRunID: "job-older-ready", QueueName: "default", Status: AutomationJobStatusReady, Priority: 100, ScheduledAt: now.Add(-2 * time.Minute), MaxAttempts: 3, IdempotencyKey: "run-claim-order:older", ShardBucket: 0, CreatedAt: now, UpdatedAt: now})
	require.NoError(t, err)

	claimed, err := store.ClaimJobs(ctx, ClaimJobsRequest{WorkspaceID: "ws-1", QueueName: "default", ShardBucket: 0, LeaseOwner: "runner", LeaseDuration: time.Minute, Limit: 1, Now: now})
	require.NoError(t, err)
	require.Len(t, claimed, 1)
	require.Equal(t, "job-older-ready", claimed[0].JobRunID)
}

func TestAutomationQueueClaimIsWorkspaceScoped(t *testing.T) {
	ctx := context.Background()
	store := newAutomationStoreForTest(t)
	now := time.Date(2026, 6, 5, 13, 0, 0, 0, time.UTC)
	for _, workspaceID := range []string{"ws-1", "ws-2"} {
		runID := "run-" + workspaceID
		jobRunID := "job-" + workspaceID
		_, err := store.CreateRun(ctx, AutomationRun{ID: runID, WorkspaceID: workspaceID, Name: "Scoped", Status: AutomationRunStatusRunning, CreatedAt: now, UpdatedAt: now})
		require.NoError(t, err)
		require.NoError(t, store.CreateJobRuns(ctx, []AutomationJobRun{{ID: jobRunID, WorkspaceID: workspaceID, RunID: runID, JobID: "build", JobType: "github_actions_job", Status: AutomationJobRunStatusQueued, RunsOnJSON: `["self-hosted"]`, ScheduledAt: now, CreatedAt: now, UpdatedAt: now}}))
		_, err = store.EnqueueJob(ctx, AutomationJob{WorkspaceID: workspaceID, RunID: runID, JobRunID: jobRunID, QueueName: "default", Status: AutomationJobStatusReady, Priority: 10, ScheduledAt: now, MaxAttempts: 3, IdempotencyKey: runID + ":build", LabelsJSON: `["self-hosted"]`, ShardBucket: 0, CreatedAt: now, UpdatedAt: now})
		require.NoError(t, err)
	}

	claimed, err := store.ClaimJobs(ctx, ClaimJobsRequest{WorkspaceID: "ws-1", QueueName: "default", ShardBucket: 0, LeaseOwner: "runner-ws-1", LeaseDuration: time.Minute, Limit: 10, Labels: []string{"self-hosted"}, Now: now.Add(time.Second)})
	require.NoError(t, err)
	require.Len(t, claimed, 1)
	require.Equal(t, "ws-1", claimed[0].WorkspaceID)
	require.Equal(t, "job-ws-1", claimed[0].JobRunID)

	ws2Job, err := store.GetJobRun(ctx, "ws-2", "job-ws-2")
	require.NoError(t, err)
	require.Equal(t, AutomationJobRunStatusQueued, ws2Job.Status)
}

func TestAutomationQueueClaimRejectsLabelsAboveSubsetCap(t *testing.T) {
	ctx := context.Background()
	store := newAutomationStoreForTest(t)
	now := time.Date(2026, 6, 2, 15, 37, 0, 0, time.UTC)
	labels := []string{"a", "b", "c", "d", "e", "f", "g", "h", "i"}
	_, err := store.ClaimJobs(ctx, ClaimJobsRequest{WorkspaceID: "ws-1", QueueName: "default", ShardBucket: 0, LeaseOwner: "runner", Limit: 1, Labels: labels, MaxRunnerLabels: 32, Now: now})
	require.ErrorContains(t, err, "automation runner labels exceed maximum count")
}

func TestAutomationQueueConcurrentClaimDoesNotDuplicate(t *testing.T) {
	ctx := context.Background()
	store := newAutomationStoreForConcurrentTest(t)
	now := time.Date(2026, 6, 2, 15, 40, 0, 0, time.UTC)
	_, err := store.CreateRun(ctx, AutomationRun{ID: "run-concurrent-claim", WorkspaceID: "ws-1", Name: "Queue", Status: AutomationRunStatusRunning, CreatedAt: now, UpdatedAt: now})
	require.NoError(t, err)
	require.NoError(t, store.CreateJobRuns(ctx, []AutomationJobRun{{ID: "job-one", WorkspaceID: "ws-1", RunID: "run-concurrent-claim", JobID: "one", JobType: "github_actions_job", Status: AutomationJobRunStatusQueued, ScheduledAt: now, CreatedAt: now, UpdatedAt: now}}))
	job, err := store.EnqueueJob(ctx, AutomationJob{WorkspaceID: "ws-1", RunID: "run-concurrent-claim", JobRunID: "job-one", QueueName: "default", Status: AutomationJobStatusReady, Priority: 10, ScheduledAt: now, MaxAttempts: 3, IdempotencyKey: "run-concurrent-claim:one", ShardBucket: 0, CreatedAt: now, UpdatedAt: now})
	require.NoError(t, err)

	const workers = 8
	var wg sync.WaitGroup
	claimedIDs := make(chan int64, workers)
	errs := make(chan error, workers)
	for idx := 0; idx < workers; idx++ {
		wg.Add(1)
		go func(idx int) {
			defer wg.Done()
			claimed, err := store.ClaimJobs(ctx, ClaimJobsRequest{WorkspaceID: "ws-1", QueueName: "default", ShardBucket: 0, LeaseOwner: "worker", LeaseDuration: time.Minute, Limit: 1, Now: now.Add(time.Duration(idx) * time.Millisecond)})
			if err != nil {
				errs <- err
				return
			}
			for _, item := range claimed {
				claimedIDs <- item.ID
			}
		}(idx)
	}
	wg.Wait()
	close(claimedIDs)
	close(errs)
	for err := range errs {
		require.NoError(t, err)
	}
	var ids []int64
	for id := range claimedIDs {
		ids = append(ids, id)
	}
	require.Equal(t, []int64{job.ID}, ids)
}

func TestAutomationQueueMetrics(t *testing.T) {
	ctx := context.Background()
	store := newAutomationStoreForTest(t)
	now := time.Date(2026, 6, 2, 15, 45, 0, 0, time.UTC)
	leaseUntil := now.Add(-time.Second)
	_, err := store.CreateRun(ctx, AutomationRun{ID: "run-metrics", WorkspaceID: "ws-1", Name: "Queue", Status: AutomationRunStatusRunning, CreatedAt: now, UpdatedAt: now})
	require.NoError(t, err)
	require.NoError(t, store.CreateJobRuns(ctx, []AutomationJobRun{
		{ID: "job-ready", WorkspaceID: "ws-1", RunID: "run-metrics", JobID: "ready", JobType: "github_actions_job", Status: AutomationJobRunStatusQueued, ScheduledAt: now, CreatedAt: now, UpdatedAt: now},
		{ID: "job-leased", WorkspaceID: "ws-1", RunID: "run-metrics", JobID: "leased", JobType: "github_actions_job", Status: AutomationJobRunStatusRunning, ScheduledAt: now, CreatedAt: now, UpdatedAt: now},
	}))
	_, err = store.EnqueueJob(ctx, AutomationJob{WorkspaceID: "ws-1", RunID: "run-metrics", JobRunID: "job-ready", QueueName: "default", Status: AutomationJobStatusReady, Attempt: 1, ScheduledAt: now, MaxAttempts: 3, IdempotencyKey: "run-metrics:ready", ShardBucket: 0, CreatedAt: now, UpdatedAt: now})
	require.NoError(t, err)
	_, err = store.EnqueueJob(ctx, AutomationJob{WorkspaceID: "ws-1", RunID: "run-metrics", JobRunID: "job-leased", QueueName: "default", Status: AutomationJobStatusLeased, LeaseOwner: "runner-1", LeaseUntil: &leaseUntil, Attempt: 1, ScheduledAt: now, MaxAttempts: 3, IdempotencyKey: "run-metrics:leased", ShardBucket: 0, CreatedAt: now, UpdatedAt: now})
	require.NoError(t, err)

	metrics, err := store.QueueMetrics(ctx, "", now)
	require.NoError(t, err)
	require.EqualValues(t, 1, metrics.Ready)
	require.EqualValues(t, 1, metrics.Leased)
	require.EqualValues(t, 1, metrics.ExpiredLeases)
	require.EqualValues(t, 1, metrics.RetryReady)
}

func TestAutomationDashboardMetricsAreWorkspaceScoped(t *testing.T) {
	ctx := context.Background()
	store := newAutomationStoreForTest(t)
	now := time.Date(2026, 6, 2, 15, 50, 0, 0, time.UTC)
	leaseUntil := now.Add(-time.Second)
	completedAt := now.Add(-time.Minute)

	_, err := store.CreateRun(ctx, AutomationRun{ID: "run-ws-1", WorkspaceID: "ws-1", Name: "Queue", Status: AutomationRunStatusRunning, CreatedAt: now, UpdatedAt: now})
	require.NoError(t, err)
	require.NoError(t, store.CreateJobRuns(ctx, []AutomationJobRun{
		{ID: "job-ws-1-ready", WorkspaceID: "ws-1", RunID: "run-ws-1", JobID: "ready", JobType: "github_actions_job", Status: AutomationJobRunStatusSucceeded, ScheduledAt: now, CreatedAt: now, UpdatedAt: now, CompletedAt: &completedAt},
		{ID: "job-ws-1-leased", WorkspaceID: "ws-1", RunID: "run-ws-1", JobID: "leased", JobType: "github_actions_job", Status: AutomationJobRunStatusSucceeded, ScheduledAt: now, CreatedAt: now, UpdatedAt: now, CompletedAt: &completedAt},
	}))
	_, err = store.EnqueueJob(ctx, AutomationJob{WorkspaceID: "ws-1", RunID: "run-ws-1", JobRunID: "job-ws-1-ready", QueueName: "default", Status: AutomationJobStatusReady, Attempt: 1, ScheduledAt: now, MaxAttempts: 3, IdempotencyKey: "run-ws-1:ready", ShardBucket: 0, CreatedAt: now, UpdatedAt: now})
	require.NoError(t, err)
	_, err = store.EnqueueJob(ctx, AutomationJob{WorkspaceID: "ws-1", RunID: "run-ws-1", JobRunID: "job-ws-1-leased", QueueName: "default", Status: AutomationJobStatusLeased, LeaseOwner: "runner-1", LeaseUntil: &leaseUntil, Attempt: 1, ScheduledAt: now, MaxAttempts: 3, IdempotencyKey: "run-ws-1:leased", ShardBucket: 0, CreatedAt: now, UpdatedAt: now})
	require.NoError(t, err)

	_, err = store.CreateRun(ctx, AutomationRun{ID: "run-ws-2", WorkspaceID: "ws-2", Name: "Queue", Status: AutomationRunStatusRunning, CreatedAt: now, UpdatedAt: now})
	require.NoError(t, err)
	require.NoError(t, store.CreateJobRuns(ctx, []AutomationJobRun{
		{ID: "job-ws-2-ready", WorkspaceID: "ws-2", RunID: "run-ws-2", JobID: "ready", JobType: "github_actions_job", Status: AutomationJobRunStatusSucceeded, ScheduledAt: now, CreatedAt: now, UpdatedAt: now, CompletedAt: &completedAt},
	}))
	_, err = store.EnqueueJob(ctx, AutomationJob{WorkspaceID: "ws-2", RunID: "run-ws-2", JobRunID: "job-ws-2-ready", QueueName: "default", Status: AutomationJobStatusReady, Attempt: 1, ScheduledAt: now, MaxAttempts: 3, IdempotencyKey: "run-ws-2:ready", ShardBucket: 0, CreatedAt: now, UpdatedAt: now})
	require.NoError(t, err)

	queueMetrics, err := store.QueueMetrics(ctx, "ws-1", now)
	require.NoError(t, err)
	require.EqualValues(t, 1, queueMetrics.Ready)
	require.EqualValues(t, 1, queueMetrics.Leased)
	require.EqualValues(t, 1, queueMetrics.ExpiredLeases)
	require.EqualValues(t, 1, queueMetrics.RetryReady)

	runMetrics, err := store.RunMetrics(ctx, "ws-1")
	require.NoError(t, err)
	require.EqualValues(t, 1, runMetrics.Running)
	require.EqualValues(t, 1, runMetrics.TerminalInconsistent)
}

func TestAutomationQueueExpiredLeaseRequeue(t *testing.T) {
	ctx := context.Background()
	store := newAutomationStoreForTest(t)
	now := time.Date(2026, 6, 2, 16, 0, 0, 0, time.UTC)
	_, err := store.CreateRun(ctx, AutomationRun{ID: "run-1", WorkspaceID: "ws-1", Name: "Queue", Status: AutomationRunStatusRunning, CreatedAt: now, UpdatedAt: now})
	require.NoError(t, err)
	require.NoError(t, store.CreateJobRuns(ctx, []AutomationJobRun{{ID: "job-build", WorkspaceID: "ws-1", RunID: "run-1", JobID: "build", JobType: "github_actions_job", Status: AutomationJobRunStatusQueued, ScheduledAt: now, CreatedAt: now, UpdatedAt: now}}))
	job, err := store.EnqueueJob(ctx, AutomationJob{WorkspaceID: "ws-1", RunID: "run-1", JobRunID: "job-build", QueueName: "default", Status: AutomationJobStatusReady, ScheduledAt: now, MaxAttempts: 3, IdempotencyKey: "run-1:build", ShardBucket: 0, CreatedAt: now, UpdatedAt: now})
	require.NoError(t, err)
	claimed, err := store.ClaimJobs(ctx, ClaimJobsRequest{WorkspaceID: "ws-1", QueueName: "default", ShardBucket: 0, LeaseOwner: "worker-1", LeaseDuration: time.Second, Limit: 1, Now: now})
	require.NoError(t, err)
	require.Len(t, claimed, 1)
	require.Equal(t, job.ID, claimed[0].ID)

	recovered, err := store.RecoverExpiredJobs(ctx, now.Add(2*time.Second), 10)
	require.NoError(t, err)
	require.EqualValues(t, 1, recovered)
	claimedRecovered, err := store.ClaimJobs(ctx, ClaimJobsRequest{WorkspaceID: "ws-1", QueueName: "default", ShardBucket: 0, LeaseOwner: "worker-2", LeaseDuration: time.Minute, Limit: 1, Now: now.Add(33 * time.Second)})
	require.NoError(t, err)
	require.Len(t, claimedRecovered, 1)
	require.Equal(t, "worker-2", claimedRecovered[0].LeaseOwner)
	jobRun, err := store.GetJobRun(ctx, "ws-1", "job-build")
	require.NoError(t, err)
	require.Equal(t, AutomationJobRunStatusQueued, jobRun.Status)
	events, err := store.ListEvents(ctx, "ws-1", "run-1", 1, 10)
	require.NoError(t, err)
	require.Len(t, events, 1)
	require.Equal(t, "job_retry_scheduled", events[0].EventType)
}

func TestAutomationQueueExpiredLeaseRecoveryIncludesOldestJobPerWorkspace(t *testing.T) {
	ctx := context.Background()
	store := newAutomationStoreForTest(t)
	now := time.Date(2026, 6, 2, 16, 15, 0, 0, time.UTC)
	createExpiredJob := func(workspaceID string, suffix string, leaseUntil time.Time) {
		t.Helper()
		runID := "run-" + suffix
		jobRunID := "job-" + suffix
		createLeasedAutomationJob(t, ctx, store, workspaceID, runID, jobRunID, "worker-"+workspaceID, now, leaseUntil)
	}
	for i := 0; i < 5; i++ {
		createExpiredJob("ws-busy", fmt.Sprintf("busy-%02d", i), now.Add(-time.Duration(10-i)*time.Second))
	}
	createExpiredJob("ws-other", "other", now.Add(-time.Second))

	recovered, err := store.RecoverExpiredJobs(ctx, now, 2)

	require.NoError(t, err)
	require.EqualValues(t, 2, recovered)
	busyJobRun, err := store.GetJobRun(ctx, "ws-busy", "job-busy-00")
	require.NoError(t, err)
	require.Equal(t, AutomationJobRunStatusQueued, busyJobRun.Status)
	otherJobRun, err := store.GetJobRun(ctx, "ws-other", "job-other")
	require.NoError(t, err)
	require.Equal(t, AutomationJobRunStatusQueued, otherJobRun.Status)
	stillLeased, err := store.GetJobRun(ctx, "ws-busy", "job-busy-01")
	require.NoError(t, err)
	require.Equal(t, AutomationJobRunStatusRunning, stillLeased.Status)
}

func TestAutomationQueueExpiredLeaseExhaustionFailsRun(t *testing.T) {
	ctx := context.Background()
	store := newAutomationStoreForTest(t)
	now := time.Date(2026, 6, 2, 16, 30, 0, 0, time.UTC)
	_, err := store.CreateRun(ctx, AutomationRun{ID: "run-fail", WorkspaceID: "ws-1", Name: "Queue", Status: AutomationRunStatusRunning, CreatedAt: now, UpdatedAt: now})
	require.NoError(t, err)
	require.NoError(t, store.CreateJobRuns(ctx, []AutomationJobRun{{ID: "job-test", WorkspaceID: "ws-1", RunID: "run-fail", JobID: "test", JobType: "github_actions_job", Status: AutomationJobRunStatusQueued, ScheduledAt: now, CreatedAt: now, UpdatedAt: now}}))
	_, err = store.EnqueueJob(ctx, AutomationJob{WorkspaceID: "ws-1", RunID: "run-fail", JobRunID: "job-test", QueueName: "default", Status: AutomationJobStatusReady, ScheduledAt: now, MaxAttempts: 1, IdempotencyKey: "run-fail:test", ShardBucket: 0, CreatedAt: now, UpdatedAt: now})
	require.NoError(t, err)
	claimed, err := store.ClaimJobs(ctx, ClaimJobsRequest{WorkspaceID: "ws-1", QueueName: "default", ShardBucket: 0, LeaseOwner: "worker-1", LeaseDuration: time.Second, Limit: 1, Now: now})
	require.NoError(t, err)
	require.Len(t, claimed, 1)

	recovered, err := store.RecoverExpiredJobs(ctx, now.Add(2*time.Second), 10)
	require.NoError(t, err)
	require.EqualValues(t, 1, recovered)
	jobRun, err := store.GetJobRun(ctx, "ws-1", "job-test")
	require.NoError(t, err)
	require.Equal(t, AutomationJobRunStatusFailed, jobRun.Status)
	run, err := store.GetRun(ctx, "ws-1", "run-fail")
	require.NoError(t, err)
	require.Equal(t, AutomationRunStatusFailed, run.Status)
	events, err := store.ListEvents(ctx, "ws-1", "run-fail", 1, 10)
	require.NoError(t, err)
	require.Len(t, events, 2)
	require.Equal(t, "job_failed", events[0].EventType)
	require.Equal(t, "run_failed", events[1].EventType)
}