{
  "id": "out-409-recover",
  "title": "Outbound Lead adopts an existing HubSpot contact through one V2 identity-conflict command",
  "kind": "hubspot_sync",
  "executionMode": "serial",
  "executionModeReason": "This case disables outbound on and removes the run-scoped Lead materialized by the staging contact.propertychange subscription before creating its blank-link CRM Lead, then requires one finalized V2 durable identity-conflict command before accepting any link or provider PATCH assertion. Serial execution prevents other HubSpot integration cases from adding queue pressure; the workflow preflight separately rejects an already-busy connector queue.",
  "caseTimeoutMs": 780000,
  "linkedIssues": [],
  "provenance": {
    "businessScenario": "OUT-409-RECOVER - regression for the 2026-06-29 prod incident where PushLeadToHubSpotContact encountered an existing same-email HubSpot contact while the Lead had a blank HubSpot_Id__c / remote link; the V1 symptom was 409 CONFLICT. The V2 business contract is to resolve the existing contact by exact email, adopt and finalize that remote identity through one durable command, write Lead.HubSpot_Id__c, and converge the existing contact instead of creating a duplicate or leaving the Lead unlinked.",
    "reportedBy": "prod HubSpot outbound 409 RCA 2026-06-29",
    "capturedAt": "2026-06-29",
    "calibrated": "[recalibrated 2026-08-27] Staging's inbound app subscribes to contact.propertychange rather than contact.creation. The precondition waits for the resulting Lead, disables its outbound echo, then drains the source lane through the D3 authorities: every immutable Event must be directly owned or projection-covered by a terminal WorkItem, and every processed handoff must own a terminal Runtime command. Event.Status/ProcessedAt is retained history and is not the execution authority. After 60 seconds of source quiet the case removes that Lead and creates a same-email CRM Lead with HubSpot_Sync_Enabled__c=false. The current V2 command ledger does not persist per-provider-attempt response evidence, so this case proves one durable identity-conflict recovery command, exact remote adoption, and provider convergence without claiming a direct first-response 409 receipt."
  },
  "steps": [
    {
      "id": "seed-existing-hubspot-contact",
      "system": "hubspot",
      "method": "POST",
      "path": "/crm/v3/objects/contacts",
      "body": {
        "properties": {
          "email": "out409-{{__runId}}@example.com",
          "firstname": "Recover",
          "lastname": "O409{{__runId}}",
          "jobtitle": "Before409"
        }
      },
      "expectStatusRange": [200, 299],
      "assertions": [
        { "path": "response.body.id", "exists": true }
      ]
    },
    {
      "id": "find-inbound-contact-lead",
      "system": "nexuscrm",
      "authRole": "admin",
      "method": "GET",
      "path": "/services/data/v63.0/query?q=SELECT Id FROM Lead WHERE HubSpot_Id__c='{{steps.seed-existing-hubspot-contact.response.body.id}}' LIMIT 1",
      "expectStatus": 200,
      "poll": { "budgetMs": 240000, "intervalMs": 6000 },
      "onTimeout": "blocked",
      "assertions": [
        { "path": "response.body.records", "lengthAtLeast": 1 }
      ],
      "diagnostics": {
        "connectorEvent": {
          "sourceObjectId": "{{steps.seed-existing-hubspot-contact.response.body.id}}",
          "eventKind": "contact.propertychange",
          "blockOnBacklog": true,
          "blockOnForeignClaimedBy": true
        }
      },
      "timeoutMs": 8000
    },
    {
      "id": "disable-inbound-contact-outbound",
      "system": "nexuscrm",
      "authRole": "admin",
      "method": "PATCH",
      "path": "/services/data/v63.0/sobjects/Lead/{{steps.find-inbound-contact-lead.response.body.records[0].Id}}",
      "body": {
        "HubSpot_Sync_Enabled__c": false
      },
      "expectStatus": 204
    },
    {
      "id": "wait-contact-propertychanges-drained",
      "system": "tidb",
      "method": "GET",
      "sql": "SELECT CASE WHEN COUNT(*) > 0 AND MAX(E.CreatedDate) <= DATE_SUB(UTC_TIMESTAMP(), INTERVAL 60 SECOND) AND NOT EXISTS (SELECT 1 FROM `_System_ConnectorWorkItem` W JOIN `_System_ConnectorEvent` S ON S.Id = W.EventId WHERE S.Namespace = 'hubspot-crm-connector' AND S.EventKind = 'contact.propertychange' AND S.SourceObjectID = '{{steps.seed-existing-hubspot-contact.response.body.id}}' AND W.WorkType IN ('connector_event', 'workflow_action_apply') AND W.State NOT IN ('processed', 'skipped')) AND NOT EXISTS (SELECT 1 FROM `_System_ConnectorWorkItem` W JOIN `_System_ConnectorEvent` S ON S.Id = W.EventId WHERE S.Namespace = 'hubspot-crm-connector' AND S.EventKind = 'contact.propertychange' AND S.SourceObjectID = '{{steps.seed-existing-hubspot-contact.response.body.id}}' AND W.WorkType IN ('connector_event', 'workflow_action_apply') AND W.State = 'processed' AND NOT EXISTS (SELECT 1 FROM `_System_ConnectorRuntimeHandoffReceipt` R JOIN `_System_RuntimePlatformCommand` C ON C.CommandId = R.CommandId WHERE R.WorkId = W.WorkId AND R.EventId = W.EventId AND C.State IN ('COMPLETED', 'FAILED', 'POISONED'))) AND NOT EXISTS (SELECT 1 FROM `_System_ConnectorEvent` X WHERE X.Namespace = 'hubspot-crm-connector' AND X.EventKind = 'contact.propertychange' AND X.SourceObjectID = '{{steps.seed-existing-hubspot-contact.response.body.id}}' AND NOT EXISTS (SELECT 1 FROM `_System_ConnectorWorkItem` W JOIN `_System_ConnectorEvent` S ON S.Id = W.EventId WHERE W.WorkType IN ('connector_event', 'workflow_action_apply') AND (W.EventId = X.Id OR (W.InstalledPackId = X.InstalledPackId AND NULLIF(COALESCE(W.ProjectionKey, ''), '') IS NOT NULL AND COALESCE(W.SourceLaneKey, '') = COALESCE(X.SourceLaneKey, '') AND S.InstalledPackId = X.InstalledPackId AND LOWER(TRIM(S.EventKind)) = LOWER(TRIM(X.EventKind)) AND COALESCE(S.RouteVersion, 0) = COALESCE(X.RouteVersion, 0) AND COALESCE(S.CompanionPackInstallId, '') = COALESCE(X.CompanionPackInstallId, '') AND COALESCE(S.CompanionFlowVersionId, '') = COALESCE(X.CompanionFlowVersionId, '') AND ((X.EnqueueOrdinal <> 0 AND S.EnqueueOrdinal <> 0 AND (X.EnqueueOrdinal < S.EnqueueOrdinal OR (X.EnqueueOrdinal = S.EnqueueOrdinal AND X.Id <= S.Id))) OR ((X.EnqueueOrdinal = 0 OR S.EnqueueOrdinal = 0) AND (X.CreatedDate < S.CreatedDate OR (X.CreatedDate = S.CreatedDate AND (X.DeliverySequence < S.DeliverySequence OR (X.DeliverySequence = S.DeliverySequence AND X.Id <= S.Id)))))))))) THEN 1 ELSE 0 END AS Ready, COUNT(*) AS EventCount, MAX(E.CreatedDate) AS LastEventCreatedAt FROM `_System_ConnectorEvent` E WHERE E.Namespace = 'hubspot-crm-connector' AND E.EventKind = 'contact.propertychange' AND E.SourceObjectID = '{{steps.seed-existing-hubspot-contact.response.body.id}}'",
      "expectStatus": 200,
      "poll": { "budgetMs": 180000, "intervalMs": 6000 },
      "onTimeout": "blocked",
      "assertions": [
        { "path": "response.body.records[0].Ready", "numericEquals": 1 }
      ]
    },
    {
      "id": "clear-inbound-contact-link",
      "system": "nexuscrm",
      "authRole": "admin",
      "method": "PATCH",
      "path": "/services/data/v63.0/sobjects/Lead/{{steps.find-inbound-contact-lead.response.body.records[0].Id}}",
      "body": {
        "HubSpot_Id__c": null
      },
      "expectStatus": 204
    },
    {
      "id": "delete-inbound-contact-lead",
      "system": "nexuscrm",
      "authRole": "admin",
      "method": "DELETE",
      "path": "/services/data/v63.0/sobjects/Lead/{{steps.find-inbound-contact-lead.response.body.records[0].Id}}",
      "expectStatus": 204
    },
    {
      "id": "assert-no-inbound-link-precondition",
      "system": "nexuscrm",
      "authRole": "admin",
      "method": "GET",
      "path": "/services/data/v63.0/query?q=SELECT COUNT() FROM Lead WHERE HubSpot_Id__c='{{steps.seed-existing-hubspot-contact.response.body.id}}'",
      "expectStatus": 200,
      "poll": { "budgetMs": 60000, "intervalMs": 6000 },
      "onTimeout": "fail",
      "assertions": [
        { "path": "response.body.totalSize", "numericEquals": 0 }
      ]
    },
    {
      "id": "create-blank-link-lead",
      "system": "nexuscrm",
      "authRole": "admin",
      "method": "POST",
      "path": "/services/data/v63.0/sobjects/Lead",
      "body": {
        "FirstName": "Recover",
        "LastName": "O409{{__runId}}",
        "Company": "O409 Co {{__runId}}",
        "Email": "out409-{{__runId}}@example.com",
        "Title": "Before409",
        "HubSpot_Sync_Enabled__c": false
      },
      "expectStatusRange": [200, 299],
      "assertions": [
        { "path": "response.body.id", "exists": true }
      ]
    },
    {
      "id": "assert-lead-starts-unlinked",
      "system": "nexuscrm",
      "authRole": "admin",
      "method": "GET",
      "path": "/services/data/v63.0/query?q=SELECT HubSpot_Id__c FROM Lead WHERE Id='{{steps.create-blank-link-lead.response.body.id}}' LIMIT 1",
      "expectStatus": 200,
      "assertions": [
        { "path": "response.body.records[0].HubSpot_Id__c", "exists": false }
      ]
    },
    {
      "id": "enable-outbound-sync",
      "system": "nexuscrm",
      "authRole": "admin",
      "method": "PATCH",
      "path": "/services/data/v63.0/sobjects/Lead/{{steps.create-blank-link-lead.response.body.id}}",
      "body": {
        "Title": "Recovered409-{{__runId}}",
        "HubSpot_Sync_Enabled__c": true
      },
      "expectStatus": 204
    },
    {
      "id": "assert-identity-conflict-command-finalized",
      "system": "tidb",
      "method": "GET",
      "sql": "SELECT Id, State, AttemptCount, ResponseStatus, RemoteId, CASE WHEN FailureCode IS NULL OR FailureCode = '' THEN 1 ELSE 0 END AS FailureCodeAbsent FROM `_System_IntegrationOutboundCommand` WHERE SourceObject = 'Lead' AND SourceRecordId = '{{steps.create-blank-link-lead.response.body.id}}' AND FlowActionName = 'PushLeadToHubSpotContact' ORDER BY CreatedDate DESC LIMIT 1",
      "expectStatus": 200,
      "poll": { "budgetMs": 240000, "intervalMs": 6000 },
      "onTimeout": "fail",
      "assertions": [
        { "path": "response.body.records", "lengthAtLeast": 1 },
        { "path": "response.body.records[0].State", "equals": "succeeded" },
        { "path": "response.body.records[0].AttemptCount", "numericEquals": 2 },
        { "path": "response.body.records[0].ResponseStatus", "numericEquals": 200 },
        { "path": "response.body.records[0].RemoteId", "equals": "{{steps.seed-existing-hubspot-contact.response.body.id}}" },
        { "path": "response.body.records[0].FailureCodeAbsent", "numericEquals": 1 }
      ]
    },
    {
      "id": "wait-adopt-existing-contact",
      "system": "nexuscrm",
      "authRole": "admin",
      "method": "GET",
      "path": "/services/data/v63.0/query?q=SELECT HubSpot_Id__c FROM Lead WHERE Id='{{steps.create-blank-link-lead.response.body.id}}' LIMIT 1",
      "expectStatus": 200,
      "poll": { "budgetMs": 240000, "intervalMs": 6000 },
      "onTimeout": "fail",
      "assertions": [
        { "path": "response.body.records[0].HubSpot_Id__c", "equals": "{{steps.seed-existing-hubspot-contact.response.body.id}}" }
      ]
    },
    {
      "id": "assert-existing-contact-patched",
      "system": "hubspot",
      "method": "GET",
      "path": "/crm/v3/objects/contacts/{{steps.seed-existing-hubspot-contact.response.body.id}}?properties=email,jobtitle",
      "expectStatus": 200,
      "poll": { "budgetMs": 120000, "intervalMs": 6000 },
      "onTimeout": "fail",
      "assertions": [
        { "path": "response.body.properties.email", "includes": "out409-{{__runId}}@example.com", "caseInsensitive": true },
        { "path": "response.body.properties.jobtitle", "equals": "Recovered409-{{__runId}}" }
      ]
    },
    {
      "id": "assert-single-durable-command",
      "system": "tidb",
      "method": "GET",
      "sql": "SELECT COUNT(*) AS CommandCount FROM `_System_IntegrationOutboundCommand` WHERE SourceObject = 'Lead' AND SourceRecordId = '{{steps.create-blank-link-lead.response.body.id}}' AND FlowActionName = 'PushLeadToHubSpotContact'",
      "expectStatus": 200,
      "assertions": [
        { "path": "response.body.records[0].CommandCount", "numericEquals": 1 }
      ]
    },
    {
      "id": "assert-one-linked-lead",
      "system": "nexuscrm",
      "authRole": "admin",
      "method": "GET",
      "path": "/services/data/v63.0/query?q=SELECT COUNT() FROM Lead WHERE HubSpot_Id__c='{{steps.seed-existing-hubspot-contact.response.body.id}}'",
      "expectStatus": 200,
      "poll": { "budgetMs": 60000, "intervalMs": 6000 },
      "onTimeout": "fail",
      "assertions": [
        { "path": "response.body.totalSize", "numericEquals": 1 }
      ]
    }
  ]
}