{ "id": "contact-propertychange-coalescing", "title": "One Contact delta burst preserves siblings, attribution, clear ordering, and source time", "kind": "hubspot_sync", "optional": false, "executionMode": "serial", "executionModeReason": "The case owns one ordered same-Contact webhook batch and its exact Event/Runtime-command terminal shape, so unrelated connector traffic must not overlap it.", "caseGroups": ["hubspot-contact-inbound-regression"], "caseTimeoutMs": 1500000, "linkedIssues": [3877, 4405, 5075], "provenance": { "businessScenario": "A settled Contact receives one numeric-portal delta generation containing all three late attribution fields, an explicit Title clear followed by a newer value, and a Disqualified lifecycle transition. The generation must preserve the existing FirstName sibling, keep the newest Title, populate all attribution fields, use lifecycle occurredAt for Disqualified_Date__c, execute one coalesced mapper, and remain stable after a later settle window.", "reportedBy": "Amber / #3877 production data-loss investigation", "capturedAt": "2026-08-26", "upstreamSource": "closed-unmerged L2 PR #3901, re-authored against the D3 terminal contract", "calibrated": "The signed webhook uses numeric portalId to retain #5075 coverage. CommandId is the cross-service handoff identity; Integration and Runtime ScopeDigest values are intentionally service-local. The selected projection must join through ConnectorRuntimeHandoffReceipt to one COMPLETED Runtime PlatformCommand. The capped FlowAuditLog is not durable acceptance authority. The fixture captures a whole-second source clock so the occurredAt-derived DateTime remains exactly comparable after storage at the declared field precision. The final 60-second readback folds the settled Country/Industry regression lesson into the canonical case instead of restoring duplicate cases." }, "steps": [ { "id": "create-contact", "system": "hubspot", "method": "POST", "path": "/crm/v3/objects/contacts", "body": { "properties": { "email": "coalescing-{{__runId}}@example.com", "firstname": "CoalSibling", "lastname": "Coalescing{{__runId}}", "jobtitle": "CoalescingBaseline" } }, "expectStatusRange": [200, 299], "assertions": [{ "path": "response.body.id", "exists": true }] }, { "id": "wait-linked-lead", "system": "nexuscrm", "authRole": "admin", "method": "GET", "path": "/services/data/v63.0/query?q=SELECT Id,HubSpot_Id__c,FirstName FROM Lead WHERE HubSpot_Id__c='{{steps.create-contact.response.body.id}}' AND IsConverted=false LIMIT 1", "expectStatus": 200, "poll": { "budgetMs": 240000, "intervalMs": 6000 }, "onTimeout": "blocked", "assertions": [ { "path": "response.body.records[0].Id", "exists": true }, { "path": "response.body.records[0].HubSpot_Id__c", "equals": "{{steps.create-contact.response.body.id}}" }, { "path": "response.body.records[0].FirstName", "equals": "CoalSibling" } ] }, { "id": "wait-creation-fanout-drained", "system": "tidb", "method": "GET", "sql": "SELECT CASE WHEN COUNT(*) > 0 AND MAX(E.CreatedDate) <= DATE_SUB(UTC_TIMESTAMP(), INTERVAL 60 SECOND) AND NOT EXISTS (SELECT 1 FROM `_System_ConnectorWorkItem` W JOIN `_System_ConnectorEvent` S ON S.Id = W.EventId WHERE S.Namespace = 'hubspot-crm-connector' AND S.EventKind = 'contact.propertychange' AND S.SourceObjectID = '{{steps.create-contact.response.body.id}}' AND W.WorkType IN ('connector_event', 'workflow_action_apply') AND W.State NOT IN ('processed', 'skipped')) AND NOT EXISTS (SELECT 1 FROM `_System_ConnectorWorkItem` W JOIN `_System_ConnectorEvent` S ON S.Id = W.EventId WHERE S.Namespace = 'hubspot-crm-connector' AND S.EventKind = 'contact.propertychange' AND S.SourceObjectID = '{{steps.create-contact.response.body.id}}' AND W.WorkType IN ('connector_event', 'workflow_action_apply') AND W.State = 'processed' AND NOT EXISTS (SELECT 1 FROM `_System_ConnectorRuntimeHandoffReceipt` R JOIN `_System_RuntimePlatformCommand` C ON C.CommandId = R.CommandId WHERE R.WorkId = W.WorkId AND R.EventId = W.EventId AND C.State IN ('COMPLETED', 'FAILED', 'POISONED'))) AND NOT EXISTS (SELECT 1 FROM `_System_ConnectorEvent` X WHERE X.Namespace = 'hubspot-crm-connector' AND X.EventKind = 'contact.propertychange' AND X.SourceObjectID = '{{steps.create-contact.response.body.id}}' AND NOT EXISTS (SELECT 1 FROM `_System_ConnectorWorkItem` W JOIN `_System_ConnectorEvent` S ON S.Id = W.EventId WHERE W.WorkType IN ('connector_event', 'workflow_action_apply') AND (W.EventId = X.Id OR (W.InstalledPackId = X.InstalledPackId AND NULLIF(COALESCE(W.ProjectionKey, ''), '') IS NOT NULL AND COALESCE(W.SourceLaneKey, '') = COALESCE(X.SourceLaneKey, '') AND S.InstalledPackId = X.InstalledPackId AND LOWER(TRIM(S.EventKind)) = LOWER(TRIM(X.EventKind)) AND COALESCE(S.RouteVersion, 0) = COALESCE(X.RouteVersion, 0) AND COALESCE(S.CompanionPackInstallId, '') = COALESCE(X.CompanionPackInstallId, '') AND COALESCE(S.CompanionFlowVersionId, '') = COALESCE(X.CompanionFlowVersionId, '') AND ((X.EnqueueOrdinal <> 0 AND S.EnqueueOrdinal <> 0 AND (X.EnqueueOrdinal < S.EnqueueOrdinal OR (X.EnqueueOrdinal = S.EnqueueOrdinal AND X.Id <= S.Id))) OR ((X.EnqueueOrdinal = 0 OR S.EnqueueOrdinal = 0) AND (X.CreatedDate < S.CreatedDate OR (X.CreatedDate = S.CreatedDate AND (X.DeliverySequence < S.DeliverySequence OR (X.DeliverySequence = S.DeliverySequence AND X.Id <= S.Id)))))))))) THEN 1 ELSE 0 END AS Ready, COUNT(*) AS EventCount, MAX(E.CreatedDate) AS LastEventCreatedAt FROM `_System_ConnectorEvent` E WHERE E.Namespace = 'hubspot-crm-connector' AND E.EventKind = 'contact.propertychange' AND E.SourceObjectID = '{{steps.create-contact.response.body.id}}'", "expectStatus": 200, "poll": { "budgetMs": 240000, "intervalMs": 6000 }, "onTimeout": "blocked", "assertions": [ { "path": "response.body.records[0].Ready", "numericEquals": 1 } ] }, { "id": "capture-burst-clock", "system": "tidb", "method": "GET", "sql": "SELECT ClockMs AS LeadSourceAtMs, ClockMs + 1 AS RecentSourceAtMs, ClockMs + 2 AS RecentDetailAtMs, ClockMs + 3 AS InitialTitleAtMs, ClockMs + 4 AS ClearedTitleAtMs, ClockMs + 5 AS FinalTitleAtMs, ClockMs + 6 AS LifecycleAtMs FROM (SELECT CAST(UNIX_TIMESTAMP(UTC_TIMESTAMP()) * 1000 AS UNSIGNED) AS ClockMs) AS clock", "expectStatus": 200, "assertions": [ { "path": "response.body.records[0].LeadSourceAtMs", "exists": true }, { "path": "response.body.records[0].LifecycleAtMs", "exists": true } ] }, { "id": "send-coalesced-propertychange-burst", "system": "connector_webhook", "method": "POST", "path": "/webhooks/hubspot-crm-connector/webhook", "body": [ { "portalId": 51280875, "appId": 43277960, "subscriptionId": 9720011, "eventId": "coalescing-lead-source-{{__runId}}", "occurredAt": "{{steps.capture-burst-clock.response.body.records[0].LeadSourceAtMs}}", "subscriptionType": "contact.propertyChange", "attemptNumber": 0, "objectId": "{{steps.create-contact.response.body.id}}", "propertyName": "lead_source_details", "propertyValue": "LinkedIn - InMail", "changeSource": "AUTOMATION_PLATFORM" }, { "portalId": 51280875, "appId": 43277960, "subscriptionId": 9720012, "eventId": "coalescing-recent-source-{{__runId}}", "occurredAt": "{{steps.capture-burst-clock.response.body.records[0].RecentSourceAtMs}}", "subscriptionType": "contact.propertyChange", "attemptNumber": 0, "objectId": "{{steps.create-contact.response.body.id}}", "propertyName": "recent_source", "propertyValue": "Cloud Sign-Up", "changeSource": "AUTOMATION_PLATFORM" }, { "portalId": 51280875, "appId": 43277960, "subscriptionId": 9720013, "eventId": "coalescing-recent-detail-{{__runId}}", "occurredAt": "{{steps.capture-burst-clock.response.body.records[0].RecentDetailAtMs}}", "subscriptionType": "contact.propertyChange", "attemptNumber": 0, "objectId": "{{steps.create-contact.response.body.id}}", "propertyName": "recent_source_details", "propertyValue": "Cloud Sign-Up", "changeSource": "AUTOMATION_PLATFORM" }, { "portalId": 51280875, "appId": 43277960, "subscriptionId": 9720014, "eventId": "coalescing-title-initial-{{__runId}}", "occurredAt": "{{steps.capture-burst-clock.response.body.records[0].InitialTitleAtMs}}", "subscriptionType": "contact.propertyChange", "attemptNumber": 0, "objectId": "{{steps.create-contact.response.body.id}}", "propertyName": "jobtitle", "propertyValue": "CoalescedInitial{{__runId}}", "changeSource": "API" }, { "portalId": 51280875, "appId": 43277960, "subscriptionId": 9720015, "eventId": "coalescing-title-clear-{{__runId}}", "occurredAt": "{{steps.capture-burst-clock.response.body.records[0].ClearedTitleAtMs}}", "subscriptionType": "contact.propertyChange", "attemptNumber": 0, "objectId": "{{steps.create-contact.response.body.id}}", "propertyName": "jobtitle", "propertyValue": "", "changeSource": "API" }, { "portalId": 51280875, "appId": 43277960, "subscriptionId": 9720016, "eventId": "coalescing-title-final-{{__runId}}", "occurredAt": "{{steps.capture-burst-clock.response.body.records[0].FinalTitleAtMs}}", "subscriptionType": "contact.propertyChange", "attemptNumber": 0, "objectId": "{{steps.create-contact.response.body.id}}", "propertyName": "jobtitle", "propertyValue": "CoalescedFinal{{__runId}}", "changeSource": "API" }, { "portalId": 51280875, "appId": 43277960, "subscriptionId": 9720017, "eventId": "coalescing-lifecycle-{{__runId}}", "occurredAt": "{{steps.capture-burst-clock.response.body.records[0].LifecycleAtMs}}", "subscriptionType": "contact.propertyChange", "attemptNumber": 0, "objectId": "{{steps.create-contact.response.body.id}}", "propertyName": "lifecyclestage", "propertyValue": "68053680", "changeSource": "API" } ], "expectStatus": 202, "assertions": [{ "path": "response.body", "exists": true }] }, { "id": "assert-coalesced-lead-state", "system": "nexuscrm", "authRole": "admin", "method": "GET", "path": "/services/data/v63.0/query?q=SELECT Id,FirstName,Title,Lead_Source_Details__c,Recent_Source__c,Recent_Source_Detail__c,Lifecycle_Stage__c,Disqualified_Date__c FROM Lead WHERE Id='{{steps.wait-linked-lead.response.body.records[0].Id}}' LIMIT 1", "expectStatus": 200, "poll": { "budgetMs": 300000, "intervalMs": 6000 }, "onTimeout": "fail", "assertions": [ { "path": "response.body.records[0].FirstName", "equals": "CoalSibling" }, { "path": "response.body.records[0].Title", "equals": "CoalescedFinal{{__runId}}" }, { "path": "response.body.records[0].Lead_Source_Details__c", "equals": "Linkedin - InMail" }, { "path": "response.body.records[0].Recent_Source__c", "equals": "Direct Traffic" }, { "path": "response.body.records[0].Recent_Source_Detail__c", "equals": "TiDB Cloud website" }, { "path": "response.body.records[0].Lifecycle_Stage__c", "equals": "68053680" }, { "path": "response.body.records[0].Disqualified_Date__c", "exists": true } ] }, { "id": "assert-burst-d3-terminal-shape", "system": "tidb", "method": "GET", "sql": "SELECT COUNT(DISTINCT E.Id) AS EventCount, COUNT(DISTINCT CASE WHEN NOT EXISTS (SELECT 1 FROM `_System_ConnectorWorkItem` O JOIN `_System_ConnectorEvent` S ON S.Id = O.EventId WHERE O.WorkType IN ('connector_event', 'workflow_action_apply') AND (O.EventId = E.Id OR (O.InstalledPackId = E.InstalledPackId AND NULLIF(COALESCE(O.ProjectionKey, ''), '') IS NOT NULL AND COALESCE(O.SourceLaneKey, '') = COALESCE(E.SourceLaneKey, '') AND S.InstalledPackId = E.InstalledPackId AND LOWER(TRIM(S.EventKind)) = LOWER(TRIM(E.EventKind)) AND COALESCE(S.RouteVersion, 0) = COALESCE(E.RouteVersion, 0) AND COALESCE(S.CompanionPackInstallId, '') = COALESCE(E.CompanionPackInstallId, '') AND COALESCE(S.CompanionFlowVersionId, '') = COALESCE(E.CompanionFlowVersionId, '') AND ((E.EnqueueOrdinal <> 0 AND S.EnqueueOrdinal <> 0 AND (E.EnqueueOrdinal < S.EnqueueOrdinal OR (E.EnqueueOrdinal = S.EnqueueOrdinal AND E.Id <= S.Id))) OR ((E.EnqueueOrdinal = 0 OR S.EnqueueOrdinal = 0) AND (E.CreatedDate < S.CreatedDate OR (E.CreatedDate = S.CreatedDate AND (E.DeliverySequence < S.DeliverySequence OR (E.DeliverySequence = S.DeliverySequence AND E.Id <= S.Id))))))))) THEN E.Id END) AS UncoveredEventCount, COUNT(DISTINCT CASE WHEN W.State NOT IN ('processed', 'skipped') THEN W.WorkId END) AS NonterminalOrDeadWorkCount, COUNT(DISTINCT CASE WHEN W.State = 'processed' AND NOT EXISTS (SELECT 1 FROM `_System_ConnectorRuntimeHandoffReceipt` R JOIN `_System_RuntimePlatformCommand` C ON BINARY C.CommandId = BINARY R.CommandId WHERE R.WorkId = W.WorkId AND R.EventId = W.EventId AND C.State IN ('COMPLETED', 'FAILED', 'POISONED')) THEN W.WorkId END) AS ProcessedWithoutTerminalCommandCount, COUNT(DISTINCT CASE WHEN W.State = 'processed' AND P.State = 'COMPLETED' THEN P.Id END) AS RuntimeCompletedCount, COUNT(DISTINCT CASE WHEN P.State IN ('FAILED', 'POISONED') THEN P.Id END) AS TerminalFailureCount FROM `_System_ConnectorEvent` E LEFT JOIN `_System_ConnectorWorkItem` W ON W.EventId = E.Id AND W.WorkType IN ('connector_event', 'workflow_action_apply') LEFT JOIN `_System_ConnectorRuntimeHandoffReceipt` H ON H.WorkId = W.WorkId AND H.EventId = E.Id LEFT JOIN `_System_RuntimePlatformCommand` P ON BINARY P.CommandId = BINARY H.CommandId WHERE E.Namespace = 'hubspot-crm-connector' AND E.EventKind = 'contact.propertychange' AND E.SourceObjectID = '{{steps.create-contact.response.body.id}}' AND COALESCE(JSON_UNQUOTE(JSON_EXTRACT(E.Payload, '$.payload.eventId')), JSON_UNQUOTE(JSON_EXTRACT(E.Payload, '$.eventId'))) IN ('coalescing-lead-source-{{__runId}}', 'coalescing-recent-source-{{__runId}}', 'coalescing-recent-detail-{{__runId}}', 'coalescing-title-initial-{{__runId}}', 'coalescing-title-clear-{{__runId}}', 'coalescing-title-final-{{__runId}}', 'coalescing-lifecycle-{{__runId}}')", "expectStatus": 200, "poll": { "budgetMs": 300000, "intervalMs": 6000 }, "onTimeout": "fail", "assertions": [ { "path": "response.body.records[0].EventCount", "numericEquals": 7 }, { "path": "response.body.records[0].UncoveredEventCount", "numericEquals": 0 }, { "path": "response.body.records[0].NonterminalOrDeadWorkCount", "numericEquals": 0 }, { "path": "response.body.records[0].ProcessedWithoutTerminalCommandCount", "numericEquals": 0 }, { "path": "response.body.records[0].RuntimeCompletedCount", "numericEquals": 1 }, { "path": "response.body.records[0].TerminalFailureCount", "numericEquals": 0 } ] }, { "id": "assert-lifecycle-source-time", "system": "tidb", "method": "GET", "sql": "SELECT ABS(UNIX_TIMESTAMP(Disqualified_Date__c) - FLOOR({{steps.capture-burst-clock.response.body.records[0].LifecycleAtMs}} / 1000)) AS SourceTimestampDeltaSeconds FROM `Lead` WHERE Id = '{{steps.wait-linked-lead.response.body.records[0].Id}}' LIMIT 1", "expectStatus": 200, "assertions": [{ "path": "response.body.records[0].SourceTimestampDeltaSeconds", "numericEquals": 0 }] }, { "id": "assert-coalesced-lead-state-settled", "system": "nexuscrm", "authRole": "admin", "method": "GET", "path": "/services/data/v63.0/query?q=SELECT Id,FirstName,Title,Lead_Source_Details__c,Recent_Source__c,Recent_Source_Detail__c,Lifecycle_Stage__c,Disqualified_Date__c FROM Lead WHERE Id='{{steps.wait-linked-lead.response.body.records[0].Id}}' LIMIT 1", "delayMs": 60000, "expectStatus": 200, "assertions": [ { "path": "response.body.records[0].FirstName", "equals": "CoalSibling" }, { "path": "response.body.records[0].Title", "equals": "CoalescedFinal{{__runId}}" }, { "path": "response.body.records[0].Lead_Source_Details__c", "equals": "Linkedin - InMail" }, { "path": "response.body.records[0].Recent_Source__c", "equals": "Direct Traffic" }, { "path": "response.body.records[0].Recent_Source_Detail__c", "equals": "TiDB Cloud website" }, { "path": "response.body.records[0].Lifecycle_Stage__c", "equals": "68053680" }, { "path": "response.body.records[0].Disqualified_Date__c", "exists": true } ] } ] }