{ "id": "nexuscrm-3873-history-sequence-lock-order", "title": "Mixed HubSpot property-change burst completes after transaction-wide history-sequence lock planning", "kind": "hubspot_sync", "executionMode": "serial", "executionModeReason": "The case owns a four-contact connector queue burst and exact Event/WorkItem assertions; parallel cases would contaminate the lane evidence and reduce the intended lock-order pressure.", "caseTimeoutMs": 1200000, "linkedIssues": [], "provenance": { "businessScenario": "补充hubspot integration test case防止回归", "reportedBy": "Amber", "capturedAt": "2026-07-22", "upstreamIssue": "https://github.com/pingcap-inc/NexusCRM/issues/3873", "upstreamFix": "https://github.com/pingcap-inc/NexusCRM/pull/3874", "calibrated": "Replays the production HubSpot Contact -> Lead Flow shape behind #3873: recent_source_details first updates Recent_Source_Detail__c, while hubspot_owner_id updates HubSpot_Owner_Id__c, and StampPostMappingLeadOwnerShadow later stamps the bound owner shadow in the same pessimistic transaction. The case first drains all four contacts' creation-time propertychange events and requires 60 seconds of source-scoped quiet, then resets the owner shadows before sending the target burst. The mixed four-lane burst puts the shared audit-v2 SystemModstamp/owner sequence rows under the opposite-order pressure that previously surfaced as TiDB Error 1213. PASS requires the exact four submitted Events to own processed WorkItems, no terminal history-sequence lock failure, and transition-sensitive Lead convergence. The deterministic lock barrier remains in the L0 TiDB integration test from #3874; this L2 case protects the real HubSpot route and metadata assembly." }, "steps": [ { "id": "find-bound-owner", "system": "nexuscrm", "authRole": "admin", "method": "GET", "path": "/services/data/v63.0/query?q=SELECT HubSpotOwnerId__c,CRMPrincipalId__c FROM HubSpotOwnerBinding__c WHERE BindingStatus__c='bound' AND CRMPrincipalType__c='User' AND HubSpotOwnerId__c='45282439' LIMIT 1", "expectStatus": 200, "poll": { "budgetMs": 30000, "intervalMs": 3000 }, "onTimeout": "blocked", "assertions": [ { "path": "response.body.records", "lengthAtLeast": 1 }, { "path": "response.body.records[0].HubSpotOwnerId__c", "equals": "45282439" }, { "path": "response.body.records[0].CRMPrincipalId__c", "exists": true } ] }, { "id": "create-four-hubspot-contacts", "system": "hubspot", "method": "POST", "path": "/crm/v3/objects/contacts/batch/create", "generate": { "count": 4, "into": "inputs", "itemTemplate": { "properties": { "email": "fix3873-{{__runId}}-{{i}}@example.com", "firstname": "HistorySequence{{i}}", "lastname": "LockOrder{{__runId}}", "lifecyclestage": "lead" } } }, "expectStatusRange": [200, 299], "assertions": [ { "path": "response.body.results", "lengthAtLeast": 4 } ] }, { "id": "wait-four-leads", "system": "tidb", "method": "GET", "sql": "SELECT I.SortKey, L.Id, L.HubSpot_Id__c FROM (SELECT 1 AS SortKey, '{{steps.create-four-hubspot-contacts.response.body.results[0].id}}' AS RemoteId UNION ALL SELECT 2, '{{steps.create-four-hubspot-contacts.response.body.results[1].id}}' UNION ALL SELECT 3, '{{steps.create-four-hubspot-contacts.response.body.results[2].id}}' UNION ALL SELECT 4, '{{steps.create-four-hubspot-contacts.response.body.results[3].id}}') I JOIN `Lead` L ON BINARY L.HubSpot_Id__c = BINARY I.RemoteId AND L.IsConverted = 0 AND L.IsDeleted = 0 ORDER BY I.SortKey", "expectStatus": 200, "poll": { "budgetMs": 240000, "intervalMs": 6000 }, "onTimeout": "blocked", "assertions": [ { "path": "response.body.records", "lengthAtLeast": 4 } ] }, { "id": "wait-creation-propertychanges-drained", "system": "tidb", "method": "GET", "sql": "SELECT CASE WHEN COUNT(DISTINCT E.SourceObjectID) = 4 AND MAX(E.CreatedDate) <= DATE_SUB(UTC_TIMESTAMP(), INTERVAL 60 SECOND) AND NOT EXISTS (SELECT 1 FROM `_System_ConnectorWorkItem` W JOIN `_System_ConnectorEvent` S ON S.Id = W.EventId WHERE S.Namespace = 'hubspot-crm-connector' AND S.EventKind = 'contact.propertychange' AND S.SourceObjectID IN ('{{steps.create-four-hubspot-contacts.response.body.results[0].id}}', '{{steps.create-four-hubspot-contacts.response.body.results[1].id}}', '{{steps.create-four-hubspot-contacts.response.body.results[2].id}}', '{{steps.create-four-hubspot-contacts.response.body.results[3].id}}') AND W.WorkType IN ('connector_event', 'workflow_action_apply') AND W.State NOT IN ('processed', 'skipped')) AND NOT EXISTS (SELECT 1 FROM `_System_ConnectorWorkItem` W JOIN `_System_ConnectorEvent` S ON S.Id = W.EventId WHERE S.Namespace = 'hubspot-crm-connector' AND S.EventKind = 'contact.propertychange' AND S.SourceObjectID IN ('{{steps.create-four-hubspot-contacts.response.body.results[0].id}}', '{{steps.create-four-hubspot-contacts.response.body.results[1].id}}', '{{steps.create-four-hubspot-contacts.response.body.results[2].id}}', '{{steps.create-four-hubspot-contacts.response.body.results[3].id}}') AND W.WorkType IN ('connector_event', 'workflow_action_apply') AND W.State = 'processed' AND NOT EXISTS (SELECT 1 FROM `_System_ConnectorRuntimeHandoffReceipt` R JOIN `_System_RuntimePlatformCommand` C ON C.CommandId = R.CommandId WHERE R.WorkId = W.WorkId AND R.EventId = W.EventId AND C.State IN ('COMPLETED', 'FAILED', 'POISONED'))) AND NOT EXISTS (SELECT 1 FROM `_System_ConnectorEvent` X WHERE X.Namespace = 'hubspot-crm-connector' AND X.EventKind = 'contact.propertychange' AND X.SourceObjectID IN ('{{steps.create-four-hubspot-contacts.response.body.results[0].id}}', '{{steps.create-four-hubspot-contacts.response.body.results[1].id}}', '{{steps.create-four-hubspot-contacts.response.body.results[2].id}}', '{{steps.create-four-hubspot-contacts.response.body.results[3].id}}') AND NOT EXISTS (SELECT 1 FROM `_System_ConnectorWorkItem` W JOIN `_System_ConnectorEvent` S ON S.Id = W.EventId WHERE W.WorkType IN ('connector_event', 'workflow_action_apply') AND (W.EventId = X.Id OR (W.InstalledPackId = X.InstalledPackId AND NULLIF(COALESCE(W.ProjectionKey, ''), '') IS NOT NULL AND COALESCE(W.SourceLaneKey, '') = COALESCE(X.SourceLaneKey, '') AND S.InstalledPackId = X.InstalledPackId AND LOWER(TRIM(S.EventKind)) = LOWER(TRIM(X.EventKind)) AND COALESCE(S.RouteVersion, 0) = COALESCE(X.RouteVersion, 0) AND COALESCE(S.CompanionPackInstallId, '') = COALESCE(X.CompanionPackInstallId, '') AND COALESCE(S.CompanionFlowVersionId, '') = COALESCE(X.CompanionFlowVersionId, '') AND ((X.EnqueueOrdinal <> 0 AND S.EnqueueOrdinal <> 0 AND (X.EnqueueOrdinal < S.EnqueueOrdinal OR (X.EnqueueOrdinal = S.EnqueueOrdinal AND X.Id <= S.Id))) OR ((X.EnqueueOrdinal = 0 OR S.EnqueueOrdinal = 0) AND (X.CreatedDate < S.CreatedDate OR (X.CreatedDate = S.CreatedDate AND (X.DeliverySequence < S.DeliverySequence OR (X.DeliverySequence = S.DeliverySequence AND X.Id <= S.Id)))))))))) THEN 1 ELSE 0 END AS Ready, COUNT(*) AS EventCount, COUNT(DISTINCT E.SourceObjectID) AS SourceCount, MAX(E.CreatedDate) AS LastEventCreatedAt FROM `_System_ConnectorEvent` E WHERE E.Namespace = 'hubspot-crm-connector' AND E.EventKind = 'contact.propertychange' AND E.SourceObjectID IN ('{{steps.create-four-hubspot-contacts.response.body.results[0].id}}', '{{steps.create-four-hubspot-contacts.response.body.results[1].id}}', '{{steps.create-four-hubspot-contacts.response.body.results[2].id}}', '{{steps.create-four-hubspot-contacts.response.body.results[3].id}}')", "expectStatus": 200, "poll": { "budgetMs": 300000, "intervalMs": 6000 }, "onTimeout": "blocked", "assertions": [ { "path": "response.body.records[0].Ready", "numericEquals": 1 }, { "path": "response.body.records[0].SourceCount", "numericEquals": 4 } ] }, { "id": "prepare-lead-1", "system": "nexuscrm", "authRole": "admin", "method": "PATCH", "path": "/services/data/v63.0/sobjects/Lead/{{steps.wait-four-leads.response.body.records[0].Id}}", "body": { "OwnerId": "{{steps.find-bound-owner.response.body.records[0].CRMPrincipalId__c}}", "HubSpot_Owner_Id__c": null, "HubSpot_Sync_Enabled__c": false }, "expectStatus": 204 }, { "id": "prepare-lead-2", "system": "nexuscrm", "authRole": "admin", "method": "PATCH", "path": "/services/data/v63.0/sobjects/Lead/{{steps.wait-four-leads.response.body.records[1].Id}}", "body": { "OwnerId": "{{steps.find-bound-owner.response.body.records[0].CRMPrincipalId__c}}", "HubSpot_Owner_Id__c": null, "HubSpot_Sync_Enabled__c": false }, "expectStatus": 204 }, { "id": "prepare-lead-3", "system": "nexuscrm", "authRole": "admin", "method": "PATCH", "path": "/services/data/v63.0/sobjects/Lead/{{steps.wait-four-leads.response.body.records[2].Id}}", "body": { "OwnerId": "{{steps.find-bound-owner.response.body.records[0].CRMPrincipalId__c}}", "HubSpot_Owner_Id__c": null, "HubSpot_Sync_Enabled__c": false }, "expectStatus": 204 }, { "id": "prepare-lead-4", "system": "nexuscrm", "authRole": "admin", "method": "PATCH", "path": "/services/data/v63.0/sobjects/Lead/{{steps.wait-four-leads.response.body.records[3].Id}}", "body": { "OwnerId": "{{steps.find-bound-owner.response.body.records[0].CRMPrincipalId__c}}", "HubSpot_Owner_Id__c": null, "HubSpot_Sync_Enabled__c": false }, "expectStatus": 204 }, { "id": "assert-precondition", "system": "tidb", "method": "GET", "sql": "SELECT COUNT(*) AS LeadCount, SUM(CASE WHEN L.OwnerId = '{{steps.find-bound-owner.response.body.records[0].CRMPrincipalId__c}}' THEN 1 ELSE 0 END) AS BoundOwnerCount, SUM(CASE WHEN L.HubSpot_Owner_Id__c IS NULL OR L.HubSpot_Owner_Id__c = '' THEN 1 ELSE 0 END) AS ClearedShadowCount, SUM(CASE WHEN L.HubSpot_Id__c IN ('{{steps.create-four-hubspot-contacts.response.body.results[1].id}}', '{{steps.create-four-hubspot-contacts.response.body.results[3].id}}') AND (L.HubSpot_Owner_Id__c IS NULL OR L.HubSpot_Owner_Id__c = '') THEN 1 ELSE 0 END) AS ClearedOwnerEventShadowCount FROM `Lead` L WHERE L.HubSpot_Id__c IN ('{{steps.create-four-hubspot-contacts.response.body.results[0].id}}', '{{steps.create-four-hubspot-contacts.response.body.results[1].id}}', '{{steps.create-four-hubspot-contacts.response.body.results[2].id}}', '{{steps.create-four-hubspot-contacts.response.body.results[3].id}}') AND L.IsConverted = 0 AND L.IsDeleted = 0", "expectStatus": 200, "assertions": [ { "path": "response.body.records[0].LeadCount", "numericEquals": 4 }, { "path": "response.body.records[0].BoundOwnerCount", "numericEquals": 4 }, { "path": "response.body.records[0].ClearedShadowCount", "numericEquals": 4 }, { "path": "response.body.records[0].ClearedOwnerEventShadowCount", "numericEquals": 2 } ] }, { "id": "capture-webhook-time", "system": "tidb", "method": "GET", "sql": "SELECT CAST(UNIX_TIMESTAMP(UTC_TIMESTAMP(3)) * 1000 AS UNSIGNED) AS OccurredAtMillis", "expectStatus": 200, "assertions": [ { "path": "response.body.records[0].OccurredAtMillis", "exists": true } ] }, { "id": "send-opposite-order-propertychange-burst", "system": "connector_webhook", "method": "POST", "path": "/webhooks/hubspot-crm-connector/webhook", "body": [ { "portalId": "{{portalId || \"51280875\"}}", "appId": 44166359, "subscriptionId": 6723583, "eventId": "fix-3873-recent-1-{{__runId}}", "occurredAt": "{{steps.capture-webhook-time.response.body.records[0].OccurredAtMillis}}", "subscriptionType": "contact.propertyChange", "attemptNumber": 0, "objectId": "{{steps.create-four-hubspot-contacts.response.body.results[0].id}}", "propertyName": "recent_source_details", "propertyValue": "LinkedIn - InMail", "changeSource": "AUTOMATION_PLATFORM" }, { "portalId": "{{portalId || \"51280875\"}}", "appId": 44166359, "subscriptionId": 6723583, "eventId": "fix-3873-owner-1-{{__runId}}", "occurredAt": "{{steps.capture-webhook-time.response.body.records[0].OccurredAtMillis}}", "subscriptionType": "contact.propertyChange", "attemptNumber": 0, "objectId": "{{steps.create-four-hubspot-contacts.response.body.results[1].id}}", "propertyName": "hubspot_owner_id", "propertyValue": "{{steps.find-bound-owner.response.body.records[0].HubSpotOwnerId__c}}", "changeSource": "AUTOMATION_PLATFORM" }, { "portalId": "{{portalId || \"51280875\"}}", "appId": 44166359, "subscriptionId": 6723583, "eventId": "fix-3873-recent-2-{{__runId}}", "occurredAt": "{{steps.capture-webhook-time.response.body.records[0].OccurredAtMillis}}", "subscriptionType": "contact.propertyChange", "attemptNumber": 0, "objectId": "{{steps.create-four-hubspot-contacts.response.body.results[2].id}}", "propertyName": "recent_source_details", "propertyValue": "Cloud Sign-Up", "changeSource": "AUTOMATION_PLATFORM" }, { "portalId": "{{portalId || \"51280875\"}}", "appId": 44166359, "subscriptionId": 6723583, "eventId": "fix-3873-owner-2-{{__runId}}", "occurredAt": "{{steps.capture-webhook-time.response.body.records[0].OccurredAtMillis}}", "subscriptionType": "contact.propertyChange", "attemptNumber": 0, "objectId": "{{steps.create-four-hubspot-contacts.response.body.results[3].id}}", "propertyName": "hubspot_owner_id", "propertyValue": "{{steps.find-bound-owner.response.body.records[0].HubSpotOwnerId__c}}", "changeSource": "AUTOMATION_PLATFORM" } ], "expectStatus": 202, "assertions": [ { "path": "response.body", "exists": true } ] }, { "id": "assert-terminal-workitems-no-history-lock-error", "system": "tidb", "method": "GET", "sql": "SELECT COUNT(DISTINCT E.Id) AS EventCount, COUNT(DISTINCT CASE WHEN W.State = 'processed' THEN E.Id END) AS ProcessedProjectionCount, COUNT(DISTINCT CASE WHEN W.State = 'dead' THEN E.Id END) AS DeadProjectionCount, COUNT(DISTINCT CASE WHEN COALESCE(E.LastError, '') LIKE '%reserve field history sequence%' OR COALESCE(E.LastError, '') LIKE '%field-history sequence lock order%' OR COALESCE(E.LastError, '') LIKE '%Error 1213%' OR COALESCE(E.LastError, '') LIKE '%Error 1205%' OR COALESCE(E.LastError, '') LIKE '%Error 1105%' OR COALESCE(W.FailureSummary, '') LIKE '%reserve field history sequence%' OR COALESCE(W.FailureSummary, '') LIKE '%field-history sequence lock order%' OR COALESCE(W.FailureSummary, '') LIKE '%Error 1213%' OR COALESCE(W.FailureSummary, '') LIKE '%Error 1205%' OR COALESCE(W.FailureSummary, '') LIKE '%Error 1105%' THEN E.Id END) AS HistorySequenceFailureCount FROM `_System_ConnectorEvent` E LEFT JOIN `_System_ConnectorWorkItem` W ON W.EventId = E.Id AND W.WorkType IN ('connector_event', 'workflow_action_apply') WHERE E.Namespace = 'hubspot-crm-connector' AND E.EventKind = 'contact.propertychange' AND E.SourceObjectID IN ('{{steps.create-four-hubspot-contacts.response.body.results[0].id}}', '{{steps.create-four-hubspot-contacts.response.body.results[1].id}}', '{{steps.create-four-hubspot-contacts.response.body.results[2].id}}', '{{steps.create-four-hubspot-contacts.response.body.results[3].id}}') AND COALESCE(JSON_UNQUOTE(JSON_EXTRACT(E.Payload, '$.payload.eventId')), JSON_UNQUOTE(JSON_EXTRACT(E.Payload, '$.eventId'))) IN ('fix-3873-recent-1-{{__runId}}', 'fix-3873-owner-1-{{__runId}}', 'fix-3873-recent-2-{{__runId}}', 'fix-3873-owner-2-{{__runId}}')", "expectStatus": 200, "poll": { "budgetMs": 240000, "intervalMs": 6000 }, "onTimeout": "fail", "assertions": [ { "path": "response.body.records[0].EventCount", "numericEquals": 4 }, { "path": "response.body.records[0].ProcessedProjectionCount", "numericEquals": 4 }, { "path": "response.body.records[0].DeadProjectionCount", "numericEquals": 0 }, { "path": "response.body.records[0].HistorySequenceFailureCount", "numericEquals": 0 } ] }, { "id": "assert-four-leads-converged", "system": "tidb", "method": "GET", "sql": "SELECT COUNT(*) AS LeadCount, SUM(CASE WHEN L.OwnerId = '{{steps.find-bound-owner.response.body.records[0].CRMPrincipalId__c}}' THEN 1 ELSE 0 END) AS BoundOwnerCount, SUM(CASE WHEN L.HubSpot_Owner_Id__c = '{{steps.find-bound-owner.response.body.records[0].HubSpotOwnerId__c}}' THEN 1 ELSE 0 END) AS BoundShadowCount, SUM(CASE WHEN L.HubSpot_Id__c IN ('{{steps.create-four-hubspot-contacts.response.body.results[1].id}}', '{{steps.create-four-hubspot-contacts.response.body.results[3].id}}') AND L.HubSpot_Owner_Id__c = '{{steps.find-bound-owner.response.body.records[0].HubSpotOwnerId__c}}' THEN 1 ELSE 0 END) AS OwnerEventShadowCount, SUM(CASE WHEN (L.HubSpot_Id__c = '{{steps.create-four-hubspot-contacts.response.body.results[0].id}}' AND L.Recent_Source_Detail__c = 'Linkedin - InMail') OR (L.HubSpot_Id__c = '{{steps.create-four-hubspot-contacts.response.body.results[2].id}}' AND L.Recent_Source_Detail__c = 'TiDB Cloud website') THEN 1 ELSE 0 END) AS RecentSourceDetailCount FROM `Lead` L WHERE L.HubSpot_Id__c IN ('{{steps.create-four-hubspot-contacts.response.body.results[0].id}}', '{{steps.create-four-hubspot-contacts.response.body.results[1].id}}', '{{steps.create-four-hubspot-contacts.response.body.results[2].id}}', '{{steps.create-four-hubspot-contacts.response.body.results[3].id}}') AND L.IsConverted = 0 AND L.IsDeleted = 0", "expectStatus": 200, "poll": { "budgetMs": 240000, "intervalMs": 6000 }, "onTimeout": "fail", "assertions": [ { "path": "response.body.records[0].LeadCount", "numericEquals": 4 }, { "path": "response.body.records[0].BoundOwnerCount", "numericEquals": 4 }, { "path": "response.body.records[0].BoundShadowCount", "numericEquals": 4 }, { "path": "response.body.records[0].OwnerEventShadowCount", "numericEquals": 2 }, { "path": "response.body.records[0].RecentSourceDetailCount", "numericEquals": 2 } ] } ] }