{ "id": "webhook-retry-receipt-idempotency", "title": "HubSpot retry attempts create distinct receipts but one Event and one CRM effect", "kind": "hubspot_sync", "optional": false, "executionMode": "serial", "executionModeReason": "The case drains the exact source lane, then counts two exact signed deliveries, one deduplicated Event, one projection-aware terminal Runtime command, and one CRM effect; unrelated webhook traffic must not overlap the correlation window.", "caseTimeoutMs": 960000, "linkedIssues": [5488], "provenance": { "businessScenario": "HubSpot retries the same provider event with attemptNumber incremented and a newly signed raw body. Attempt 0 and attempt 1 must each receive a distinct durable verified ingress receipt, while Event identity intentionally excludes attemptNumber so the business event and CRM mutation execute only once.", "reportedBy": "Historical fix review of nexuscrm-app-packs PR #551", "capturedAt": "2026-08-26", "upstreamFix": "https://github.com/pingcap-inc/nexuscrm-app-packs/pull/551", "calibrated": "Both requests use the ordinary HubSpot v1 signed webhook path. Before injection the case requires the contact.propertychange source lane to be quiet and terminal. Receipt correlation follows EnqueueMetadata.deliveryId to the one exact Event; execution accepts only a direct or source-projection-covering WorkItem joined through ConnectorRuntimeHandoffReceipt to one COMPLETED Runtime PlatformCommand by CommandId. The CRM state and one active Lead are the business-effect authority; the capped FlowAuditLog is not used." }, "steps": [ { "id": "create-contact", "system": "hubspot", "method": "POST", "path": "/crm/v3/objects/contacts", "body": { "properties": { "email": "retry-receipt-{{__runId}}@example.com", "firstname": "RetryReceipt", "lastname": "Before{{__runId}}", "jobtitle": "BeforeRetry" } }, "expectStatusRange": [200, 299], "assertions": [{ "path": "response.body.id", "exists": true }] }, { "id": "wait-linked-lead", "system": "nexuscrm", "authRole": "admin", "method": "GET", "path": "/services/data/v63.0/query?q=SELECT Id,HubSpot_Id__c FROM Lead WHERE HubSpot_Id__c='{{steps.create-contact.response.body.id}}' AND IsConverted=false LIMIT 1", "expectStatus": 200, "poll": { "budgetMs": 240000, "intervalMs": 6000 }, "onTimeout": "blocked", "assertions": [ { "path": "response.body.records[0].Id", "exists": true }, { "path": "response.body.records[0].HubSpot_Id__c", "equals": "{{steps.create-contact.response.body.id}}" } ] }, { "id": "wait-creation-fanout-drained", "system": "tidb", "method": "GET", "sql": "SELECT CASE WHEN COUNT(*) > 0 AND MAX(E.CreatedDate) <= DATE_SUB(UTC_TIMESTAMP(), INTERVAL 60 SECOND) AND NOT EXISTS (SELECT 1 FROM `_System_ConnectorWorkItem` W JOIN `_System_ConnectorEvent` S ON S.Id = W.EventId WHERE S.Namespace = 'hubspot-crm-connector' AND S.EventKind = 'contact.propertychange' AND S.SourceObjectID = '{{steps.create-contact.response.body.id}}' AND W.WorkType IN ('connector_event', 'workflow_action_apply') AND W.State NOT IN ('processed', 'skipped')) AND NOT EXISTS (SELECT 1 FROM `_System_ConnectorWorkItem` W JOIN `_System_ConnectorEvent` S ON S.Id = W.EventId WHERE S.Namespace = 'hubspot-crm-connector' AND S.EventKind = 'contact.propertychange' AND S.SourceObjectID = '{{steps.create-contact.response.body.id}}' AND W.WorkType IN ('connector_event', 'workflow_action_apply') AND W.State = 'processed' AND NOT EXISTS (SELECT 1 FROM `_System_ConnectorRuntimeHandoffReceipt` R JOIN `_System_RuntimePlatformCommand` C ON C.CommandId = R.CommandId WHERE R.WorkId = W.WorkId AND R.EventId = W.EventId AND C.State IN ('COMPLETED', 'FAILED', 'POISONED'))) AND NOT EXISTS (SELECT 1 FROM `_System_ConnectorEvent` X WHERE X.Namespace = 'hubspot-crm-connector' AND X.EventKind = 'contact.propertychange' AND X.SourceObjectID = '{{steps.create-contact.response.body.id}}' AND NOT EXISTS (SELECT 1 FROM `_System_ConnectorWorkItem` W JOIN `_System_ConnectorEvent` S ON S.Id = W.EventId WHERE W.WorkType IN ('connector_event', 'workflow_action_apply') AND (W.EventId = X.Id OR (W.InstalledPackId = X.InstalledPackId AND NULLIF(COALESCE(W.ProjectionKey, ''), '') IS NOT NULL AND COALESCE(W.SourceLaneKey, '') = COALESCE(X.SourceLaneKey, '') AND S.InstalledPackId = X.InstalledPackId AND LOWER(TRIM(S.EventKind)) = LOWER(TRIM(X.EventKind)) AND COALESCE(S.RouteVersion, 0) = COALESCE(X.RouteVersion, 0) AND COALESCE(S.CompanionPackInstallId, '') = COALESCE(X.CompanionPackInstallId, '') AND COALESCE(S.CompanionFlowVersionId, '') = COALESCE(X.CompanionFlowVersionId, '') AND ((X.EnqueueOrdinal <> 0 AND S.EnqueueOrdinal <> 0 AND (X.EnqueueOrdinal < S.EnqueueOrdinal OR (X.EnqueueOrdinal = S.EnqueueOrdinal AND X.Id <= S.Id))) OR ((X.EnqueueOrdinal = 0 OR S.EnqueueOrdinal = 0) AND (X.CreatedDate < S.CreatedDate OR (X.CreatedDate = S.CreatedDate AND (X.DeliverySequence < S.DeliverySequence OR (X.DeliverySequence = S.DeliverySequence AND X.Id <= S.Id)))))))))) THEN 1 ELSE 0 END AS Ready, COUNT(*) AS EventCount, MAX(E.CreatedDate) AS LastEventCreatedAt FROM `_System_ConnectorEvent` E WHERE E.Namespace = 'hubspot-crm-connector' AND E.EventKind = 'contact.propertychange' AND E.SourceObjectID = '{{steps.create-contact.response.body.id}}'", "expectStatus": 200, "poll": { "budgetMs": 240000, "intervalMs": 6000 }, "onTimeout": "blocked", "assertions": [ { "path": "response.body.records[0].Ready", "numericEquals": 1 } ] }, { "id": "capture-retry-clock", "system": "tidb", "method": "GET", "sql": "SELECT CAST(UNIX_TIMESTAMP(UTC_TIMESTAMP(3)) * 1000 AS UNSIGNED) AS OccurredAtMillis", "expectStatus": 200, "assertions": [{ "path": "response.body.records[0].OccurredAtMillis", "exists": true }] }, { "id": "send-attempt-zero", "system": "connector_webhook", "method": "POST", "path": "/webhooks/hubspot-crm-connector/webhook", "body": [ { "portalId": 51280875, "appId": 44166359, "subscriptionId": 6723583, "eventId": "retry-receipt-{{__runId}}", "occurredAt": "{{steps.capture-retry-clock.response.body.records[0].OccurredAtMillis}}", "subscriptionType": "contact.propertyChange", "attemptNumber": 0, "objectId": "{{steps.create-contact.response.body.id}}", "propertyName": "jobtitle", "propertyValue": "RetryApplied{{__runId}}", "changeSource": "AUTOMATION_PLATFORM" } ], "expectStatus": 202, "assertions": [{ "path": "response.body", "exists": true }] }, { "id": "send-attempt-one", "system": "connector_webhook", "method": "POST", "path": "/webhooks/hubspot-crm-connector/webhook", "body": [ { "portalId": 51280875, "appId": 44166359, "subscriptionId": 6723583, "eventId": "retry-receipt-{{__runId}}", "occurredAt": "{{steps.capture-retry-clock.response.body.records[0].OccurredAtMillis}}", "subscriptionType": "contact.propertyChange", "attemptNumber": 1, "objectId": "{{steps.create-contact.response.body.id}}", "propertyName": "jobtitle", "propertyValue": "RetryApplied{{__runId}}", "changeSource": "AUTOMATION_PLATFORM" } ], "expectStatus": 202, "assertions": [{ "path": "response.body", "exists": true }] }, { "id": "assert-one-lead-effect", "system": "nexuscrm", "authRole": "admin", "method": "GET", "path": "/services/data/v63.0/query?q=SELECT Id,Title FROM Lead WHERE Id='{{steps.wait-linked-lead.response.body.records[0].Id}}' LIMIT 1", "expectStatus": 200, "poll": { "budgetMs": 300000, "intervalMs": 6000 }, "onTimeout": "fail", "assertions": [ { "path": "response.body.records", "lengthAtLeast": 1 }, { "path": "response.body.records[0].Title", "equals": "RetryApplied{{__runId}}" } ] }, { "id": "assert-two-receipts-one-event", "system": "tidb", "method": "GET", "sql": "SELECT COUNT(DISTINCT R.Id) AS ReceiptCount, COUNT(DISTINCT R.ReceiptKey) AS DistinctReceiptKeyCount, COUNT(DISTINCT R.BodyHash) AS DistinctBodyHashCount, COUNT(DISTINCT CASE WHEN R.Status = 'enqueued' THEN R.Id END) AS EnqueuedReceiptCount, COUNT(DISTINCT E.Id) AS EventCount FROM `_System_ConnectorEvent` E JOIN `_System_ConnectorIngressReceipt` R ON JSON_UNQUOTE(JSON_EXTRACT(R.EnqueueMetadata, '$.deliveryId')) = E.DeliveryId WHERE E.Namespace = 'hubspot-crm-connector' AND E.EventKind = 'contact.propertychange' AND E.SourceObjectID = '{{steps.create-contact.response.body.id}}' AND COALESCE(JSON_UNQUOTE(JSON_EXTRACT(E.Payload, '$.payload.eventId')), JSON_UNQUOTE(JSON_EXTRACT(E.Payload, '$.eventId'))) = 'retry-receipt-{{__runId}}'", "expectStatus": 200, "poll": { "budgetMs": 300000, "intervalMs": 6000 }, "onTimeout": "fail", "assertions": [ { "path": "response.body.records[0].ReceiptCount", "numericEquals": 2 }, { "path": "response.body.records[0].DistinctReceiptKeyCount", "numericEquals": 2 }, { "path": "response.body.records[0].DistinctBodyHashCount", "numericEquals": 2 }, { "path": "response.body.records[0].EnqueuedReceiptCount", "numericEquals": 2 }, { "path": "response.body.records[0].EventCount", "numericEquals": 1 } ] }, { "id": "assert-retry-event-projection-owner", "system": "tidb", "method": "GET", "sql": "SELECT E.Id AS EventId, W.WorkId, W.EventId AS WorkItemEventId, W.Generation AS WorkItemGeneration, W.State AS WorkItemState, W.LaneOrdinal AS WorkItemLaneOrdinal, S.Id AS SelectedEventId, S.EnqueueOrdinal AS SelectedEventOrdinal FROM `_System_ConnectorEvent` E JOIN `_System_ConnectorWorkItem` W ON W.InstalledPackId = E.InstalledPackId AND W.WorkType IN ('connector_event', 'workflow_action_apply') AND (W.EventId = E.Id OR (NULLIF(COALESCE(W.ProjectionKey, ''), '') IS NOT NULL AND COALESCE(W.SourceLaneKey, '') = COALESCE(E.SourceLaneKey, ''))) JOIN `_System_ConnectorEvent` S ON S.Id = W.EventId WHERE E.Namespace = 'hubspot-crm-connector' AND E.EventKind = 'contact.propertychange' AND E.SourceObjectID = '{{steps.create-contact.response.body.id}}' AND COALESCE(JSON_UNQUOTE(JSON_EXTRACT(E.Payload, '$.payload.eventId')), JSON_UNQUOTE(JSON_EXTRACT(E.Payload, '$.eventId'))) = 'retry-receipt-{{__runId}}' AND (W.EventId = E.Id OR (S.InstalledPackId = E.InstalledPackId AND LOWER(TRIM(S.EventKind)) = LOWER(TRIM(E.EventKind)) AND COALESCE(S.RouteVersion, 0) = COALESCE(E.RouteVersion, 0) AND COALESCE(S.CompanionPackInstallId, '') = COALESCE(E.CompanionPackInstallId, '') AND COALESCE(S.CompanionFlowVersionId, '') = COALESCE(E.CompanionFlowVersionId, '') AND ((E.EnqueueOrdinal <> 0 AND S.EnqueueOrdinal <> 0 AND (E.EnqueueOrdinal < S.EnqueueOrdinal OR (E.EnqueueOrdinal = S.EnqueueOrdinal AND E.Id <= S.Id))) OR ((E.EnqueueOrdinal = 0 OR S.EnqueueOrdinal = 0) AND (E.CreatedDate < S.CreatedDate OR (E.CreatedDate = S.CreatedDate AND (E.DeliverySequence < S.DeliverySequence OR (E.DeliverySequence = S.DeliverySequence AND E.Id <= S.Id)))))))) ORDER BY W.LaneOrdinal DESC, W.Generation DESC LIMIT 1", "expectStatus": 200, "poll": { "budgetMs": 240000, "intervalMs": 6000 }, "onTimeout": "fail", "assertions": [ { "path": "response.body.records", "lengthAtLeast": 1 }, { "path": "response.body.records[0].EventId", "exists": true }, { "path": "response.body.records[0].WorkId", "exists": true }, { "path": "response.body.records[0].WorkItemEventId", "exists": true }, { "path": "response.body.records[0].WorkItemGeneration", "exists": true }, { "path": "response.body.records[0].WorkItemState", "equals": "processed" }, { "path": "response.body.records[0].WorkItemLaneOrdinal", "exists": true }, { "path": "response.body.records[0].SelectedEventId", "exists": true }, { "path": "response.body.records[0].SelectedEventOrdinal", "exists": true } ] }, { "id": "assert-one-runtime-completion", "system": "tidb", "method": "GET", "sql": "SELECT COUNT(DISTINCT CASE WHEN P.State = 'COMPLETED' THEN P.Id END) AS RuntimeCompletedCount, COUNT(DISTINCT CASE WHEN P.State IN ('FAILED', 'POISONED') THEN P.Id END) AS TerminalFailureCount, COUNT(DISTINCT CASE WHEN P.State IN ('ACCEPTED', 'PROCESSING', 'BLOCKED') THEN P.Id END) AS NonterminalRuntimeCount FROM `_System_ConnectorRuntimeHandoffReceipt` H JOIN `_System_RuntimePlatformCommand` P ON BINARY P.CommandId = BINARY H.CommandId WHERE H.WorkId = '{{steps.assert-retry-event-projection-owner.response.body.records[0].WorkId}}' AND H.EventId = '{{steps.assert-retry-event-projection-owner.response.body.records[0].WorkItemEventId}}'", "expectStatus": 200, "poll": { "budgetMs": 240000, "intervalMs": 6000 }, "onTimeout": "fail", "assertions": [ { "path": "response.body.records[0].RuntimeCompletedCount", "numericEquals": 1 }, { "path": "response.body.records[0].TerminalFailureCount", "numericEquals": 0 }, { "path": "response.body.records[0].NonterminalRuntimeCount", "numericEquals": 0 } ] }, { "id": "assert-one-active-lead", "system": "tidb", "method": "GET", "sql": "SELECT COUNT(*) AS ActiveLeadCount FROM `Lead` WHERE HubSpot_Id__c = '{{steps.create-contact.response.body.id}}' AND IsConverted = 0 AND IsDeleted = 0", "expectStatus": 200, "assertions": [{ "path": "response.body.records[0].ActiveLeadCount", "numericEquals": 1 }] } ] }