Cron Workflows not being executed correctly

Community Support
1

We are occasionally seeing workflows that are configured to run with a Cron Schedule, skipping certain executions / not being executed at all.
Example below where the workflow is stuck in WorkflowExecutionStarted even after the first workflow task backoff is completed:


{
  "eventId": "1",
  "eventTime": "2025-05-30T06:47:45.692651115Z",
  "eventType": "WorkflowExecutionStarted",
  "version": "0",
  "taskId": "241887236",
  "workerMayIgnore": false,
  "workflowExecutionStartedEventAttributes": {
    "workflowType": {
      "name": "TestWorkflow"
    },
    "parentWorkflowNamespace": "",
    "parentWorkflowNamespaceId": "",
    "parentWorkflowExecution": null,
    "parentInitiatedEventId": "0",
    "taskQueue": {
      "name": "workspace-crud",
      "kind": "Normal",
      "normalName": ""
    },
    "input": {
      ...
    },
    "workflowExecutionTimeout": "0s",
    "workflowRunTimeout": "0s",
    "workflowTaskTimeout": "10s",
    "continuedExecutionRunId": "",
    "initiator": "CronSchedule",
    "continuedFailure": null,
    "lastCompletionResult": null,
    "originalExecutionRunId": "a0a91e76-b9d0-4b05-a89f-a7e306bbe6a6",
    "identity": "23@cluster-pod-545b7c88c6-zr5zv@",
    "firstExecutionRunId": "a0a91e76-b9d0-4b05-a89f-a7e306bbe6a6",
    "retryPolicy": {
      "initialInterval": "1s",
      "backoffCoefficient": 2,
      "maximumInterval": "60s",
      "maximumAttempts": 1,
      "nonRetryableErrorTypes": []
    },
    "attempt": 1,
    "workflowExecutionExpirationTime": null,
    "cronSchedule": "11,41 * * * *",
    "firstWorkflowTaskBackoff": "1395s",
    "memo": null,
    "searchAttributes": null,
    "prevAutoResetPoints": null,
    "header": {
      "fields": {}
    },
    "parentInitiatedEventVersion": "0",
    "workflowId": "test-workflow",
    "sourceVersionStamp": null
  },
  "name": "WorkflowExecutionStarted",
  "id": "1",
  "timestamp": "2025-05-30 UTC 06:47:45.69",
  "classification": "Started",
  "category": "workflow",
  "attributes": {
    "type": "workflowExecutionStartedEventAttributes",
    "workflowType": "TestWorkflow",
    "parentWorkflowNamespace": "",
    "parentWorkflowNamespaceId": "",
    "parentWorkflowExecution": null,
    "parentInitiatedEventId": "0",
    "taskQueue": {
      "name": "workspace-crud",
      "kind": "Normal",
      "normalName": ""
    },
    "input": {
        ...
    },
    "workflowExecutionTimeout": "",
    "workflowRunTimeout": "",
    "workflowTaskTimeout": "10 seconds",
    "continuedExecutionRunId": "",
    "initiator": "CronSchedule",
    "continuedFailure": null,
    "lastCompletionResult": null,
    "originalExecutionRunId": "a0a91e76-b9d0-4b05-a89f-a7e306bbe6a6",
    "identity": "23@cluster-pod-545b7c88c6-zr5zv@",
    "firstExecutionRunId": "a0a91e76-b9d0-4b05-a89f-a7e306bbe6a6",
    "retryPolicy": {
      "initialInterval": "1s",
      "backoffCoefficient": 2,
      "maximumInterval": "60s",
      "maximumAttempts": 1,
      "nonRetryableErrorTypes": []
    },
    "attempt": 1,
    "workflowExecutionExpirationTime": "",
    "cronSchedule": "11,41 * * * *",
    "firstWorkflowTaskBackoff": "23 minutes, 15 seconds",
    "memo": null,
    "searchAttributes": null,
    "prevAutoResetPoints": null,
    "header": {
      "fields": {}
    },
    "parentInitiatedEventVersion": "0",
    "workflowId": "test-workflow",
    "sourceVersionStamp": null
  }
}

We are currently on Temporal Server Version v1.22.4.
Rescheduling the workflow does not help.

2

even after the first workflow task backoff is completed:

Did first workflow task backoff timer for this execution ever fire? Hard to tell without knowing the “current time” to know if it fired or not from your message.
Do you have server metrics? Was your service and db stable at the that time?