@maxim Some of the workflow executions are throwing below exception.
java.lang.IllegalStateException:
No command scheduled that corresponds to event_id: 12
event_time {
seconds: 1634056687
nanos: 384584718
}
event_type: EVENT_TYPE_ACTIVITY_TASK_SCHEDULED
task_id: 1490026512
activity_task_scheduled_event_attributes {
activity_id: "a3395dd2-0918-3892-a45a-b802027866d8"
activity_type {
name: "InsertTradeReconRecord"
}
task_queue {
name: "INVEST_CRYPTO_RECON_WORKFLOW_TASK_QUEUE"
kind: TASK_QUEUE_KIND_NORMAL
}
header {
}
input {
payloads {
metadata {
key: "encoding"
value: "json/plain"
}
data: "\"3b60eccf-8959-4d9f-bcc0-b4cb918514cd\""
}
payloads {
metadata {
key: "encoding"
value: "json/plain"
}
data: "\"Crypto_Trade_Recon_Order-12602202\""
}
payloads {
metadata {
key: "encoding"
value: "json/plain"
}
data: "\"IN_PROGRESS\""
}
payloads {
metadata {
key: "encoding"
value: "json/plain"
}
data: "12602202"
}
}
schedule_to_close_timeout {
}
schedule_to_start_timeout {
}
start_to_close_timeout {
seconds: 300
}
heartbeat_timeout {
}
workflow_task_completed_event_id: 4
retry_policy {
initial_interval {
seconds: 60
}
backoff_coefficient: 1.0
maximum_interval {
seconds: 60
}
maximum_attempts: 10
}
}
(Most recent call first)
at io.temporal.internal.statemachines.WorkflowStateMachines.handleCommandEvent (WorkflowStateMachines.java:244)
at io.temporal.internal.statemachines.WorkflowStateMachines.handleEventImpl (WorkflowStateMachines.java:199)
at io.temporal.internal.statemachines.WorkflowStateMachines.handleEvent (WorkflowStateMachines.java:178)
at io.temporal.internal.replay.ReplayWorkflowRunTaskHandler.handleEvent (ReplayWorkflowRunTaskHandler.java:140)
at io.temporal.internal.replay.ReplayWorkflowRunTaskHandler.handleWorkflowTaskImpl (ReplayWorkflowRunTaskHandler.java:180)
at io.temporal.internal.replay.ReplayWorkflowRunTaskHandler.handleWorkflowTask (ReplayWorkflowRunTaskHandler.java:150)
at io.temporal.internal.replay.ReplayWorkflowTaskHandler.handleWorkflowTaskWithEmbeddedQuery (ReplayWorkflowTaskHandler.java:201)
at io.temporal.internal.replay.ReplayWorkflowTaskHandler.handleWorkflowTask (ReplayWorkflowTaskHandler.java:114)
at io.temporal.internal.worker.WorkflowWorker$TaskHandlerImpl.handle (WorkflowWorker.java:319)
at io.temporal.internal.worker.WorkflowWorker$TaskHandlerImpl.handle (WorkflowWorker.java:279)
at io.temporal.internal.worker.PollTaskExecutor.lambda$process$0 (PollTaskExecutor.java:73)
at datadog.trace.bootstrap.instrumentation.java.concurrent.Wrapper.run (Wrapper.java:25)
at java.util.concurrent.ThreadPoolExecutor.runWorker (ThreadPoolExecutor.java:1128)
at java.util.concurrent.ThreadPoolExecutor$Worker.run (ThreadPoolExecutor.java:628)
at java.lang.Thread.run (Thread.java:829)
Looks like an activity configured to publish datadog metrics is the cause of the failure but I am not sure whats missing in this activity. Also this is a new activity which is added to an existing workflow.
metricsActivity = Workflow.newLocalActivityStub(MetricsActivity.class, LocalActivityOptions.newBuilder()
.setScheduleToCloseTimeout(Duration.ofSeconds(5))
.build());