Using the JS/TS API, I’ve got a worker going with a pretty large workflow, running some tasks in parallel. Somewhere in the middle of running, it seems to die. By ‘it’, I mean all of Temporal. The docker containers are shut down, and these are the logs I could see from the temporal container.
Is this an indication of too much data being sent back and forth in one activity step? Where can I find out exactly which activity step does this? I don’t see much for debugging where I could troubleshoot it.
Errors below.
temporal | {"level":"info","ts":"2023-09-13T20:13:38.120Z","msg":"none","service":"matching","component":"matching-engine","wf-task-queue-name":"13954@Machine.local-9ce46ad94dd248f5bfe92b4d9bbf3584","wf-task-queue-type":"Workflow","wf-namespace":"default","lifecycle":"Stopped","logging-call-at":"taskQueueManager.go:350"}
temporal | {"level":"info","ts":"2023-09-13T20:13:38.120Z","msg":"Taskqueue manager context is cancelled, shutting down","service":"matching","component":"matching-engine","wf-task-queue-name":"13954@Machine.local-9ce46ad94dd248f5bfe92b4d9bbf3584","wf-task-queue-type":"Workflow","wf-namespace":"default","logging-call-at":"taskReader.go:157"}
temporal | {"level":"warn","ts":"2023-09-13T20:13:41.801Z","msg":"Blob data size exceeds the warning limit.","service":"frontend","wf-namespace":"8bae17f0-2b81-47c9-b15f-f2bfa56337d2","wf-id":"foo-650217c6cba0e9442da6f7b8","wf-run-id":"8c048c50-04c4-4231-bd6e-08e5240b193d","wf-size":321793,"blob-size-violation-operation":"RespondActivityTaskCompleted","logging-call-at":"util.go:692"}
temporal | {"level":"warn","ts":"2023-09-13T20:13:43.423Z","msg":"Activity failure size exceeds warning limit for mutable state.","service":"history","shard-id":1,"address":"172.21.0.3:7234","wf-namespace":"default","wf-id":"foo-650217c6cba0e9442da6f7b8","wf-run-id":"8c048c50-04c4-4231-bd6e-08e5240b193d","blob-size":2153,"blob-size-violation-operation":"RetryActivity","logging-call-at":"mutable_state_impl.go:4290"}
temporal | {"level":"warn","ts":"2023-09-13T20:13:47.969Z","msg":"Blob data size exceeds the warning limit.","service":"frontend","wf-namespace":"8bae17f0-2b81-47c9-b15f-f2bfa56337d2","wf-id":"foo-650217c6cba0e9442da6f7b8","wf-run-id":"8c048c50-04c4-4231-bd6e-08e5240b193d","wf-size":312471,"blob-size-violation-operation":"RespondActivityTaskCompleted","logging-call-at":"util.go:692"}
temporal | {"level":"warn","ts":"2023-09-13T20:13:52.617Z","msg":"Blob data size exceeds the warning limit.","service":"frontend","wf-namespace":"8bae17f0-2b81-47c9-b15f-f2bfa56337d2","wf-id":"foo-650217c6cba0e9442da6f7b8","wf-run-id":"8c048c50-04c4-4231-bd6e-08e5240b193d","wf-size":312337,"blob-size-violation-operation":"RespondActivityTaskCompleted","logging-call-at":"util.go:692"}
temporal | {"level":"warn","ts":"2023-09-13T20:13:55.645Z","msg":"Blob data size exceeds the warning limit.","service":"frontend","wf-namespace":"8bae17f0-2b81-47c9-b15f-f2bfa56337d2","wf-id":"foo-650217c6cba0e9442da6f7b8","wf-run-id":"8c048c50-04c4-4231-bd6e-08e5240b193d","wf-size":312337,"blob-size-violation-operation":"RespondActivityTaskCompleted","logging-call-at":"util.go:692"}
temporal | {"level":"info","ts":"2023-09-13T20:13:57.483Z","msg":"none","service":"matching","component":"matching-engine","wf-task-queue-name":"/_sys/default-worker-tq/3","wf-task-queue-type":"Workflow","wf-namespace":"temporal-system","lifecycle":"Started","logging-call-at":"taskQueueManager.go:320"}
temporal | {"level":"warn","ts":"2023-09-13T20:13:59.128Z","msg":"Blob data size exceeds the warning limit.","service":"frontend","wf-namespace":"8bae17f0-2b81-47c9-b15f-f2bfa56337d2","wf-id":"foo-650217c6cba0e9442da6f7b8","wf-run-id":"8c048c50-04c4-4231-bd6e-08e5240b193d","wf-size":312337,"blob-size-violation-operation":"RespondActivityTaskCompleted","logging-call-at":"util.go:692"}
temporal | {"level":"info","ts":"2023-09-13T20:13:59.477Z","msg":"none","service":"matching","component":"matching-engine","wf-task-queue-name":"/_sys/default-worker-tq/1","wf-task-queue-type":"Workflow","wf-namespace":"temporal-system","lifecycle":"Stopped","logging-call-at":"taskQueueManager.go:350"}
temporal | {"level":"info","ts":"2023-09-13T20:13:59.477Z","msg":"Taskqueue manager context is cancelled, shutting down","service":"matching","component":"matching-engine","wf-task-queue-name":"/_sys/default-worker-tq/1","wf-task-queue-type":"Workflow","wf-namespace":"temporal-system","logging-call-at":"taskReader.go:157"}
temporal | {"level":"warn","ts":"2023-09-13T20:14:01.087Z","msg":"Blob data size exceeds the warning limit.","service":"frontend","wf-namespace":"8bae17f0-2b81-47c9-b15f-f2bfa56337d2","wf-id":"foo-650217c6cba0e9442da6f7b8","wf-run-id":"8c048c50-04c4-4231-bd6e-08e5240b193d","wf-size":312337,"blob-size-violation-operation":"RespondActivityTaskCompleted","logging-call-at":"util.go:692"}
temporal | {"level":"info","ts":"2023-09-13T20:14:12.238Z","msg":"Taskqueue manager context is cancelled, shutting down","service":"matching","component":"matching-engine","wf-task-queue-name":"14223@Machine.local-446114db3aa04465a6071105d5d06d8e","wf-task-queue-type":"Workflow","wf-namespace":"default","logging-call-at":"taskReader.go:157"}
temporal | {"level":"info","ts":"2023-09-13T20:14:12.238Z","msg":"none","service":"matching","component":"matching-engine","wf-task-queue-name":"14223@Machine.local-446114db3aa04465a6071105d5d06d8e","wf-task-queue-type":"Workflow","wf-namespace":"default","lifecycle":"Stopped","logging-call-at":"taskQueueManager.go:350"}
temporal | {"level":"info","ts":"2023-09-13T20:14:40.106Z","msg":"none","service":"matching","component":"matching-engine","wf-task-queue-name":"14460@Machine.local-300792dd4b4b442c9c606fd488716b5e","wf-task-queue-type":"Workflow","wf-namespace":"default","lifecycle":"Stopped","logging-call-at":"taskQueueManager.go:350"}
temporal | {"level":"info","ts":"2023-09-13T20:14:40.106Z","msg":"Taskqueue manager context is cancelled, shutting down","service":"matching","component":"matching-engine","wf-task-queue-name":"14460@Machine.local-300792dd4b4b442c9c606fd488716b5e","wf-task-queue-type":"Workflow","wf-namespace":"default","logging-call-at":"taskReader.go:157"}
temporal | {"level":"info","ts":"2023-09-13T20:14:49.131Z","msg":"none","service":"matching","component":"matching-engine","wf-task-queue-name":"14654@Machine.local-9f6b18d0d8b745c993dbac46dda3861c","wf-task-queue-type":"Workflow","wf-namespace":"default","lifecycle":"Stopped","logging-call-at":"taskQueueManager.go:350"}
temporal | {"level":"info","ts":"2023-09-13T20:14:49.131Z","msg":"Taskqueue manager context is cancelled, shutting down","service":"matching","component":"matching-engine","wf-task-queue-name":"14654@Machine.local-9f6b18d0d8b745c993dbac46dda3861c","wf-task-queue-type":"Workflow","wf-namespace":"default","logging-call-at":"taskReader.go:157"}
temporal | {"level":"info","ts":"2023-09-13T20:14:56.481Z","msg":"none","service":"matching","component":"matching-engine","wf-task-queue-name":"/_sys/default-worker-tq/1","wf-task-queue-type":"Workflow","wf-namespace":"temporal-system","lifecycle":"Started","logging-call-at":"taskQueueManager.go:320"}