Hello! We are getting “DEADLINE_EXCEEDED: deadline exceeded after ~70s” error when testing a long-running workflow using in-memory Temporal (TestWorkflowExtension).
This happens if there is long running activity or if a workflow is executing several short activities but as long as total workflow running time exceeds 70 seconds, this exception gets thrown.
The issue does NOT happen with a locally running Temporal (tests configured via setUseExternalService(true) and .setTarget(“127.0.0.1:7233”)). It only happens with in-memory Temporal instance.
Below is stack trace and code of the repro (the actual code does not use sleeps, of course, it makes API calls, data transformation, etc.)
Stack:
io.grpc.StatusRuntimeException: DEADLINE_EXCEEDED: deadline exceeded after 69.999644732s. []
at io.grpc.stub.ClientCalls.toStatusRuntimeException(ClientCalls.java:262)
at io.grpc.stub.ClientCalls.getUnchecked(ClientCalls.java:243)
at io.grpc.stub.ClientCalls.blockingUnaryCall(ClientCalls.java:156)
at io.temporal.api.workflowservice.v1.WorkflowServiceGrpc$WorkflowServiceBlockingStub.getWorkflowExecutionHistory(WorkflowServiceGrpc.java:2625)
at io.temporal.internal.common.WorkflowExecutionUtils.lambda$getInstanceCloseEvent$1(WorkflowExecutionUtils.java:257)
at io.temporal.internal.common.GrpcRetryer.retryWithResult(GrpcRetryer.java:97)
at io.temporal.internal.common.WorkflowExecutionUtils.getInstanceCloseEvent(WorkflowExecutionUtils.java:245)
at io.temporal.internal.common.WorkflowExecutionUtils.getWorkflowExecutionResult(WorkflowExecutionUtils.java:133)
at io.temporal.internal.client.RootWorkflowClientInvoker.getResult(RootWorkflowClientInvoker.java:94)
at io.temporal.internal.sync.WorkflowStubImpl.getResult(WorkflowStubImpl.java:243)
at io.temporal.internal.sync.WorkflowStubImpl.getResult(WorkflowStubImpl.java:225)
at io.temporal.testing.TestWorkflowEnvironmentInternal$TimeLockingInterceptor$TimeLockingWorkflowStub.getResult(TestWorkflowEnvironmentInternal.java:295)
at io.temporal.internal.sync.WorkflowInvocationHandler$SyncWorkflowInvocationHandler.startWorkflow(WorkflowInvocationHandler.java:315)
at io.temporal.internal.sync.WorkflowInvocationHandler$SyncWorkflowInvocationHandler.invoke(WorkflowInvocationHandler.java:270)
at io.temporal.internal.sync.WorkflowInvocationHandler.invoke(WorkflowInvocationHandler.java:178)
at jdk.proxy3/jdk.proxy3.$Proxy202.run(Unknown Source)
Code
@WorkflowInterface
public interface LongRunningWorkflow {
@WorkflowMethod
void run();
}
public class LongRunningWorkflowImpl implements LongRunningWorkflow {
private final ActivityOptions activityOptions = ActivityOptions.newBuilder()
.setScheduleToCloseTimeout(Duration.ofHours(1))
.build();
private final SleepActivities activities = Workflow.newActivityStub(SleepActivities.class, activityOptions);
@Override
public void run() {
activities.sleep(Duration.ofSeconds(30));
activities.sleep(Duration.ofSeconds(30));
activities.sleep(Duration.ofSeconds(30));
activities.sleep(Duration.ofSeconds(30));
}
}
@ActivityInterface
public interface SleepActivities {
@ActivityMethod
boolean sleep(Duration duration);
}
public class SleepActivitiesImpl implements SleepActivities {
private static final Logger logger = LoggerFactory.getLogger(SleepActivitiesImpl.class);
@Override
public boolean sleep(Duration duration) {
try {
logger.info("Sleeping for {} sec", duration.toSeconds());
Thread.sleep(duration.toMillis());
} catch (InterruptedException e) {
e.printStackTrace();
}
return true;
}
}
@ExtendWith(MockitoExtension.class)
class LongRunningWorkflowImplTest {
private static final SleepActivitiesImpl sleepActivities = new SleepActivitiesImpl();
@RegisterExtension
public static final TestWorkflowExtension testWorkflowExtension =
TestWorkflowExtension.newBuilder()
.setWorkflowClientOptions(TemporalConfiguration.getWorkflowClientOptions("default"))
.setWorkflowTypes(LongRunningWorkflowImpl.class)
.setActivityImplementations(sleepActivities)
.build();
@Test
void test_run(LongRunningWorkflow workflow) {
workflow.run();
}
}