We are getting the following, anyinuts please
level=“ERROR” logger=“io.temporal.internal.worker.Poller” throwable="io.grpc.StatusRuntimeException: INTERNAL: Panic! This is a bug!
at io.grpc.stub.ClientCalls.toStatusRuntimeException(ClientCalls.java:262)
at io.grpc.stub.ClientCalls.getUnchecked(ClientCalls.java:243)
at io.grpc.stub.ClientCalls.blockingUnaryCall(ClientCalls.java:156)
at io.temporal.api.workflowservice.v1.WorkflowServiceGrpc$WorkflowServiceBlockingStub.pollWorkflowTaskQueue(WorkflowServiceGrpc.java:2639)
at io.temporal.internal.worker.WorkflowPollTask.poll(WorkflowPollTask.java:81)
at io.temporal.internal.worker.WorkflowPollTask.poll(WorkflowPollTask.java:37)
at io.temporal.internal.worker.Poller$PollExecutionTask.run(Poller.java:265)
at io.temporal.internal.worker.Poller$PollLoopTask.run(Poller.java:241)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base/java.lang.Thread.run(Thread.java:832)
Caused by: java.lang.OutOfMemoryError: unable to create native thread: possibly out of memory or process/resource limits reached
at java.base/java.lang.Thread.start0(Native Method)
at java.base/java.lang.Thread.start(Thread.java:801)
at java.base/java.util.concurrent.ThreadPoolExecutor.addWorker(ThreadPoolExecutor.java:939)
at java.base/java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1356)
at io.grpc.internal.ManagedChannelImpl$3.execute(ManagedChannelImpl.java:646)
at io.grpc.internal.DnsNameResolver.resolve(DnsNameResolver.java:389)
at io.grpc.internal.DnsNameResolver.start(DnsNameResolver.java:206)
at io.grpc.internal.ManagedChannelImpl.exitIdleMode(ManagedChannelImpl.java:411)
at io.grpc.internal.ManagedChannelImpl.enterIdleMode(ManagedChannelImpl.java:426)
at io.grpc.internal.ManagedChannelImpl.access$900(ManagedChannelImpl.java:118)
at io.grpc.internal.ManagedChannelImpl$1PrepareToLoseNetworkRunnable.run(ManagedChannelImpl.java:1326)
at io.grpc.SynchronizationContext.drain(SynchronizationContext.java:95)
at io.grpc.SynchronizationContext.execute(SynchronizationContext.java:127)
at io.grpc.internal.ManagedChannelImpl.enterIdle(ManagedChannelImpl.java:1330)
at io.grpc.internal.ForwardingManagedChannel.enterIdle(ForwardingManagedChannel.java:88)
at io.temporal.serviceclient.WorkflowServiceStubsImpl.lambda$enterGrpcIdleChannelStateTask$0(WorkflowServiceStubsImpl.java:189)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305)
Make sure that you create a single instance of WorkflowServiceStubs and WorkflowClient and the per process. I’ve seen cases when they were created per request.