Skip to content

Commit 52aa9e8

Browse files
Add option to configure the deadline of getSystemInfo (#2240)
Add get system info timeout
1 parent 238c5e1 commit 52aa9e8

File tree

4 files changed

+222
-10
lines changed

4 files changed

+222
-10
lines changed

temporal-serviceclient/src/main/java/io/temporal/serviceclient/ChannelManager.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -346,7 +346,7 @@ public Supplier<Capabilities> getServerCapabilities() {
346346
SystemInfoInterceptor.getServerCapabilitiesWithRetryOrThrow(
347347
serverCapabilitiesFuture,
348348
interceptedChannel,
349-
deadlineFrom(options.getHealthCheckAttemptTimeout()));
349+
deadlineFrom(options.getSystemInfoTimeout()));
350350
}
351351

352352
private static Deadline deadlineFrom(Duration duration) {

temporal-serviceclient/src/main/java/io/temporal/serviceclient/ServiceStubsOptions.java

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,12 @@ public class ServiceStubsOptions {
7171
*/
7272
protected final Duration healthCheckAttemptTimeout;
7373

74+
/**
75+
* SystemInfoTimeout specifies how to long to wait for service response on each health check
76+
* attempt. Default: 5s.
77+
*/
78+
protected final Duration systemInfoTimeout;
79+
7480
/**
7581
* HealthCheckTimeout defines how long client should be sending health check requests to the
7682
* server before concluding that it is unavailable. Defaults to 10s.
@@ -128,6 +134,7 @@ public class ServiceStubsOptions {
128134
this.enableHttps = that.enableHttps;
129135
this.sslContext = that.sslContext;
130136
this.healthCheckAttemptTimeout = that.healthCheckAttemptTimeout;
137+
this.systemInfoTimeout = that.systemInfoTimeout;
131138
this.healthCheckTimeout = that.healthCheckTimeout;
132139
this.enableKeepAlive = that.enableKeepAlive;
133140
this.keepAliveTime = that.keepAliveTime;
@@ -150,6 +157,7 @@ public class ServiceStubsOptions {
150157
SslContext sslContext,
151158
Duration healthCheckAttemptTimeout,
152159
Duration healthCheckTimeout,
160+
Duration systemInfoTimeout,
153161
boolean enableKeepAlive,
154162
Duration keepAliveTime,
155163
Duration keepAliveTimeout,
@@ -168,6 +176,7 @@ public class ServiceStubsOptions {
168176
this.sslContext = sslContext;
169177
this.healthCheckAttemptTimeout = healthCheckAttemptTimeout;
170178
this.healthCheckTimeout = healthCheckTimeout;
179+
this.systemInfoTimeout = systemInfoTimeout;
171180
this.enableKeepAlive = enableKeepAlive;
172181
this.keepAliveTime = keepAliveTime;
173182
this.keepAliveTimeout = keepAliveTimeout;
@@ -233,6 +242,13 @@ public Duration getHealthCheckAttemptTimeout() {
233242
return healthCheckAttemptTimeout;
234243
}
235244

245+
/**
246+
* @return The timeout for the RPC made by the client to fetch server capabilities.
247+
*/
248+
public Duration getSystemInfoTimeout() {
249+
return systemInfoTimeout;
250+
}
251+
236252
/**
237253
* @return duration of time to wait while checking server connection when creating new client
238254
*/
@@ -337,6 +353,7 @@ public boolean equals(Object o) {
337353
&& Objects.equals(sslContext, that.sslContext)
338354
&& Objects.equals(healthCheckAttemptTimeout, that.healthCheckAttemptTimeout)
339355
&& Objects.equals(healthCheckTimeout, that.healthCheckTimeout)
356+
&& Objects.equals(systemInfoTimeout, that.systemInfoTimeout)
340357
&& Objects.equals(keepAliveTime, that.keepAliveTime)
341358
&& Objects.equals(keepAliveTimeout, that.keepAliveTimeout)
342359
&& Objects.equals(rpcTimeout, that.rpcTimeout)
@@ -358,6 +375,7 @@ public int hashCode() {
358375
sslContext,
359376
healthCheckAttemptTimeout,
360377
healthCheckTimeout,
378+
systemInfoTimeout,
361379
enableKeepAlive,
362380
keepAliveTime,
363381
keepAliveTimeout,
@@ -389,6 +407,8 @@ public String toString() {
389407
+ healthCheckAttemptTimeout
390408
+ ", healthCheckTimeout="
391409
+ healthCheckTimeout
410+
+ ", systemInfoTimeout="
411+
+ systemInfoTimeout
392412
+ ", enableKeepAlive="
393413
+ enableKeepAlive
394414
+ ", keepAliveTime="
@@ -421,6 +441,7 @@ public static class Builder<T extends Builder<T>> {
421441
private String target;
422442
private Consumer<ManagedChannelBuilder<?>> channelInitializer;
423443
private Duration healthCheckAttemptTimeout;
444+
private Duration systemInfoTimeout;
424445
private Duration healthCheckTimeout;
425446
private boolean enableKeepAlive = true;
426447
private Duration keepAliveTime = Duration.ofSeconds(30);
@@ -444,6 +465,7 @@ protected Builder(ServiceStubsOptions options) {
444465
this.sslContext = options.sslContext;
445466
this.healthCheckAttemptTimeout = options.healthCheckAttemptTimeout;
446467
this.healthCheckTimeout = options.healthCheckTimeout;
468+
this.systemInfoTimeout = options.systemInfoTimeout;
447469
this.enableKeepAlive = options.enableKeepAlive;
448470
this.keepAliveTime = options.keepAliveTime;
449471
this.keepAliveTimeout = options.keepAliveTimeout;
@@ -713,6 +735,17 @@ public T setHealthCheckTimeout(Duration healthCheckTimeout) {
713735
return self();
714736
}
715737

738+
/**
739+
* Set a SystemInfoTimeout that specifies how long the client tries to fetch server
740+
* capabilities.
741+
*
742+
* @return {@code this}
743+
*/
744+
public T setSystemInfoTimeout(Duration systemInfoTimeout) {
745+
this.systemInfoTimeout = systemInfoTimeout;
746+
return self();
747+
}
748+
716749
/**
717750
* Enables keep alive ping from client to the server, which can help drop abruptly closed
718751
* connections faster.
@@ -796,6 +829,7 @@ public ServiceStubsOptions build() {
796829
this.sslContext,
797830
this.healthCheckAttemptTimeout,
798831
this.healthCheckTimeout,
832+
this.systemInfoTimeout,
799833
this.enableKeepAlive,
800834
this.keepAliveTime,
801835
this.keepAliveTimeout,
@@ -847,6 +881,8 @@ public ServiceStubsOptions validateAndBuildWithDefaults() {
847881
Duration healthCheckTimeout =
848882
this.healthCheckTimeout != null ? this.healthCheckTimeout : Duration.ofSeconds(10);
849883

884+
Duration systemInfoTimeout =
885+
this.systemInfoTimeout != null ? this.systemInfoTimeout : Duration.ofSeconds(5);
850886
return new ServiceStubsOptions(
851887
this.channel,
852888
target,
@@ -855,6 +891,7 @@ public ServiceStubsOptions validateAndBuildWithDefaults() {
855891
this.sslContext,
856892
healthCheckAttemptTimeout,
857893
healthCheckTimeout,
894+
systemInfoTimeout,
858895
this.enableKeepAlive,
859896
this.keepAliveTime,
860897
this.keepAliveTimeout,

temporal-serviceclient/src/test/java/io/temporal/serviceclient/ChannelManagerTest.java

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -134,14 +134,14 @@ public void setUp() throws Exception {
134134
}
135135

136136
@After
137-
public void tearDown() throws Exception {
137+
public void tearDown() {
138138
if (channelManager != null) {
139139
channelManager.shutdownNow();
140140
}
141141
}
142142

143143
@Test
144-
public void testGetServerCapabilities() throws Exception {
144+
public void testGetServerCapabilities() {
145145
Capabilities capabilities = channelManager.getServerCapabilities().get();
146146
assertEquals(CAPABILITIES, capabilities);
147147
assertEquals(1, getSystemInfoCount.get());
@@ -150,7 +150,7 @@ public void testGetServerCapabilities() throws Exception {
150150
}
151151

152152
@Test
153-
public void testGetServerCapabilitiesRetry() throws Exception {
153+
public void testGetServerCapabilitiesRetry() {
154154
getSystemInfoUnavailable.set(2);
155155
Capabilities capabilities = channelManager.getServerCapabilities().get();
156156
assertEquals(CAPABILITIES, capabilities);
@@ -160,7 +160,7 @@ public void testGetServerCapabilitiesRetry() throws Exception {
160160
}
161161

162162
@Test
163-
public void testGetServerCapabilitiesUnavailable() throws Exception {
163+
public void testGetServerCapabilitiesUnavailable() {
164164
getSystemInfoUnavailable.set(Integer.MAX_VALUE);
165165
try {
166166
Capabilities unused = channelManager.getServerCapabilities().get();
@@ -174,7 +174,7 @@ public void testGetServerCapabilitiesUnavailable() throws Exception {
174174
}
175175

176176
@Test
177-
public void testGetServerCapabilitiesUnimplemented() throws Exception {
177+
public void testGetServerCapabilitiesUnimplemented() {
178178
getSystemInfoUnimplemented.set(1);
179179
Capabilities capabilities = channelManager.getServerCapabilities().get();
180180
assertEquals(Capabilities.getDefaultInstance(), capabilities);
@@ -184,7 +184,7 @@ public void testGetServerCapabilitiesUnimplemented() throws Exception {
184184
}
185185

186186
@Test
187-
public void testGetServerCapabilitiesWithConnect() throws Exception {
187+
public void testGetServerCapabilitiesWithConnect() {
188188
channelManager.connect(HEALTH_CHECK_NAME, Duration.ofMillis(100));
189189
Capabilities capabilities = channelManager.getServerCapabilities().get();
190190
assertEquals(CAPABILITIES, capabilities);
@@ -194,7 +194,7 @@ public void testGetServerCapabilitiesWithConnect() throws Exception {
194194
}
195195

196196
@Test
197-
public void testGetServerCapabilitiesRetryWithConnect() throws Exception {
197+
public void testGetServerCapabilitiesRetryWithConnect() {
198198
getSystemInfoUnavailable.set(2);
199199
channelManager.connect(HEALTH_CHECK_NAME, Duration.ofMillis(100));
200200
Capabilities capabilities = channelManager.getServerCapabilities().get();
@@ -205,7 +205,7 @@ public void testGetServerCapabilitiesRetryWithConnect() throws Exception {
205205
}
206206

207207
@Test
208-
public void testGetServerCapabilitiesUnavailableWithConnect() throws Exception {
208+
public void testGetServerCapabilitiesUnavailableWithConnect() {
209209
getSystemInfoUnavailable.set(Integer.MAX_VALUE);
210210
try {
211211
channelManager.connect(HEALTH_CHECK_NAME, Duration.ofMillis(100));
@@ -220,7 +220,7 @@ public void testGetServerCapabilitiesUnavailableWithConnect() throws Exception {
220220
}
221221

222222
@Test
223-
public void testGetServerCapabilitiesUnimplementedWithConnect() throws Exception {
223+
public void testGetServerCapabilitiesUnimplementedWithConnect() {
224224
getSystemInfoUnimplemented.set(1);
225225
channelManager.connect(HEALTH_CHECK_NAME, Duration.ofMillis(100));
226226
Capabilities capabilities = channelManager.getServerCapabilities().get();
Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
/*
2+
* Copyright (C) 2022 Temporal Technologies, Inc. All Rights Reserved.
3+
*
4+
* Copyright (C) 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
5+
*
6+
* Modifications copyright (C) 2017 Uber Technologies, Inc.
7+
*
8+
* Licensed under the Apache License, Version 2.0 (the "License");
9+
* you may not use this material except in compliance with the License.
10+
* You may obtain a copy of the License at
11+
*
12+
* http://www.apache.org/licenses/LICENSE-2.0
13+
*
14+
* Unless required by applicable law or agreed to in writing, software
15+
* distributed under the License is distributed on an "AS IS" BASIS,
16+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17+
* See the License for the specific language governing permissions and
18+
* limitations under the License.
19+
*/
20+
21+
package io.temporal.serviceclient;
22+
23+
import static org.junit.Assert.assertEquals;
24+
25+
import io.grpc.ClientInterceptor;
26+
import io.grpc.ManagedChannel;
27+
import io.grpc.Status;
28+
import io.grpc.StatusRuntimeException;
29+
import io.grpc.inprocess.InProcessChannelBuilder;
30+
import io.grpc.inprocess.InProcessServerBuilder;
31+
import io.grpc.stub.StreamObserver;
32+
import io.grpc.testing.GrpcCleanupRule;
33+
import io.temporal.api.workflowservice.v1.GetSystemInfoRequest;
34+
import io.temporal.api.workflowservice.v1.GetSystemInfoResponse;
35+
import io.temporal.api.workflowservice.v1.WorkflowServiceGrpc;
36+
import java.time.Duration;
37+
import java.util.*;
38+
import java.util.concurrent.ArrayBlockingQueue;
39+
import java.util.concurrent.atomic.AtomicInteger;
40+
import org.junit.*;
41+
42+
public class SystemInfoTimeoutTest {
43+
44+
private static final GetSystemInfoResponse.Capabilities CAPABILITIES =
45+
GetSystemInfoResponse.Capabilities.newBuilder().setInternalErrorDifferentiation(true).build();
46+
47+
private static final GetSystemInfoResponse GET_SYSTEM_INFO_RESPONSE =
48+
GetSystemInfoResponse.newBuilder().setCapabilities(CAPABILITIES).build();
49+
50+
private static final RpcRetryOptions RPC_RETRY_OPTIONS =
51+
RpcRetryOptions.newBuilder()
52+
.setInitialInterval(Duration.ofMillis(10))
53+
.setBackoffCoefficient(1.0)
54+
.setMaximumAttempts(3)
55+
.setExpiration(Duration.ofMillis(100))
56+
.validateBuildWithDefaults();
57+
58+
@Rule public final GrpcCleanupRule grpcCleanupRule = new GrpcCleanupRule();
59+
private final AtomicInteger getSystemInfoCount = new AtomicInteger(0);
60+
private final AbstractQueue<Duration> getSystemInfoTimeout = new ArrayBlockingQueue<Duration>(10);
61+
62+
private final WorkflowServiceGrpc.WorkflowServiceImplBase workflowImpl =
63+
new WorkflowServiceGrpc.WorkflowServiceImplBase() {
64+
@Override
65+
public void getSystemInfo(
66+
GetSystemInfoRequest request, StreamObserver<GetSystemInfoResponse> responseObserver) {
67+
Duration timeout = getSystemInfoTimeout.poll();
68+
if (timeout != null) {
69+
try {
70+
Thread.sleep(timeout.toMillis());
71+
} catch (InterruptedException e) {
72+
throw new RuntimeException(e);
73+
}
74+
}
75+
getSystemInfoCount.getAndIncrement();
76+
responseObserver.onNext(GET_SYSTEM_INFO_RESPONSE);
77+
responseObserver.onCompleted();
78+
}
79+
};
80+
81+
private ManagedChannel managedChannel;
82+
83+
@Before
84+
public void setUp() throws Exception {
85+
getSystemInfoCount.set(0);
86+
String serverName = InProcessServerBuilder.generateName();
87+
grpcCleanupRule.register(
88+
InProcessServerBuilder.forName(serverName)
89+
.directExecutor()
90+
.addService(workflowImpl)
91+
.build()
92+
.start());
93+
managedChannel =
94+
grpcCleanupRule.register(
95+
InProcessChannelBuilder.forName(serverName).directExecutor().build());
96+
}
97+
98+
@Test
99+
public void testGetServerCapabilitiesTimeoutExceeded() {
100+
WorkflowServiceStubsOptions serviceStubsOptions =
101+
WorkflowServiceStubsOptions.newBuilder()
102+
.setChannel(managedChannel)
103+
.setRpcRetryOptions(RPC_RETRY_OPTIONS)
104+
.setSystemInfoTimeout(Duration.ofSeconds(1))
105+
.validateAndBuildWithDefaults();
106+
107+
ClientInterceptor deadlineInterceptor =
108+
new GrpcDeadlineInterceptor(
109+
serviceStubsOptions.getRpcTimeout(),
110+
serviceStubsOptions.getRpcLongPollTimeout(),
111+
serviceStubsOptions.getRpcQueryTimeout());
112+
113+
ChannelManager channelManager =
114+
new ChannelManager(serviceStubsOptions, Collections.singletonList(deadlineInterceptor));
115+
116+
getSystemInfoTimeout.add(Duration.ofSeconds(2));
117+
118+
StatusRuntimeException sre =
119+
Assert.assertThrows(
120+
StatusRuntimeException.class, () -> channelManager.getServerCapabilities().get());
121+
assertEquals(Status.Code.DEADLINE_EXCEEDED, sre.getStatus().getCode());
122+
}
123+
124+
@Test
125+
public void testGetServerCapabilitiesRetry() {
126+
WorkflowServiceStubsOptions serviceStubsOptions =
127+
WorkflowServiceStubsOptions.newBuilder()
128+
.setChannel(managedChannel)
129+
.setRpcRetryOptions(RPC_RETRY_OPTIONS)
130+
.setRpcTimeout(Duration.ofMillis(500))
131+
.setSystemInfoTimeout(Duration.ofSeconds(5))
132+
.validateAndBuildWithDefaults();
133+
134+
ClientInterceptor deadlineInterceptor =
135+
new GrpcDeadlineInterceptor(
136+
serviceStubsOptions.getRpcTimeout(),
137+
serviceStubsOptions.getRpcLongPollTimeout(),
138+
serviceStubsOptions.getRpcQueryTimeout());
139+
140+
ChannelManager channelManager =
141+
new ChannelManager(serviceStubsOptions, Collections.singletonList(deadlineInterceptor));
142+
143+
getSystemInfoTimeout.add(Duration.ofSeconds(1));
144+
getSystemInfoTimeout.add(Duration.ofSeconds(1));
145+
146+
GetSystemInfoResponse.Capabilities capabilities = channelManager.getServerCapabilities().get();
147+
assertEquals(CAPABILITIES, capabilities);
148+
assertEquals(3, getSystemInfoCount.get());
149+
}
150+
151+
@Test
152+
public void testGetServerCapabilitiesTimeout() {
153+
WorkflowServiceStubsOptions serviceStubsOptions =
154+
WorkflowServiceStubsOptions.newBuilder()
155+
.setChannel(managedChannel)
156+
.setRpcRetryOptions(RPC_RETRY_OPTIONS)
157+
.setSystemInfoTimeout(Duration.ofSeconds(10))
158+
.validateAndBuildWithDefaults();
159+
160+
ClientInterceptor deadlineInterceptor =
161+
new GrpcDeadlineInterceptor(
162+
serviceStubsOptions.getRpcTimeout(),
163+
serviceStubsOptions.getRpcLongPollTimeout(),
164+
serviceStubsOptions.getRpcQueryTimeout());
165+
166+
ChannelManager channelManager =
167+
new ChannelManager(serviceStubsOptions, Collections.singletonList(deadlineInterceptor));
168+
169+
getSystemInfoTimeout.add(Duration.ofSeconds(6));
170+
171+
GetSystemInfoResponse.Capabilities capabilities = channelManager.getServerCapabilities().get();
172+
assertEquals(CAPABILITIES, capabilities);
173+
assertEquals(1, getSystemInfoCount.get());
174+
}
175+
}

0 commit comments

Comments
 (0)