Skip to content

Commit 8fa0ea8

Browse files
GaneshSPatilGaneshSPatil
committed
Recycle kubernetes client on periodic basis
Use go.kubernetes.elastic-agent.plugin.client.recycle.interval.in.minutes system property to specify Client recycle interval
1 parent c0a2504 commit 8fa0ea8

File tree

3 files changed

+212
-9
lines changed

3 files changed

+212
-9
lines changed

‎src/main/java/cd/go/contrib/elasticagent/KubernetesAgentInstances.java

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import org.joda.time.DateTime;
2626
import org.joda.time.Period;
2727

28+
import java.net.SocketTimeoutException;
2829
import java.util.ArrayList;
2930
import java.util.Date;
3031
import java.util.Map;
@@ -146,8 +147,20 @@ public Agents instancesCreatedAfterTimeout(PluginSettings settings, Agents agent
146147
@Override
147148
public void refreshAll(PluginSettings properties) {
148149
LOG.debug("[Refresh Instances] Syncing k8s elastic agent pod information for cluster {}.", properties);
149-
KubernetesClient client = factory.client(properties);
150-
PodList list = client.pods().list();
150+
PodList list = null;
151+
try {
152+
KubernetesClient client = factory.client(properties);
153+
list = client.pods().list();
154+
} catch (Exception e) {
155+
LOG.error("Error occurred while trying to list kubernetes pods:", e);
156+
157+
if (e.getCause() instanceof SocketTimeoutException) {
158+
LOG.error("Error caused due to SocketTimeoutException. This generally happens due to stale kubernetes client. Clearing out existing kubernetes client and creating a new one!");
159+
factory.clearOutExistingClient();
160+
KubernetesClient client = factory.client(properties);
161+
list = client.pods().list();
162+
}
163+
}
151164

152165
instances.clear();
153166
for (Pod pod : list.getItems()) {

‎src/main/java/cd/go/contrib/elasticagent/KubernetesClientFactory.java

Lines changed: 62 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,32 +19,60 @@
1919
import io.fabric8.kubernetes.client.ConfigBuilder;
2020
import io.fabric8.kubernetes.client.DefaultKubernetesClient;
2121
import io.fabric8.kubernetes.client.KubernetesClient;
22+
import org.apache.commons.lang3.StringUtils;
23+
24+
import java.util.concurrent.TimeUnit;
2225

2326
import static cd.go.contrib.elasticagent.KubernetesPlugin.LOG;
2427
import static java.text.MessageFormat.format;
2528

2629
public class KubernetesClientFactory {
2730
private static final KubernetesClientFactory KUBERNETES_CLIENT_FACTORY = new KubernetesClientFactory();
31+
private final Clock clock;
2832
private KubernetesClient client;
29-
private PluginSettings pluginSettings;
33+
private PluginSettings clusterProfileConfigurations;
34+
private long clientCreatedTime;
35+
private long kubernetesClientRecycleIntervalInMinutes = -1;
36+
public static final String CLIENT_RECYCLE_SYSTEM_PROPERTY_KEY = "go.kubernetes.elastic-agent.plugin.client.recycle.interval.in.minutes";
37+
38+
public KubernetesClientFactory() {
39+
this.clock = Clock.DEFAULT;
40+
}
41+
42+
//used for testing..
43+
public KubernetesClientFactory(Clock clock) {
44+
this.clock = clock;
45+
}
3046

3147
public static KubernetesClientFactory instance() {
3248
return KUBERNETES_CLIENT_FACTORY;
3349
}
3450

35-
public synchronized KubernetesClient client(PluginSettings pluginSettings) {
36-
if (pluginSettings.equals(this.pluginSettings) && this.client != null) {
51+
public synchronized KubernetesClient client(PluginSettings clusterProfileConfigurations) {
52+
clearOutClientOnTimer();
53+
if (clusterProfileConfigurations.equals(this.clusterProfileConfigurations) && this.client != null) {
3754
LOG.debug("Using previously created client.");
3855
return this.client;
3956
}
4057

41-
LOG.debug(format("Creating a new client because {0}.", (client == null) ? "client is null" : "plugin setting is changed"));
42-
this.pluginSettings = pluginSettings;
43-
this.client = createClientFor(pluginSettings);
58+
LOG.debug(format("Creating a new client because {0}.", (client == null) ? "client is null" : "cluster profile configurations has changed"));
59+
this.clusterProfileConfigurations = clusterProfileConfigurations;
60+
this.client = createClientFor(clusterProfileConfigurations);
61+
this.clientCreatedTime = this.clock.now().getMillis();
4462
LOG.debug("New client is created.");
63+
4564
return this.client;
4665
}
4766

67+
private void clearOutClientOnTimer() {
68+
long currentTime = this.clock.now().getMillis();
69+
long differenceInMinutes = TimeUnit.MILLISECONDS.toMinutes(currentTime - this.clientCreatedTime);
70+
if (differenceInMinutes > getKubernetesClientRecycleInterval()) {
71+
LOG.info("Recycling kubernetes client on timer...");
72+
clearOutExistingClient();
73+
}
74+
}
75+
4876
private KubernetesClient createClientFor(PluginSettings pluginSettings) {
4977
final ConfigBuilder configBuilder = new ConfigBuilder()
5078
.withOauthToken(pluginSettings.getSecurityToken())
@@ -56,6 +84,33 @@ private KubernetesClient createClientFor(PluginSettings pluginSettings) {
5684
}
5785

5886
public void clearOutExistingClient() {
59-
this.client = null;
87+
if (this.client != null) {
88+
LOG.debug("Terminating existing kubernetes client...");
89+
this.client.close();
90+
this.client = null;
91+
}
92+
}
93+
94+
private long getKubernetesClientRecycleInterval() {
95+
//if the value is already read, send it..
96+
if (this.kubernetesClientRecycleIntervalInMinutes != -1) {
97+
return this.kubernetesClientRecycleIntervalInMinutes;
98+
}
99+
100+
String property = System.getProperty(CLIENT_RECYCLE_SYSTEM_PROPERTY_KEY);
101+
if (StringUtils.isBlank(property)) {
102+
//set default to 10 minutes when system property is not specified
103+
this.kubernetesClientRecycleIntervalInMinutes = 10;
104+
return this.kubernetesClientRecycleIntervalInMinutes;
105+
}
106+
107+
try {
108+
this.kubernetesClientRecycleIntervalInMinutes = Integer.valueOf(property);
109+
} catch (Exception e) {
110+
//set default value to 10 minutes when parsing user input fails
111+
this.kubernetesClientRecycleIntervalInMinutes = 10;
112+
}
113+
114+
return this.kubernetesClientRecycleIntervalInMinutes;
60115
}
61116
}

‎src/test/java/cd/go/contrib/elasticagent/KubernetesClientFactoryTest.java

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
/*
2+
* Copyright 2019 ThoughtWorks, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package cd.go.contrib.elasticagent;
18+
19+
import com.google.gson.Gson;
20+
import io.fabric8.kubernetes.client.KubernetesClient;
21+
import org.joda.time.DateTime;
22+
import org.junit.Before;
23+
import org.junit.Test;
24+
25+
import java.util.HashMap;
26+
import java.util.Map;
27+
28+
import static cd.go.contrib.elasticagent.KubernetesClientFactory.CLIENT_RECYCLE_SYSTEM_PROPERTY_KEY;
29+
import static org.junit.Assert.assertEquals;
30+
import static org.junit.Assert.assertNotEquals;
31+
32+
public class KubernetesClientFactoryTest {
33+
private PluginSettings pluginSettings;
34+
private KubernetesClientFactory factory;
35+
private Clock.TestClock clock;
36+
37+
@Before
38+
public void setUp() throws Exception {
39+
System.clearProperty(CLIENT_RECYCLE_SYSTEM_PROPERTY_KEY);
40+
final Map<String, Object> pluginSettingsMap = new HashMap<>();
41+
pluginSettingsMap.put("go_server_url", "https://foo.go.cd/go");
42+
pluginSettingsMap.put("auto_register_timeout", "13");
43+
pluginSettingsMap.put("pending_pods_count", "14");
44+
pluginSettingsMap.put("kubernetes_cluster_url", "https://cloud.example.com");
45+
pluginSettingsMap.put("security_token", "foo-token");
46+
pluginSettingsMap.put("namespace", "gocd");
47+
48+
clock = new Clock.TestClock();
49+
factory = new KubernetesClientFactory(clock);
50+
pluginSettings = PluginSettings.fromJSON(new Gson().toJson(pluginSettingsMap));
51+
}
52+
53+
@Test
54+
public void shouldInitializeClient() {
55+
KubernetesClient client = factory.client(pluginSettings);
56+
}
57+
58+
@Test
59+
public void shouldReuseTheExistingClientIfNotTimeElapsed() {
60+
KubernetesClient client = factory.client(pluginSettings);
61+
62+
clock.set(new DateTime().plusMinutes(1));
63+
KubernetesClient client2 = factory.client(pluginSettings);
64+
assertEquals(client, client2);
65+
66+
clock.set(new DateTime().plusMinutes(2));
67+
KubernetesClient client3 = factory.client(pluginSettings);
68+
assertEquals(client, client3);
69+
70+
clock.set(new DateTime().plusMinutes(5));
71+
KubernetesClient client4 = factory.client(pluginSettings);
72+
assertEquals(client, client4);
73+
74+
clock.set(new DateTime().plusMinutes(9));
75+
KubernetesClient client5 = factory.client(pluginSettings);
76+
assertEquals(client, client5);
77+
}
78+
79+
@Test
80+
public void shouldRecycleClientOnTimer() {
81+
KubernetesClient client = factory.client(pluginSettings);
82+
83+
clock.set(new DateTime().plusMinutes(9));
84+
85+
KubernetesClient client2 = factory.client(pluginSettings);
86+
assertEquals(client, client2);
87+
88+
clock.set(new DateTime().plusMinutes(11));
89+
90+
KubernetesClient clientAfterTimeElapse = factory.client(pluginSettings);
91+
assertNotEquals(client, clientAfterTimeElapse);
92+
}
93+
94+
@Test
95+
public void shouldReadClientRecycleIntervalFromSystemProperty() {
96+
System.setProperty(CLIENT_RECYCLE_SYSTEM_PROPERTY_KEY, "2");
97+
98+
KubernetesClient client = factory.client(pluginSettings);
99+
100+
clock.set(new DateTime().plusMinutes(1));
101+
KubernetesClient client2 = factory.client(pluginSettings);
102+
assertEquals(client, client2);
103+
104+
clock.set(new DateTime().plusMinutes(3));
105+
KubernetesClient client3 = factory.client(pluginSettings);
106+
assertNotEquals(client, client3);
107+
}
108+
109+
@Test
110+
public void shouldSetClientRecycleIntervalToDefaultValueWhenInvalidValueForSystemPropertyIsProvided() {
111+
System.setProperty(CLIENT_RECYCLE_SYSTEM_PROPERTY_KEY, "two");
112+
113+
KubernetesClient client = factory.client(pluginSettings);
114+
115+
clock.set(new DateTime().plusMinutes(1));
116+
KubernetesClient client2 = factory.client(pluginSettings);
117+
assertEquals(client, client2);
118+
119+
clock.set(new DateTime().plusMinutes(9));
120+
KubernetesClient client3 = factory.client(pluginSettings);
121+
assertEquals(client, client3);
122+
123+
clock.set(new DateTime().plusMinutes(11));
124+
KubernetesClient client4 = factory.client(pluginSettings);
125+
assertNotEquals(client, client4);
126+
}
127+
128+
@Test
129+
public void shouldAllowExplicitlyClearingClient() {
130+
KubernetesClient client = factory.client(pluginSettings);
131+
factory.clearOutExistingClient();
132+
KubernetesClient client2 = factory.client(pluginSettings);
133+
assertNotEquals(client, client2);
134+
}
135+
}

0 commit comments

Comments
 (0)