Skip to content
This repository was archived by the owner on Sep 16, 2024. It is now read-only.

Commit 621964d

Browse files
rjrudinrjrudin
committed
#425: Can now create replicas for a database with primaries on one host
Also deprecated "setCreateForestsOnOneHost"
1 parent f52d89b commit 621964d

File tree

8 files changed

+127
-33
lines changed

8 files changed

+127
-33
lines changed

‎src/main/java/com/marklogic/appdeployer/command/databases/DeployDatabaseCommand.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ public class DeployDatabaseCommand extends AbstractCommand implements UndoableCo
6767
* Passed on to DeployForestsCommand. If forests are to be created, controls whether forests on created on every
6868
* host or only one one host.
6969
*/
70+
@Deprecated
7071
private boolean createForestsOnEachHost = true;
7172

7273
private int undoSortOrder;
@@ -399,10 +400,16 @@ public void setDatabaseName(String databaseName) {
399400
this.databaseName = databaseName;
400401
}
401402

403+
@Deprecated
402404
public boolean isCreateForestsOnEachHost() {
403405
return createForestsOnEachHost;
404406
}
405407

408+
/**
409+
* Use appConfig.setDatabasesWithForestsOnOneHost
410+
* @param createForestsOnEachHost
411+
*/
412+
@Deprecated
406413
public void setCreateForestsOnEachHost(boolean createForestsOnEachHost) {
407414
this.createForestsOnEachHost = createForestsOnEachHost;
408415
}

‎src/main/java/com/marklogic/appdeployer/command/forests/DeployForestsCommand.java

Lines changed: 51 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,10 @@ public class DeployForestsCommand extends AbstractCommand {
4444
private String databaseName;
4545
private String forestFilename;
4646
private String forestPayload;
47+
48+
@Deprecated
4749
private boolean createForestsOnEachHost = true;
50+
4851
private HostCalculator hostCalculator;
4952

5053
private ForestBuilder forestBuilder = new ForestBuilder();
@@ -101,7 +104,7 @@ protected void createForestsViaForestEndpoint(CommandContext context, List<Fores
101104

102105
/**
103106
* Public so that it can be reused without actually saving any of the forests.
104-
*
107+
* <p>
105108
* This also facilitates the creation of forests for many databases at one time. A client can call this on a set of
106109
* these commands to construct a list of many forests that can be created via CMA in one request.
107110
*
@@ -113,19 +116,20 @@ protected void createForestsViaForestEndpoint(CommandContext context, List<Fores
113116
public List<Forest> buildForests(CommandContext context, boolean includeReplicas) {
114117
String template = buildForestTemplate(context, new ForestManager(context.getManageClient()));
115118

116-
List<String> hostNames = new HostManager(context.getManageClient()).getHostNames();
117-
hostNames = determineHostNamesForForest(context, hostNames);
119+
final List<String> allHostNames = new HostManager(context.getManageClient()).getHostNames();
120+
ForestHostNames forestHostNames = determineHostNamesForForest(context, allHostNames);
118121

119122
// Need to know what primary forests exist already in case more need to be added, or a new host has been added
120123
List<Forest> forests = getExistingPrimaryForests(context, this.databaseName);
121-
ForestPlan forestPlan = new ForestPlan(this.databaseName, hostNames)
124+
ForestPlan forestPlan = new ForestPlan(this.databaseName, forestHostNames.getPrimaryForestHostNames())
125+
.withReplicaHostNames(forestHostNames.getReplicaForestHostNames())
122126
.withTemplate(template)
123127
.withForestsPerDataDirectory(this.forestsPerHost)
124128
.withExistingForests(forests);
125129

126130
if (includeReplicas) {
127131
Map<String, Integer> map = context.getAppConfig().getDatabaseNamesAndReplicaCounts();
128-
if (map != null) {
132+
if (map != null && map.containsKey(this.databaseName)) {
129133
int count = map.get(this.databaseName);
130134
if (count > 0) {
131135
forestPlan.withReplicaCount(count);
@@ -178,26 +182,32 @@ protected String buildForestTemplate(CommandContext context, ForestManager fores
178182

179183
/**
180184
* @param context
181-
* @param hostNames
182-
* @return
185+
* @param allHostNames
186+
* @return a ForestHostNames instance that defines the list of host names that can be used for primary forests and
187+
* that can be used for replica forests. As of 4.1.0, the only reason these will differ is when a database is
188+
* configured to only have forests on one host, or when the deprecated setCreateForestsOnOneHost method is used.
183189
*/
184-
protected List<String> determineHostNamesForForest(CommandContext context, List<String> hostNames) {
190+
protected ForestHostNames determineHostNamesForForest(CommandContext context, final List<String> allHostNames) {
185191
Set<String> databaseNames = context.getAppConfig().getDatabasesWithForestsOnOneHost();
186192
boolean onlyOnOneHost = databaseNames != null && databaseNames.contains(this.databaseName);
187193

188-
if (!createForestsOnEachHost || onlyOnOneHost) {
189-
String first = hostNames.get(0);
190-
logger.info(format("Only creating forests on the first host: " + first));
191-
hostNames = new ArrayList<>();
192-
hostNames.add(first);
193-
return hostNames;
194+
if (hostCalculator == null) {
195+
hostCalculator = new DefaultHostCalculator(new DefaultHostNameProvider(context.getManageClient()));
194196
}
195197

196-
if (hostCalculator == null) {
197-
return new DefaultHostCalculator(new DefaultHostNameProvider(context.getManageClient())).calculateHostNames(this.databaseName, context);
198+
// "candidate" = hosts that can be used for primary and replica forests
199+
List<String> candidateHostNames = hostCalculator.calculateHostNames(this.databaseName, context);
200+
List<String> primaryForestHostNames = new ArrayList<>();
201+
202+
if (!createForestsOnEachHost || onlyOnOneHost) {
203+
String first = allHostNames.get(0);
204+
logger.info(format("Only creating forests on the first host: " + first));
205+
primaryForestHostNames.add(first);
206+
} else {
207+
primaryForestHostNames.addAll(candidateHostNames);
198208
}
199209

200-
return hostCalculator.calculateHostNames(this.databaseName, context);
210+
return new ForestHostNames(primaryForestHostNames, candidateHostNames);
201211
}
202212

203213
public int getForestsPerHost() {
@@ -228,10 +238,16 @@ public void setForestPayload(String forestPayload) {
228238
this.forestPayload = forestPayload;
229239
}
230240

241+
@Deprecated
231242
public boolean isCreateForestsOnEachHost() {
232243
return createForestsOnEachHost;
233244
}
234245

246+
/**
247+
* Use appConfig.setDatabasesWithForestsOnOneHost
248+
* @param createForestsOnEachHost
249+
*/
250+
@Deprecated
235251
public void setCreateForestsOnEachHost(boolean createForestsOnEachHost) {
236252
this.createForestsOnEachHost = createForestsOnEachHost;
237253
}
@@ -253,3 +269,21 @@ public void setForestBuilder(ForestBuilder forestBuilder) {
253269
this.forestBuilder = forestBuilder;
254270
}
255271
}
272+
273+
class ForestHostNames {
274+
private List<String> primaryForestHostNames;
275+
private List<String> replicaForestHostNames;
276+
277+
public ForestHostNames(List<String> primaryForestHostNames, List<String> replicaForestHostNames) {
278+
this.primaryForestHostNames = primaryForestHostNames;
279+
this.replicaForestHostNames = replicaForestHostNames;
280+
}
281+
282+
public List<String> getPrimaryForestHostNames() {
283+
return primaryForestHostNames;
284+
}
285+
286+
public List<String> getReplicaForestHostNames() {
287+
return replicaForestHostNames;
288+
}
289+
}

‎src/main/java/com/marklogic/appdeployer/command/forests/DistributedReplicaBuilderStrategy.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ public void buildReplicas(List<Forest> forests, ForestPlan forestPlan, AppConfig
1717
List<String> replicaDataDirectories, ForestNamingStrategy fns)
1818
{
1919
final String databaseName = forestPlan.getDatabaseName();
20-
final List<String> hostNames = forestPlan.getHostNames();
20+
final List<String> hostNames = forestPlan.getReplicaHostNames();
2121
final int replicaCount = forestPlan.getReplicaCount();
2222

2323
HashMap<String, List<Forest>> hostToForests = new HashMap<String, List<Forest>>();

‎src/main/java/com/marklogic/appdeployer/command/forests/ForestBuilder.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ protected Map<String, Map<String, List<Forest>>> existingForestsMap(ForestPlan f
151151
*/
152152
public void addReplicasToForests(List<Forest> forests, ForestPlan forestPlan, AppConfig appConfig, List<String> dataDirectories) {
153153
final String databaseName = forestPlan.getDatabaseName();
154-
final List<String> hostNames = forestPlan.getHostNames();
154+
final List<String> hostNames = forestPlan.getReplicaHostNames();
155155
final int replicaCount = forestPlan.getReplicaCount();
156156

157157
if (replicaCount >= hostNames.size()) {

‎src/main/java/com/marklogic/appdeployer/command/forests/ForestPlan.java

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ public class ForestPlan {
1010

1111
private String databaseName;
1212
private List<String> hostNames;
13+
private List<String> replicaHostNames;
1314
private String template;
1415
private int forestsPerDataDirectory = 1;
1516
private List<Forest> existingForests = new ArrayList<>();
@@ -19,9 +20,15 @@ public ForestPlan(String databaseName, String... hostNames) {
1920
this(databaseName, Arrays.asList(hostNames));
2021
}
2122

23+
/**
24+
* @param databaseName
25+
* @param hostNames the list of hosts that primary and replica forests can be created on. If the list of replica
26+
* forest host names differs, use withReplicaHostNames
27+
*/
2228
public ForestPlan(String databaseName, List<String> hostNames) {
2329
this.databaseName = databaseName;
2430
this.hostNames = hostNames;
31+
this.replicaHostNames = hostNames;
2532
}
2633

2734
public ForestPlan withTemplate(String template) {
@@ -44,6 +51,11 @@ public ForestPlan withReplicaCount(int count) {
4451
return this;
4552
}
4653

54+
public ForestPlan withReplicaHostNames(List<String> replicaHostNames) {
55+
this.replicaHostNames = replicaHostNames;
56+
return this;
57+
}
58+
4759
public String getDatabaseName() {
4860
return databaseName;
4961
}
@@ -67,4 +79,8 @@ public String getTemplate() {
6779
public List<Forest> getExistingForests() {
6880
return existingForests;
6981
}
82+
83+
public List<String> getReplicaHostNames() {
84+
return replicaHostNames;
85+
}
7086
}

‎src/test/java/com/marklogic/appdeployer/command/forests/BuildForestReplicaTest.java

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import org.junit.Assert;
99
import org.junit.Test;
1010

11+
import java.util.Arrays;
1112
import java.util.HashMap;
1213
import java.util.List;
1314
import java.util.Map;
@@ -61,7 +62,7 @@ public void sameNumberOfForestsAsHosts() {
6162
}
6263

6364
/**
64-
* Similar to the above test, just even more forests.
65+
* Similar to the above test, just even more forests.
6566
*/
6667
@Test
6768
public void numberOfForestsPerHostIsMoreThanDoubleTheNumberOfHosts() {
@@ -91,6 +92,34 @@ public void numberOfForestsPerHostIsMoreThanDoubleTheNumberOfHosts() {
9192
});
9293
}
9394

95+
/**
96+
* Verifies that replicaHostNames is used for generating the replicas. This is for databases that are configured
97+
* to have their primary forests on a single host, which is often the case for modules, schemas, and triggers.
98+
*/
99+
@Test
100+
public void primaryForestsOnOneHost() {
101+
AppConfig appConfig = newAppConfig("mlForestsPerHost", "db,2");
102+
103+
ForestPlan plan = new ForestPlan("db", "host1").withReplicaCount(2);
104+
assertEquals(1, plan.getHostNames().size());
105+
assertEquals(1, plan.getReplicaHostNames().size());
106+
plan.withReplicaHostNames(Arrays.asList("host1", "host2", "host3"));
107+
assertEquals(1, plan.getHostNames().size());
108+
assertEquals(3, plan.getReplicaHostNames().size());
109+
110+
List<Forest> forests = builder.buildForests(plan, appConfig);
111+
112+
assertEquals(2, forests.size());
113+
forests.forEach(forest -> {
114+
assertEquals("host1", forest.getHost());
115+
assertEquals(2, forest.getForestReplica().size());
116+
for (ForestReplica replica : forest.getForestReplica()) {
117+
String host = replica.getHost();
118+
assertTrue(host.equals("host2") || host.equals("host3"));
119+
}
120+
});
121+
}
122+
94123
@Test
95124
public void customNamingStrategyWithDistributedStrategy() {
96125
AppConfig appConfig = newAppConfig("mlForestsPerHost", "my-database,2");

‎src/test/java/com/marklogic/appdeployer/command/forests/CreateForestsOnOneHostTest.java

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -27,22 +27,27 @@ public void test() {
2727
HostCalculator hostCalculator = new DefaultHostCalculator(new TestHostNameProvider("host1", "host2", "host3"));
2828
command.setHostCalculator(hostCalculator);
2929

30-
List<String> hostNames = command.determineHostNamesForForest(context, fakeHostNames);
31-
assertEquals(3, hostNames.size());
30+
ForestHostNames hostNames = command.determineHostNamesForForest(context, fakeHostNames);
31+
assertEquals(3, hostNames.getPrimaryForestHostNames().size());
3232

3333
command.setCreateForestsOnEachHost(false);
3434
hostNames = command.determineHostNamesForForest(context, fakeHostNames);
35-
assertEquals(1, hostNames.size());
36-
assertEquals("host1", hostNames.get(0));
35+
assertEquals(1, hostNames.getPrimaryForestHostNames().size());
36+
assertEquals("host1", hostNames.getPrimaryForestHostNames().get(0));
37+
assertEquals(
38+
"When forests aren't created on each host, all hosts should still be available for replica forests, " +
39+
"with the expectation that the primary forest host will still not be used",
40+
3, hostNames.getReplicaForestHostNames().size());
3741

3842
command.setCreateForestsOnEachHost(true);
3943
hostNames = command.determineHostNamesForForest(context, fakeHostNames);
40-
assertEquals(3, hostNames.size());
44+
assertEquals(3, hostNames.getPrimaryForestHostNames().size());
4145

4246
Set<String> names = new HashSet<>();
4347
names.add("test-db");
4448
appConfig.setDatabasesWithForestsOnOneHost(names);
4549
hostNames = command.determineHostNamesForForest(context, fakeHostNames);
46-
assertEquals(1, hostNames.size());
50+
assertEquals(1, hostNames.getPrimaryForestHostNames().size());
51+
assertEquals(3, hostNames.getReplicaForestHostNames().size());
4752
}
4853
}

‎src/test/java/com/marklogic/appdeployer/command/forests/CreateForestsOnSpecificHostsTest.java

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,9 @@ public void test() {
2626

2727
command.setHostCalculator(new DefaultHostCalculator(new TestHostNameProvider(fakeHostNames.toArray(new String[]{}))));
2828
// Verify we get all 3 hosts back when nothing special is configured
29-
List<String> hostNames = command.determineHostNamesForForest(context, fakeHostNames);
30-
assertEquals(3, hostNames.size());
29+
ForestHostNames hostNames = command.determineHostNamesForForest(context, fakeHostNames);
30+
assertEquals(3, hostNames.getPrimaryForestHostNames().size());
31+
assertEquals(3, hostNames.getReplicaForestHostNames().size());
3132

3233
// Select 2 of the 3 hosts for test-db
3334
Properties props = new Properties();
@@ -37,16 +38,18 @@ public void test() {
3738
context = new CommandContext(appConfig, null, null);
3839

3940
hostNames = command.determineHostNamesForForest(context, fakeHostNames);
40-
assertEquals(2, hostNames.size());
41-
assertTrue(hostNames.contains("host1"));
42-
assertTrue(hostNames.contains("host2"));
41+
assertEquals(2, hostNames.getPrimaryForestHostNames().size());
42+
assertEquals(2, hostNames.getReplicaForestHostNames().size());
43+
assertTrue(hostNames.getPrimaryForestHostNames().contains("host1"));
44+
assertTrue(hostNames.getPrimaryForestHostNames().contains("host2"));
4345

4446
// Select 1 of the 3 hosts, and include a bad host; the bad one should be ignored
4547
props.setProperty("mlDatabaseHosts", "some-other-db,host2,test-db,bad-host|host2");
4648
appConfig = factory.newAppConfig();
4749
context = new CommandContext(appConfig, null, null);
4850
hostNames = command.determineHostNamesForForest(context, fakeHostNames);
49-
assertEquals(1, hostNames.size());
50-
assertTrue(hostNames.contains("host2"));
51+
assertEquals(1, hostNames.getPrimaryForestHostNames().size());
52+
assertEquals(1, hostNames.getReplicaForestHostNames().size());
53+
assertTrue(hostNames.getPrimaryForestHostNames().contains("host2"));
5154
}
5255
}

0 commit comments

Comments
 (0)