MLE-20741 Now accounting for zones when creating forest replicas

rjrudin · rjrudin · commit 523bd3889a1d · 2025-07-07T11:23:04.000-04:00
This hooks up the host-to-zone mapping so that it's fed into the forest replica planner. Currently only being tested via manual testing, which is consistent with how "real" replicas have been created in the past.

Enhanced the docker-compose file to create a 6-host cluster for manual testing purposes.

Story isn't quite done though. Need to add some tests for creating forests on a single host, as that seems a bit off.
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -14,3 +14,103 @@ services:
     ports:
       - "8000-8008:8000-8008"
       - "8028:8028"
+
+  # The six-node cluster defined below is used for manual testing of operations that require a cluster of 3 or more
+  # hosts, such as creating forest replicas.
+  node1:
+    image: "progressofficial/marklogic-db:latest"
+    platform: linux/amd64
+    hostname: node1.local
+    environment:
+      - MARKLOGIC_INIT=true
+      - MARKLOGIC_ADMIN_USERNAME=admin
+      - MARKLOGIC_ADMIN_PASSWORD=admin
+    volumes:
+      - ./docker/marklogic/node1/logs:/var/opt/MarkLogic/Logs
+    ports:
+      - 8100-8103:8000-8003
+
+  node2:
+    image: "progressofficial/marklogic-db:latest"
+    platform: linux/amd64
+    hostname: node2.local
+    depends_on:
+      - node1
+    environment:
+      - MARKLOGIC_INIT=true
+      - MARKLOGIC_ADMIN_USERNAME=admin
+      - MARKLOGIC_ADMIN_PASSWORD=admin
+      - MARKLOGIC_JOIN_CLUSTER=true
+      - MARKLOGIC_BOOTSTRAP_HOST=node1.local
+    volumes:
+      - ./docker/marklogic/node2/logs:/var/opt/MarkLogic/Logs
+    ports:
+      - 8200-8203:8000-8003
+
+  node3:
+    image: "progressofficial/marklogic-db:latest"
+    platform: linux/amd64
+    hostname: node3.local
+    depends_on:
+      - node1
+    environment:
+      - MARKLOGIC_INIT=true
+      - MARKLOGIC_ADMIN_USERNAME=admin
+      - MARKLOGIC_ADMIN_PASSWORD=admin
+      - MARKLOGIC_JOIN_CLUSTER=true
+      - MARKLOGIC_BOOTSTRAP_HOST=node1.local
+    volumes:
+      - ./docker/marklogic/node3/logs:/var/opt/MarkLogic/Logs
+    ports:
+      - 8300-8303:8000-8003
+
+  node4:
+    image: "progressofficial/marklogic-db:latest"
+    platform: linux/amd64
+    hostname: node4.local
+    depends_on:
+      - node1
+    environment:
+      - MARKLOGIC_INIT=true
+      - MARKLOGIC_ADMIN_USERNAME=admin
+      - MARKLOGIC_ADMIN_PASSWORD=admin
+      - MARKLOGIC_JOIN_CLUSTER=true
+      - MARKLOGIC_BOOTSTRAP_HOST=node1.local
+    volumes:
+      - ./docker/marklogic/node4/logs:/var/opt/MarkLogic/Logs
+    ports:
+      - 8500-8503:8000-8003
+
+  node5:
+    image: "progressofficial/marklogic-db:latest"
+    platform: linux/amd64
+    hostname: node5.local
+    depends_on:
+      - node1
+    environment:
+      - MARKLOGIC_INIT=true
+      - MARKLOGIC_ADMIN_USERNAME=admin
+      - MARKLOGIC_ADMIN_PASSWORD=admin
+      - MARKLOGIC_JOIN_CLUSTER=true
+      - MARKLOGIC_BOOTSTRAP_HOST=node1.local
+    volumes:
+      - ./docker/marklogic/node5/logs:/var/opt/MarkLogic/Logs
+    ports:
+      - 8600-8603:8000-8003
+
+  node6:
+    image: "progressofficial/marklogic-db:latest"
+    platform: linux/amd64
+    hostname: node6.local
+    depends_on:
+      - node1
+    environment:
+      - MARKLOGIC_INIT=true
+      - MARKLOGIC_ADMIN_USERNAME=admin
+      - MARKLOGIC_ADMIN_PASSWORD=admin
+      - MARKLOGIC_JOIN_CLUSTER=true
+      - MARKLOGIC_BOOTSTRAP_HOST=node1.local
+    volumes:
+      - ./docker/marklogic/node6/logs:/var/opt/MarkLogic/Logs
+    ports:
+      - 8700-8703:8000-8003
diff --git a/ml-app-deployer/src/main/java/com/marklogic/appdeployer/command/forests/ConfigureForestReplicasCommand.java b/ml-app-deployer/src/main/java/com/marklogic/appdeployer/command/forests/ConfigureForestReplicasCommand.java
@@ -34,19 +34,18 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
 
 /**
- * Command for configuring - i.e. creating and setting - replica forests for existing databases.
- * <p>
- * Very useful for the out-of-the-box forests such as Security, Schemas, App-Services, and Meters, which normally need
- * replicas for failover in a cluster.
+ * Command for configuring - i.e. creating and setting - replica forests for existing databases. The expectation is that
+ * {@code DeployForestsCommand} is used for creating primary forests, while this command is used for creating replica
+ * forests based on existing primary forests.
  */
 public class ConfigureForestReplicasCommand extends AbstractUndoableCommand {
 
 	private Map<String, Integer> databaseNamesAndReplicaCounts = new HashMap<>();
 	private boolean deleteReplicasOnUndo = true;
 	private GroupHostNamesProvider groupHostNamesProvider;
+	private ForestBuilder forestBuilder = new ForestBuilder();
 
 	public ConfigureForestReplicasCommand() {
 		setExecuteSortOrder(SortOrderConstants.DEPLOY_FOREST_REPLICAS);
@@ -128,39 +127,47 @@ protected void deleteReplicas(String forestName, ForestManager forestMgr) {
 	 * @param hostNames
 	 * @param context
 	 */
-	protected void configureDatabaseReplicaForests(String databaseName, int replicaCount, List<String> hostNames, CommandContext context) {
+	private void configureDatabaseReplicaForests(String databaseName, int replicaCount, List<String> hostNames, CommandContext context) {
 		List<Forest> forestsNeedingReplicas = determineForestsNeedingReplicas(databaseName, context);
 
-		ForestBuilder forestBuilder = new ForestBuilder();
 		List<String> selectedHostNames = getHostNamesForDatabaseForests(databaseName, hostNames, context);
-		ForestPlan forestPlan = new ForestPlan(databaseName, selectedHostNames).withReplicaCount(replicaCount);
 		List<String> dataDirectories = forestBuilder.determineDataDirectories(databaseName, context.getAppConfig());
+
+		final ForestPlan forestPlan = new ForestPlan(databaseName, selectedHostNames)
+			.withHostsToZones(new HostManager(context.getManageClient()).getHostNamesAndZones())
+			.withReplicaCount(replicaCount);
+
 		forestBuilder.addReplicasToForests(forestsNeedingReplicas, forestPlan, context.getAppConfig(), dataDirectories);
 
+		// Trim off all forests details so only the replicas are saved.
 		List<Forest> forestsWithOnlyReplicas = forestsNeedingReplicas.stream().map(forest -> {
 			Forest forestWithOnlyReplicas = new Forest();
 			forestWithOnlyReplicas.setForestName(forest.getForestName());
 			forestWithOnlyReplicas.setForestReplica(forest.getForestReplica());
 			return forestWithOnlyReplicas;
-		}).collect(Collectors.toList());
+		}).toList();
 
 		// As of 4.5.3, try CMA first so that this can be done in a single request instead of one request per forest.
 		if (context.getAppConfig().getCmaConfig().isDeployForests()) {
 			try {
-				Configuration config = new Configuration();
-				forestsWithOnlyReplicas.forEach(forest -> {
-					config.addForest(forest.toObjectNode());
-				});
-				new Configurations(config).submit(context.getManageClient());
+				saveReplicasViaCma(forestsWithOnlyReplicas, context);
 				return;
 			} catch (Exception ex) {
-				logger.warn("Unable to create forest replicas via CMA; cause: " + ex.getMessage() + "; will " +
-					"fall back to using /manage/v2.");
+				logger.warn("Unable to create forest replicas via CMA; cause: {}; will fall back to using /manage/v2.", ex.getMessage());
 			}
 		}
 
-		// If we get here, either CMA usage is disabled or an error occurred with CMA. Just use /manage/v2 to submit
-		// each forest one-by-one.
+		// If we get here, either CMA usage is disabled or an error occurred with CMA.
+		saveReplicasOneByOne(forestsWithOnlyReplicas, context);
+	}
+
+	private void saveReplicasViaCma(List<Forest> forestsWithOnlyReplicas, CommandContext context) {
+		Configuration config = new Configuration();
+		forestsWithOnlyReplicas.forEach(forest -> config.addForest(forest.toObjectNode()));
+		new Configurations(config).submit(context.getManageClient());
+	}
+
+	private void saveReplicasOneByOne(List<Forest> forestsWithOnlyReplicas, CommandContext context) {
 		ForestManager forestManager = new ForestManager(context.getManageClient());
 		forestsWithOnlyReplicas.forEach(forest -> {
 			String forestName = forest.getForestName();
diff --git a/ml-app-deployer/src/main/java/com/marklogic/appdeployer/command/forests/DeployForestsCommand.java b/ml-app-deployer/src/main/java/com/marklogic/appdeployer/command/forests/DeployForestsCommand.java
@@ -28,6 +28,7 @@
 import com.marklogic.mgmt.resource.databases.DatabaseManager;
 import com.marklogic.mgmt.resource.forests.ForestManager;
 import com.marklogic.mgmt.resource.hosts.DefaultHostNameProvider;
+import com.marklogic.mgmt.resource.hosts.HostManager;
 import org.springframework.util.StringUtils;
 
 import java.io.File;
@@ -122,11 +123,11 @@ protected void createForestsViaForestEndpoint(CommandContext context, List<Fores
 	 * these commands to construct a list of many forests that can be created via CMA in one request.
 	 *
 	 * @param context
-	 * @param includeReplicas This command currently doesn't make use of this feature; it's here so that other clients
-	 *                        can get a preview of the forests to be created, including replicas.
+	 * @param previewingForestCreation When previewing forest creation, replicas will be built. If not previewing
+	 *                                 forest creation, then only primary forests will be built.
 	 * @return
 	 */
-	public List<Forest> buildForests(CommandContext context, boolean includeReplicas) {
+	public List<Forest> buildForests(CommandContext context, boolean previewingForestCreation) {
 		// Need to know what primary forests exist already in case more need to be added, or a new host has been added
 		List<Forest> existingPrimaryForests = null;
 
@@ -141,36 +142,35 @@ public List<Forest> buildForests(CommandContext context, boolean includeReplicas
 			existingPrimaryForests = getExistingPrimaryForests(context, this.databaseName);
 		}
 
-		return buildForests(context, includeReplicas, existingPrimaryForests);
+		return buildForests(context, previewingForestCreation, existingPrimaryForests);
 	}
 
 	/**
 	 * @param context
-	 * @param includeReplicas
+	 * @param previewingForestCreation
 	 * @param existingPrimaryForests
 	 * @return
 	 */
-	protected List<Forest> buildForests(CommandContext context, boolean includeReplicas, List<Forest> existingPrimaryForests) {
+	protected List<Forest> buildForests(CommandContext context, boolean previewingForestCreation, List<Forest> existingPrimaryForests) {
 		ForestHostNames forestHostNames = determineHostNamesForForest(context, existingPrimaryForests);
 
 		final String template = buildForestTemplate(context, new ForestManager(context.getManageClient()));
 
 		ForestPlan forestPlan = new ForestPlan(this.databaseName, forestHostNames.getPrimaryForestHostNames())
-			.withReplicaHostNames(forestHostNames.getReplicaForestHostNames())
 			.withTemplate(template)
 			.withForestsPerDataDirectory(this.forestsPerHost)
 			.withExistingForests(existingPrimaryForests);
 
-		if (includeReplicas) {
+		if (previewingForestCreation) {
 			Map<String, Integer> map = context.getAppConfig().getDatabaseNamesAndReplicaCounts();
 			if (map != null && map.containsKey(this.databaseName)) {
 				int count = map.get(this.databaseName);
 				if (count > 0) {
-					// Need to pass in host-to-zone mapping.
-					// And what to do about ConfigureForestReplicasCommand??? We may be better off removing that as part of
-					// the 6.0 release so that we only have replica creation in one place. Would need to verify that this command
-					// still works for OOTB databases.
-					forestPlan.withReplicaCount(count);
+					// If replicas are needed while previewing forest creation, we need to fetch some additional data
+					// to ensure the correct replicas are previewed.
+					forestPlan.withReplicaCount(count)
+						.withHostsToZones(new HostManager(context.getManageClient()).getHostNamesAndZones())
+						.withReplicaHostNames(forestHostNames.getReplicaForestHostNames());
 				}
 			}
 		}
@@ -232,7 +232,7 @@ protected String buildForestTemplate(CommandContext context, ForestManager fores
 	 * @param existingPrimaryForests
 	 * @return a ForestHostNames instance that defines the list of host names that can be used for primary forests and
 	 * that can be used for replica forests. As of 4.1.0, the only reason these will differ is when a database is
-	 * configured to only have forests on one host, or when the deprecated setCreateForestsOnOneHost method is used.
+	 * configured to only have forests on one host.
 	 */
 	protected ForestHostNames determineHostNamesForForest(CommandContext context, List<Forest> existingPrimaryForests) {
 		if (hostCalculator == null) {
diff --git a/ml-app-deployer/src/main/java/com/marklogic/appdeployer/command/forests/ForestPlan.java b/ml-app-deployer/src/main/java/com/marklogic/appdeployer/command/forests/ForestPlan.java
@@ -68,11 +68,23 @@ public ForestPlan withReplicaCount(int count) {
 		return this;
 	}
 
+	/**
+	 * Confusingly, this is only used - at least as of 6.0.0 - when previewing forest creation. It is not used when
+	 * actually configuring forest replicas.
+	 *
+	 * @param replicaHostNames
+	 * @return
+	 */
 	public ForestPlan withReplicaHostNames(List<String> replicaHostNames) {
 		this.replicaHostNames = replicaHostNames;
 		return this;
 	}
 
+	/**
+	 * @param hostsToZones a mapping of each host name to an optional zone value for each host. The zone value can be
+	 *                     null for a host.
+	 * @since 6.0.0
+	 */
 	public ForestPlan withHostsToZones(Map<String, String> hostsToZones) {
 		this.hostsToZones = hostsToZones;
 		return this;
@@ -106,6 +118,10 @@ public List<String> getReplicaHostNames() {
 		return replicaHostNames;
 	}
 
+	/**
+	 * @return
+	 * @since 6.0.0
+	 */
 	public Map<String, String> getHostsToZones() {
 		return hostsToZones;
 	}
diff --git a/ml-app-deployer/src/main/java/com/marklogic/mgmt/resource/hosts/HostManager.java b/ml-app-deployer/src/main/java/com/marklogic/mgmt/resource/hosts/HostManager.java
@@ -28,48 +28,67 @@
 
 public class HostManager extends AbstractManager {
 
-    private ManageClient client;
+	private ManageClient manageClient;
 
-    public HostManager(ManageClient client) {
-        this.client = client;
-    }
+	public HostManager(ManageClient manageClient) {
+		this.manageClient = manageClient;
+	}
 
-    public List<String> getHostIds() {
-        return getHosts().getElementValues("/h:host-default-list/h:list-items/h:list-item/h:idref");
-    }
+	public List<String> getHostIds() {
+		return getHosts().getElementValues("/h:host-default-list/h:list-items/h:list-item/h:idref");
+	}
 
-    public List<String> getHostNames() {
-        return getHosts().getElementValues("/h:host-default-list/h:list-items/h:list-item/h:nameref");
-    }
+	public List<String> getHostNames() {
+		return getHosts().getElementValues("/h:host-default-list/h:list-items/h:list-item/h:nameref");
+	}
+
+	/**
+	 * @return a map of host names to optional zones. For performance reasons, as soon as a host is found to not have
+	 * a zone, then an empty map will be returned. Currently, the only use case for this map is to create forest replicas
+	 * across zones. But if any host does not have a zone, then zones do not matter.
+	 * @since 6.0.0
+	 */
+	public Map<String, String> getHostNamesAndZones() {
+		Map<String, String> map = new LinkedHashMap<>();
+		for (String hostName : getHostNames()) {
+			String json = manageClient.getJson("/manage/v2/hosts/%s/properties".formatted(hostName));
+			String zone = payloadParser.getPayloadFieldValue(json, "zone");
+			if (zone == null || zone.trim().isEmpty()) {
+				return map;
+			}
+			map.put(hostName, zone);
+		}
+		return map;
+	}
 
 	/**
 	 * @return a map with an entry for each host, with the key of the host being the host ID and the value being the
 	 * host name
 	 */
 	public Map<String, String> getHostIdsAndNames() {
-    	Fragment xml = getHosts();
-    	Map<String, String> map = new LinkedHashMap<>();
-	    Namespace ns = Namespace.getNamespace("http://marklogic.com/manage/hosts");
-    	for (Element el : xml.getElements("/h:host-default-list/h:list-items/h:list-item")) {
-    		String hostId = el.getChildText("idref", ns);
-    		String hostName = el.getChildText("nameref", ns);
-    		map.put(hostId, hostName);
-	    }
-    	return map;
-    }
+		Fragment xml = getHosts();
+		Map<String, String> map = new LinkedHashMap<>();
+		Namespace ns = Namespace.getNamespace("http://marklogic.com/manage/hosts");
+		for (Element el : xml.getElements("/h:host-default-list/h:list-items/h:list-item")) {
+			String hostId = el.getChildText("idref", ns);
+			String hostName = el.getChildText("nameref", ns);
+			map.put(hostId, hostName);
+		}
+		return map;
+	}
 
-    public Fragment getHosts() {
-        return client.getXml("/manage/v2/hosts");
-    }
+	public Fragment getHosts() {
+		return manageClient.getXml("/manage/v2/hosts");
+	}
 
-    public ResponseEntity<String> setHostToGroup(String hostIdOrName, String groupIdOrName) {
-        String json = format("{\"group\":\"%s\"}", groupIdOrName);
-        String url = format("/manage/v2/hosts/%s/properties", hostIdOrName);
-        return client.putJson(url, json);
-    }
+	public ResponseEntity<String> setHostToGroup(String hostIdOrName, String groupIdOrName) {
+		String json = format("{\"group\":\"%s\"}", groupIdOrName);
+		String url = format("/manage/v2/hosts/%s/properties", hostIdOrName);
+		return manageClient.putJson(url, json);
+	}
 
 	public String getAssignedGroupName(String hostIdOrName) {
 		String url = format("/manage/v2/hosts/%s/properties", hostIdOrName);
-		return payloadParser.getPayloadFieldValue(client.getJson(url), "group");
+		return payloadParser.getPayloadFieldValue(manageClient.getJson(url), "group");
 	}
 }
diff --git a/ml-app-deployer/src/test/java/com/marklogic/appdeployer/command/forests/ConfigureForestReplicasDebug.java b/ml-app-deployer/src/test/java/com/marklogic/appdeployer/command/forests/ConfigureForestReplicasDebug.java
