Merge pull request #756 from marklogic/feature/20741-consolidate

rjrudin · web-flow · commit 8d69d195db79 · 2025-07-07T13:02:12.000-04:00
MLE-20741 Now accounting for zones when creating forest replicas
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -14,3 +14,103 @@ services:
     ports:
       - "8000-8008:8000-8008"
       - "8028:8028"
+
+  # The six-node cluster defined below is used for manual testing of operations that require a cluster of 3 or more
+  # hosts, such as creating forest replicas.
+  node1:
+    image: "progressofficial/marklogic-db:latest"
+    platform: linux/amd64
+    hostname: node1.local
+    environment:
+      - MARKLOGIC_INIT=true
+      - MARKLOGIC_ADMIN_USERNAME=admin
+      - MARKLOGIC_ADMIN_PASSWORD=admin
+    volumes:
+      - ./docker/marklogic/node1/logs:/var/opt/MarkLogic/Logs
+    ports:
+      - 8100-8103:8000-8003
+
+  node2:
+    image: "progressofficial/marklogic-db:latest"
+    platform: linux/amd64
+    hostname: node2.local
+    depends_on:
+      - node1
+    environment:
+      - MARKLOGIC_INIT=true
+      - MARKLOGIC_ADMIN_USERNAME=admin
+      - MARKLOGIC_ADMIN_PASSWORD=admin
+      - MARKLOGIC_JOIN_CLUSTER=true
+      - MARKLOGIC_BOOTSTRAP_HOST=node1.local
+    volumes:
+      - ./docker/marklogic/node2/logs:/var/opt/MarkLogic/Logs
+    ports:
+      - 8200-8203:8000-8003
+
+  node3:
+    image: "progressofficial/marklogic-db:latest"
+    platform: linux/amd64
+    hostname: node3.local
+    depends_on:
+      - node1
+    environment:
+      - MARKLOGIC_INIT=true
+      - MARKLOGIC_ADMIN_USERNAME=admin
+      - MARKLOGIC_ADMIN_PASSWORD=admin
+      - MARKLOGIC_JOIN_CLUSTER=true
+      - MARKLOGIC_BOOTSTRAP_HOST=node1.local
+    volumes:
+      - ./docker/marklogic/node3/logs:/var/opt/MarkLogic/Logs
+    ports:
+      - 8300-8303:8000-8003
+
+  node4:
+    image: "progressofficial/marklogic-db:latest"
+    platform: linux/amd64
+    hostname: node4.local
+    depends_on:
+      - node1
+    environment:
+      - MARKLOGIC_INIT=true
+      - MARKLOGIC_ADMIN_USERNAME=admin
+      - MARKLOGIC_ADMIN_PASSWORD=admin
+      - MARKLOGIC_JOIN_CLUSTER=true
+      - MARKLOGIC_BOOTSTRAP_HOST=node1.local
+    volumes:
+      - ./docker/marklogic/node4/logs:/var/opt/MarkLogic/Logs
+    ports:
+      - 8500-8503:8000-8003
+
+  node5:
+    image: "progressofficial/marklogic-db:latest"
+    platform: linux/amd64
+    hostname: node5.local
+    depends_on:
+      - node1
+    environment:
+      - MARKLOGIC_INIT=true
+      - MARKLOGIC_ADMIN_USERNAME=admin
+      - MARKLOGIC_ADMIN_PASSWORD=admin
+      - MARKLOGIC_JOIN_CLUSTER=true
+      - MARKLOGIC_BOOTSTRAP_HOST=node1.local
+    volumes:
+      - ./docker/marklogic/node5/logs:/var/opt/MarkLogic/Logs
+    ports:
+      - 8600-8603:8000-8003
+
+  node6:
+    image: "progressofficial/marklogic-db:latest"
+    platform: linux/amd64
+    hostname: node6.local
+    depends_on:
+      - node1
+    environment:
+      - MARKLOGIC_INIT=true
+      - MARKLOGIC_ADMIN_USERNAME=admin
+      - MARKLOGIC_ADMIN_PASSWORD=admin
+      - MARKLOGIC_JOIN_CLUSTER=true
+      - MARKLOGIC_BOOTSTRAP_HOST=node1.local
+    volumes:
+      - ./docker/marklogic/node6/logs:/var/opt/MarkLogic/Logs
+    ports:
+      - 8700-8703:8000-8003
diff --git a/ml-app-deployer/src/main/java/com/marklogic/appdeployer/command/forests/ConfigureForestReplicasCommand.java b/ml-app-deployer/src/main/java/com/marklogic/appdeployer/command/forests/ConfigureForestReplicasCommand.java
@@ -34,19 +34,18 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
 
 /**
- * Command for configuring - i.e. creating and setting - replica forests for existing databases.
- * <p>
- * Very useful for the out-of-the-box forests such as Security, Schemas, App-Services, and Meters, which normally need
- * replicas for failover in a cluster.
+ * Command for configuring - i.e. creating and setting - replica forests for existing databases. The expectation is that
+ * {@code DeployForestsCommand} is used for creating primary forests, while this command is used for creating replica
+ * forests based on existing primary forests.
  */
 public class ConfigureForestReplicasCommand extends AbstractUndoableCommand {
 
 	private Map<String, Integer> databaseNamesAndReplicaCounts = new HashMap<>();
 	private boolean deleteReplicasOnUndo = true;
 	private GroupHostNamesProvider groupHostNamesProvider;
+	private ForestBuilder forestBuilder = new ForestBuilder();
 
 	public ConfigureForestReplicasCommand() {
 		setExecuteSortOrder(SortOrderConstants.DEPLOY_FOREST_REPLICAS);
@@ -128,39 +127,47 @@ protected void deleteReplicas(String forestName, ForestManager forestMgr) {
 	 * @param hostNames
 	 * @param context
 	 */
-	protected void configureDatabaseReplicaForests(String databaseName, int replicaCount, List<String> hostNames, CommandContext context) {
+	private void configureDatabaseReplicaForests(String databaseName, int replicaCount, List<String> hostNames, CommandContext context) {
 		List<Forest> forestsNeedingReplicas = determineForestsNeedingReplicas(databaseName, context);
 
-		ForestBuilder forestBuilder = new ForestBuilder();
 		List<String> selectedHostNames = getHostNamesForDatabaseForests(databaseName, hostNames, context);
-		ForestPlan forestPlan = new ForestPlan(databaseName, selectedHostNames).withReplicaCount(replicaCount);
 		List<String> dataDirectories = forestBuilder.determineDataDirectories(databaseName, context.getAppConfig());
+
+		final ForestPlan forestPlan = new ForestPlan(databaseName, selectedHostNames)
+			.withHostsToZones(new HostManager(context.getManageClient()).getHostNamesAndZones())
+			.withReplicaCount(replicaCount);
+
 		forestBuilder.addReplicasToForests(forestsNeedingReplicas, forestPlan, context.getAppConfig(), dataDirectories);
 
+		// Trim off all forests details so only the replicas are saved.
 		List<Forest> forestsWithOnlyReplicas = forestsNeedingReplicas.stream().map(forest -> {
 			Forest forestWithOnlyReplicas = new Forest();
 			forestWithOnlyReplicas.setForestName(forest.getForestName());
 			forestWithOnlyReplicas.setForestReplica(forest.getForestReplica());
 			return forestWithOnlyReplicas;
-		}).collect(Collectors.toList());
+		}).toList();
 
 		// As of 4.5.3, try CMA first so that this can be done in a single request instead of one request per forest.
 		if (context.getAppConfig().getCmaConfig().isDeployForests()) {
 			try {
-				Configuration config = new Configuration();
-				forestsWithOnlyReplicas.forEach(forest -> {
-					config.addForest(forest.toObjectNode());
-				});
-				new Configurations(config).submit(context.getManageClient());
+				saveReplicasViaCma(forestsWithOnlyReplicas, context);
 				return;
 			} catch (Exception ex) {
-				logger.warn("Unable to create forest replicas via CMA; cause: " + ex.getMessage() + "; will " +
-					"fall back to using /manage/v2.");
+				logger.warn("Unable to create forest replicas via CMA; cause: {}; will fall back to using /manage/v2.", ex.getMessage());
 			}
 		}
 
-		// If we get here, either CMA usage is disabled or an error occurred with CMA. Just use /manage/v2 to submit
-		// each forest one-by-one.
+		// If we get here, either CMA usage is disabled or an error occurred with CMA.
+		saveReplicasOneByOne(forestsWithOnlyReplicas, context);
+	}
+
+	private void saveReplicasViaCma(List<Forest> forestsWithOnlyReplicas, CommandContext context) {
+		Configuration config = new Configuration();
+		forestsWithOnlyReplicas.forEach(forest -> config.addForest(forest.toObjectNode()));
+		new Configurations(config).submit(context.getManageClient());
+	}
+
+	private void saveReplicasOneByOne(List<Forest> forestsWithOnlyReplicas, CommandContext context) {
 		ForestManager forestManager = new ForestManager(context.getManageClient());
 		forestsWithOnlyReplicas.forEach(forest -> {
 			String forestName = forest.getForestName();
diff --git a/ml-app-deployer/src/main/java/com/marklogic/appdeployer/command/forests/DeployForestsCommand.java b/ml-app-deployer/src/main/java/com/marklogic/appdeployer/command/forests/DeployForestsCommand.java
@@ -28,6 +28,7 @@
 import com.marklogic.mgmt.resource.databases.DatabaseManager;
 import com.marklogic.mgmt.resource.forests.ForestManager;
 import com.marklogic.mgmt.resource.hosts.DefaultHostNameProvider;
+import com.marklogic.mgmt.resource.hosts.HostManager;
 import org.springframework.util.StringUtils;
 
 import java.io.File;
@@ -122,11 +123,11 @@ protected void createForestsViaForestEndpoint(CommandContext context, List<Fores
 	 * these commands to construct a list of many forests that can be created via CMA in one request.
 	 *
 	 * @param context
-	 * @param includeReplicas This command currently doesn't make use of this feature; it's here so that other clients
-	 *                        can get a preview of the forests to be created, including replicas.
+	 * @param previewingForestCreation When previewing forest creation, replicas will be built. If not previewing
+	 *                                 forest creation, then only primary forests will be built.
 	 * @return
 	 */
-	public List<Forest> buildForests(CommandContext context, boolean includeReplicas) {
+	public List<Forest> buildForests(CommandContext context, boolean previewingForestCreation) {
 		// Need to know what primary forests exist already in case more need to be added, or a new host has been added
 		List<Forest> existingPrimaryForests = null;
 
@@ -141,36 +142,35 @@ public List<Forest> buildForests(CommandContext context, boolean includeReplicas
 			existingPrimaryForests = getExistingPrimaryForests(context, this.databaseName);
 		}
 
-		return buildForests(context, includeReplicas, existingPrimaryForests);
+		return buildForests(context, previewingForestCreation, existingPrimaryForests);
 	}
 
 	/**
 	 * @param context
-	 * @param includeReplicas
+	 * @param previewingForestCreation
 	 * @param existingPrimaryForests
 	 * @return
 	 */
-	protected List<Forest> buildForests(CommandContext context, boolean includeReplicas, List<Forest> existingPrimaryForests) {
+	protected List<Forest> buildForests(CommandContext context, boolean previewingForestCreation, List<Forest> existingPrimaryForests) {
 		ForestHostNames forestHostNames = determineHostNamesForForest(context, existingPrimaryForests);
 
 		final String template = buildForestTemplate(context, new ForestManager(context.getManageClient()));
 
 		ForestPlan forestPlan = new ForestPlan(this.databaseName, forestHostNames.getPrimaryForestHostNames())
-			.withReplicaHostNames(forestHostNames.getReplicaForestHostNames())
 			.withTemplate(template)
 			.withForestsPerDataDirectory(this.forestsPerHost)
 			.withExistingForests(existingPrimaryForests);
 
-		if (includeReplicas) {
+		if (previewingForestCreation) {
 			Map<String, Integer> map = context.getAppConfig().getDatabaseNamesAndReplicaCounts();
 			if (map != null && map.containsKey(this.databaseName)) {
 				int count = map.get(this.databaseName);
 				if (count > 0) {
-					// Need to pass in host-to-zone mapping.
-					// And what to do about ConfigureForestReplicasCommand??? We may be better off removing that as part of
-					// the 6.0 release so that we only have replica creation in one place. Would need to verify that this command
-					// still works for OOTB databases.
-					forestPlan.withReplicaCount(count);
+					// If replicas are needed while previewing forest creation, we need to fetch some additional data
+					// to ensure the correct replicas are previewed.
+					forestPlan.withReplicaCount(count)
+						.withHostsToZones(new HostManager(context.getManageClient()).getHostNamesAndZones())
+						.withReplicaHostNames(forestHostNames.getReplicaForestHostNames());
 				}
 			}
 		}
@@ -232,7 +232,7 @@ protected String buildForestTemplate(CommandContext context, ForestManager fores
 	 * @param existingPrimaryForests
 	 * @return a ForestHostNames instance that defines the list of host names that can be used for primary forests and
 	 * that can be used for replica forests. As of 4.1.0, the only reason these will differ is when a database is
-	 * configured to only have forests on one host, or when the deprecated setCreateForestsOnOneHost method is used.
+	 * configured to only have forests on one host.
 	 */
 	protected ForestHostNames determineHostNamesForForest(CommandContext context, List<Forest> existingPrimaryForests) {
 		if (hostCalculator == null) {
diff --git a/ml-app-deployer/src/main/java/com/marklogic/appdeployer/command/forests/ForestPlan.java b/ml-app-deployer/src/main/java/com/marklogic/appdeployer/command/forests/ForestPlan.java
@@ -68,11 +68,23 @@ public ForestPlan withReplicaCount(int count) {
 		return this;
 	}
 
+	/**
+	 * Confusingly, this is only used - at least as of 6.0.0 - when previewing forest creation. It is not used when
+	 * actually configuring forest replicas.
+	 *
+	 * @param replicaHostNames
+	 * @return
+	 */
 	public ForestPlan withReplicaHostNames(List<String> replicaHostNames) {
 		this.replicaHostNames = replicaHostNames;
 		return this;
 	}
 
+	/**
+	 * @param hostsToZones a mapping of each host name to an optional zone value for each host. The zone value can be
+	 *                     null for a host.
+	 * @since 6.0.0
+	 */
 	public ForestPlan withHostsToZones(Map<String, String> hostsToZones) {
 		this.hostsToZones = hostsToZones;
 		return this;
@@ -106,6 +118,10 @@ public List<String> getReplicaHostNames() {
 		return replicaHostNames;
 	}
 
+	/**
+	 * @return
+	 * @since 6.0.0
+	 */
 	public Map<String, String> getHostsToZones() {
 		return hostsToZones;
 	}
diff --git a/ml-app-deployer/src/main/java/com/marklogic/mgmt/resource/hosts/HostManager.java b/ml-app-deployer/src/main/java/com/marklogic/mgmt/resource/hosts/HostManager.java
@@ -28,48 +28,67 @@
 
 public class HostManager extends AbstractManager {
 
-    private ManageClient client;
+	private ManageClient manageClient;
 
-    public HostManager(ManageClient client) {
-        this.client = client;
-    }
+	public HostManager(ManageClient manageClient) {
+		this.manageClient = manageClient;
+	}
 
-    public List<String> getHostIds() {
-        return getHosts().getElementValues("/h:host-default-list/h:list-items/h:list-item/h:idref");
-    }
+	public List<String> getHostIds() {
+		return getHosts().getElementValues("/h:host-default-list/h:list-items/h:list-item/h:idref");
+	}
 
-    public List<String> getHostNames() {
-        return getHosts().getElementValues("/h:host-default-list/h:list-items/h:list-item/h:nameref");
-    }
+	public List<String> getHostNames() {
+		return getHosts().getElementValues("/h:host-default-list/h:list-items/h:list-item/h:nameref");
+	}
+
+	/**
+	 * @return a map of host names to optional zones. For performance reasons, as soon as a host is found to not have
+	 * a zone, then an empty map will be returned. Currently, the only use case for this map is to create forest replicas
+	 * across zones. But if any host does not have a zone, then zones do not matter.
+	 * @since 6.0.0
+	 */
+	public Map<String, String> getHostNamesAndZones() {
+		Map<String, String> map = new LinkedHashMap<>();
+		for (String hostName : getHostNames()) {
+			String json = manageClient.getJson("/manage/v2/hosts/%s/properties".formatted(hostName));
+			String zone = payloadParser.getPayloadFieldValue(json, "zone");
+			if (zone == null || zone.trim().isEmpty()) {
+				return map;
+			}
+			map.put(hostName, zone);
+		}
+		return map;
+	}
 
 	/**
 	 * @return a map with an entry for each host, with the key of the host being the host ID and the value being the
 	 * host name
 	 */
 	public Map<String, String> getHostIdsAndNames() {
-    	Fragment xml = getHosts();
-    	Map<String, String> map = new LinkedHashMap<>();
-	    Namespace ns = Namespace.getNamespace("http://marklogic.com/manage/hosts");
-    	for (Element el : xml.getElements("/h:host-default-list/h:list-items/h:list-item")) {
-    		String hostId = el.getChildText("idref", ns);
-    		String hostName = el.getChildText("nameref", ns);
-    		map.put(hostId, hostName);
-	    }
-    	return map;
-    }
+		Fragment xml = getHosts();
+		Map<String, String> map = new LinkedHashMap<>();
+		Namespace ns = Namespace.getNamespace("http://marklogic.com/manage/hosts");
+		for (Element el : xml.getElements("/h:host-default-list/h:list-items/h:list-item")) {
+			String hostId = el.getChildText("idref", ns);
+			String hostName = el.getChildText("nameref", ns);
+			map.put(hostId, hostName);
+		}
+		return map;
+	}
 
-    public Fragment getHosts() {
-        return client.getXml("/manage/v2/hosts");
-    }
+	public Fragment getHosts() {
+		return manageClient.getXml("/manage/v2/hosts");
+	}
 
-    public ResponseEntity<String> setHostToGroup(String hostIdOrName, String groupIdOrName) {
-        String json = format("{\"group\":\"%s\"}", groupIdOrName);
-        String url = format("/manage/v2/hosts/%s/properties", hostIdOrName);
-        return client.putJson(url, json);
-    }
+	public ResponseEntity<String> setHostToGroup(String hostIdOrName, String groupIdOrName) {
+		String json = format("{\"group\":\"%s\"}", groupIdOrName);
+		String url = format("/manage/v2/hosts/%s/properties", hostIdOrName);
+		return manageClient.putJson(url, json);
+	}
 
 	public String getAssignedGroupName(String hostIdOrName) {
 		String url = format("/manage/v2/hosts/%s/properties", hostIdOrName);
-		return payloadParser.getPayloadFieldValue(client.getJson(url), "group");
+		return payloadParser.getPayloadFieldValue(manageClient.getJson(url), "group");
 	}
 }
diff --git a/ml-app-deployer/src/test/java/com/marklogic/appdeployer/command/forests/ConfigureForestReplicasDebug.java b/ml-app-deployer/src/test/java/com/marklogic/appdeployer/command/forests/ConfigureForestReplicasDebug.java
