Merge pull request #487 from marklogic/release/4.5.3

Release/4.5.3

Author: rjrudin

CODEOWNERS

@@ -0,0 +1,5 @@
+# Lines starting with '#' are comments.
+# Each line is a file pattern followed by one or more owners.
+
+# These owners will be the default owners for everything in the repo.
+* @anu3990 @billfarber @rjrudin

build.gradle

@@ -8,7 +8,7 @@ plugins {
 }
 
 group = "com.marklogic"
-version = "4.5.2"
+version = "4.5.3"
 
 java {
 	sourceCompatibility = 1.8

pom.xml

@@ -12,7 +12,7 @@ It is not intended to be used to build this project.
   <modelVersion>4.0.0</modelVersion>
   <groupId>com.marklogic</groupId>
   <artifactId>ml-app-deployer</artifactId>
-  <version>4.5.2</version>
+  <version>4.5.3</version>
   <name>com.marklogic:ml-app-deployer</name>
   <description>Java client for the MarkLogic REST Management API and for deploying applications to MarkLogic</description>
   <url>https://github.com/marklogic/ml-app-deployer</url>

src/main/java/com/marklogic/appdeployer/command/CommandContext.java

@@ -20,8 +20,13 @@
 import com.marklogic.mgmt.admin.AdminManager;
 import com.marklogic.mgmt.api.configuration.Configuration;
 import com.marklogic.mgmt.api.configuration.Configurations;
+import com.marklogic.mgmt.api.forest.Forest;
+import com.marklogic.mgmt.resource.forests.ForestManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -61,6 +66,36 @@ public void addCmaConfigurationToCombinedRequest(Configuration configuration) {
 		}
 	}
 
+	/**
+	 * Added to greatly speed up performance when getting details about all the existing primary forests for each
+	 * database in the cluster. In the event that anything fails while getting the forest details, the map won't be
+	 * added to the context and any code expecting to use the map will have to fall back to using /manage/v2.
+	 *
+	 * @since 4.5.3
+	 */
+	public Map<String, List<Forest>> getMapOfPrimaryForests() {
+		if (!appConfig.getCmaConfig().isDeployForests()) {
+			return null;
+		}
+		final String key = "ml-app-deployer-mapOfPrimaryForests";
+		if (contextMap.containsKey(key)) {
+			return (Map<String, List<Forest>>) contextMap.get(key);
+		}
+		Logger logger = LoggerFactory.getLogger(getClass());
+		try {
+			logger.info("Retrieving all forest details via CMA");
+			long start = System.currentTimeMillis();
+			Map<String, List<Forest>> mapOfPrimaryForests = new ForestManager(manageClient).getMapOfPrimaryForests();
+			logger.info("Finished retrieving all forests details via CMA; duration: " + (System.currentTimeMillis() - start));
+			contextMap.put(key, mapOfPrimaryForests);
+			return mapOfPrimaryForests;
+		} catch (Exception ex) {
+			logger.warn("Unable to retrieve all forest details, cause: " + ex.getMessage() + "; will fall back to " +
+				"using /manage/v2 when needed for getting details for a forest.");
+			return null;
+		}
+	}
+
 	public Configurations getCombinedCmaRequest() {
 		return (Configurations) contextMap.get(COMBINED_CMA_REQUEST_KEY);
 	}

src/main/java/com/marklogic/appdeployer/command/forests/ConfigureForestReplicasCommand.java

@@ -19,6 +19,8 @@
 import com.marklogic.appdeployer.command.CommandContext;
 import com.marklogic.appdeployer.command.SortOrderConstants;
 import com.marklogic.mgmt.api.API;
+import com.marklogic.mgmt.api.configuration.Configuration;
+import com.marklogic.mgmt.api.configuration.Configurations;
 import com.marklogic.mgmt.api.forest.Forest;
 import com.marklogic.mgmt.mapper.DefaultResourceMapper;
 import com.marklogic.mgmt.mapper.ResourceMapper;
@@ -28,7 +30,11 @@
 import com.marklogic.mgmt.resource.groups.GroupManager;
 import com.marklogic.mgmt.resource.hosts.HostManager;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
 
 /**
  * Command for configuring - i.e. creating and setting - replica forests for existing databases.
@@ -131,19 +137,37 @@ protected void configureDatabaseReplicaForests(String databaseName, int replicaC
 		List<String> dataDirectories = forestBuilder.determineDataDirectories(databaseName, context.getAppConfig());
 		forestBuilder.addReplicasToForests(forestsNeedingReplicas, forestPlan, context.getAppConfig(), dataDirectories);
 
-		// TODO Use CMA here in the future? Need to test to see if a forest name + replicas are allowable
-		ForestManager forestManager = new ForestManager(context.getManageClient());
-		for (Forest forest : forestsNeedingReplicas) {
-			final String forestName = forest.getForestName();
-
+		List<Forest> forestsWithOnlyReplicas = forestsNeedingReplicas.stream().map(forest -> {
 			Forest forestWithOnlyReplicas = new Forest();
+			forestWithOnlyReplicas.setForestName(forest.getForestName());
 			forestWithOnlyReplicas.setForestReplica(forest.getForestReplica());
-			String json = forestWithOnlyReplicas.getJson();
+			return forestWithOnlyReplicas;
+		}).collect(Collectors.toList());
+
+		// As of 4.5.3, try CMA first so that this can be done in a single request instead of one request per forest.
+		if (context.getAppConfig().getCmaConfig().isDeployForests()) {
+			try {
+				Configuration config = new Configuration();
+				forestsWithOnlyReplicas.forEach(forest -> {
+					config.addForest(forest.toObjectNode());
+				});
+				new Configurations(config).submit(context.getManageClient());
+				return;
+			} catch (Exception ex) {
+				logger.warn("Unable to create forest replicas via CMA; cause: " + ex.getMessage() + "; will " +
+					"fall back to using /manage/v2.");
+			}
+		}
 
+		// If we get here, either CMA usage is disabled or an error occurred with CMA. Just use /manage/v2 to submit
+		// each forest one-by-one.
+		ForestManager forestManager = new ForestManager(context.getManageClient());
+		forestsWithOnlyReplicas.forEach(forest -> {
+			String forestName = forest.getForestName();
 			logger.info(format("Creating forest replicas for primary forest %s", forestName));
-			context.getManageClient().putJson(forestManager.getPropertiesPath(forestName), json);
+			context.getManageClient().putJson(forestManager.getPropertiesPath(forestName), forest.getJson());
 			logger.info(format("Finished creating forest replicas for primary forest %s", forestName));
-		}
+		});
 	}
 
 	/**
@@ -161,22 +185,41 @@ protected List<Forest> determineForestsNeedingReplicas(String databaseName, Comm
 		ResourceMapper resourceMapper = new DefaultResourceMapper(api);
 
 		List<Forest> forestsNeedingReplicas = new ArrayList<>();
+		Map<String, List<Forest>> mapOfPrimaryForests = context.getMapOfPrimaryForests();
 
-		for (String forestName : dbMgr.getForestNames(databaseName)) {
-			logger.info(format("Checking the status of forest %s to determine if it is a primary forest and whether or not it has replicas already.", forestName));
-			ForestStatus status = forestManager.getForestStatus(forestName);
-			if (!status.isPrimary()) {
-				logger.info(format("Forest %s is not a primary forest, so not configuring replica forests", forestName));
-				continue;
-			}
-			if (status.hasReplicas()) {
-				logger.info(format("Forest %s already has replicas, so not configuring replica forests", forestName));
-				continue;
-			}
+		/**
+		 * In both blocks below, a forest is not included if it already has replicas. This logic dates back to 2015,
+		 * and is likely due to uncertainty over the various scenarios that can occur if a forest does already have
+		 * replicas. At least as of August 2023, MarkLogic recommends a single replica per forest. Given that no users
+		 * have asked for this check to not be performed and based on MarkLogic's recommendation, it seems reasonable
+		 * to leave this check in for now. However, some ad hoc testing has indicated that this check is unnecessary
+		 * and that it appears to safe to vary the number of replicas per forest. So it likely would be beneficial to
+		 * remove this check at some point.
+		 */
+		if (mapOfPrimaryForests != null && mapOfPrimaryForests.containsKey(databaseName)) {
+			mapOfPrimaryForests.get(databaseName).forEach(forest -> {
+				boolean forestHasReplicasAlready = forest.getForestReplica() != null && !forest.getForestReplica().isEmpty();
+				if (!forestHasReplicasAlready) {
+					forestsNeedingReplicas.add(forest);
+				}
+			});
+		} else {
+			for (String forestName : dbMgr.getForestNames(databaseName)) {
+				logger.info(format("Checking the status of forest %s to determine if it is a primary forest and whether or not it has replicas already.", forestName));
+				ForestStatus status = forestManager.getForestStatus(forestName);
+				if (!status.isPrimary()) {
+					logger.info(format("Forest %s is not a primary forest, so not configuring replica forests", forestName));
+					continue;
+				}
+				if (status.hasReplicas()) {
+					logger.info(format("Forest %s already has replicas, so not configuring replica forests", forestName));
+					continue;
+				}
 
-			String forestJson = forestManager.getPropertiesAsJson(forestName);
-			Forest forest = resourceMapper.readResource(forestJson, Forest.class);
-			forestsNeedingReplicas.add(forest);
+				String forestJson = forestManager.getPropertiesAsJson(forestName);
+				Forest forest = resourceMapper.readResource(forestJson, Forest.class);
+				forestsNeedingReplicas.add(forest);
+			}
 		}
 
 		return forestsNeedingReplicas;

src/main/java/com/marklogic/appdeployer/command/forests/DeployForestsCommand.java

@@ -128,12 +128,23 @@ protected void createForestsViaForestEndpoint(CommandContext context, List<Fores
 	 */
 	public List<Forest> buildForests(CommandContext context, boolean includeReplicas) {
 		// Need to know what primary forests exist already in case more need to be added, or a new host has been added
-		List<Forest> existingPrimaryForests = getExistingPrimaryForests(context, this.databaseName);
+		List<Forest> existingPrimaryForests = null;
+
+		// As of 4.5.3, if CMA is enabled, then the context should contain a map of all the forests for each database
+		// being deployed. If it's not there, then /manage/v2 will be used instead.
+		Map<String, List<Forest>> mapOfPrimaryForests = context.getMapOfPrimaryForests();
+		if (mapOfPrimaryForests != null && mapOfPrimaryForests.containsKey(this.databaseName)) {
+			existingPrimaryForests = mapOfPrimaryForests.get(this.databaseName);
+		}
+
+		if (existingPrimaryForests == null) {
+			existingPrimaryForests = getExistingPrimaryForests(context, this.databaseName);
+		}
+
 		return buildForests(context, includeReplicas, existingPrimaryForests);
 	}
 
 	/**
-	 *
 	 * @param context
 	 * @param includeReplicas
 	 * @param existingPrimaryForests
@@ -163,6 +174,14 @@ protected List<Forest> buildForests(CommandContext context, boolean includeRepli
 		return forestBuilder.buildForests(forestPlan, context.getAppConfig());
 	}
 
+	/**
+	 * @param context
+	 * @param databaseName
+	 * @return
+	 * @deprecated in 4.5.3, as getting forest details one at a time can be very slow for applications with a large
+	 * number of forests.
+	 */
+	@Deprecated
 	protected List<Forest> getExistingPrimaryForests(CommandContext context, String databaseName) {
 		List<String> forestIds = new DatabaseManager(context.getManageClient()).getPrimaryForestIds(databaseName);
 		ForestManager forestMgr = new ForestManager(context.getManageClient());
@@ -259,6 +278,7 @@ public boolean isCreateForestsOnEachHost() {
 
 	/**
 	 * Use appConfig.setDatabasesWithForestsOnOneHost
+	 *
 	 * @param createForestsOnEachHost
 	 */
 	@Deprecated