#203 New properties for configuring the data/fast/large forest directories

Author: rjrudin

src/main/java/com/marklogic/appdeployer/AppConfig.java

@@ -149,6 +149,19 @@ public class AppConfig {
     // Comma-delimited string used for configuring forest replicas
     private String databaseNamesAndReplicaCounts;
 
+    // Data/fast/large directories for default forests
+    private String forestDataDirectory;
+    private String forestFastDataDirectory;
+    private String forestLargeDataDirectory;
+
+    // Comma-delimited string of database names and data directories
+    private Map<String, String> databaseDataDirectories;
+    private Map<String, String> databaseFastDataDirectories;
+    private Map<String, String> databaseLargeDataDirectories;
+    private Map<String, String> databaseReplicaDataDirectories;
+    private Map<String, String> databaseReplicaFastDataDirectories;
+    private Map<String, String> databaseReplicaLargeDataDirectories;
+
     // Configures the data-directory for replica forests built dynamically
     private String replicaForestDataDirectory;
     private String replicaForestLargeDataDirectory;
@@ -942,4 +955,76 @@ public int getModulesLoaderThreadCount() {
 	public void setModulesLoaderThreadCount(int modulesLoaderThreadCount) {
 		this.modulesLoaderThreadCount = modulesLoaderThreadCount;
 	}
+
+	public Map<String, String> getDatabaseDataDirectories() {
+		return databaseDataDirectories;
+	}
+
+	public void setDatabaseDataDirectories(Map<String, String> databaseDataDirectories) {
+		this.databaseDataDirectories = databaseDataDirectories;
+	}
+
+	public Map<String, String> getDatabaseFastDataDirectories() {
+		return databaseFastDataDirectories;
+	}
+
+	public void setDatabaseFastDataDirectories(Map<String, String> databaseFastDataDirectories) {
+		this.databaseFastDataDirectories = databaseFastDataDirectories;
+	}
+
+	public Map<String, String> getDatabaseLargeDataDirectories() {
+		return databaseLargeDataDirectories;
+	}
+
+	public void setDatabaseLargeDataDirectories(Map<String, String> databaseLargeDataDirectories) {
+		this.databaseLargeDataDirectories = databaseLargeDataDirectories;
+	}
+
+	public String getForestDataDirectory() {
+		return forestDataDirectory;
+	}
+
+	public void setForestDataDirectory(String forestDataDirectory) {
+		this.forestDataDirectory = forestDataDirectory;
+	}
+
+	public String getForestFastDataDirectory() {
+		return forestFastDataDirectory;
+	}
+
+	public void setForestFastDataDirectory(String forestFastDataDirectory) {
+		this.forestFastDataDirectory = forestFastDataDirectory;
+	}
+
+	public String getForestLargeDataDirectory() {
+		return forestLargeDataDirectory;
+	}
+
+	public void setForestLargeDataDirectory(String forestLargeDataDirectory) {
+		this.forestLargeDataDirectory = forestLargeDataDirectory;
+	}
+
+	public Map<String, String> getDatabaseReplicaDataDirectories() {
+		return databaseReplicaDataDirectories;
+	}
+
+	public void setDatabaseReplicaDataDirectories(Map<String, String> databaseReplicaDataDirectories) {
+		this.databaseReplicaDataDirectories = databaseReplicaDataDirectories;
+	}
+
+	public Map<String, String> getDatabaseReplicaFastDataDirectories() {
+		return databaseReplicaFastDataDirectories;
+	}
+
+	public void setDatabaseReplicaFastDataDirectories(Map<String, String> databaseReplicaFastDataDirectories) {
+		this.databaseReplicaFastDataDirectories = databaseReplicaFastDataDirectories;
+	}
+
+	public Map<String, String> getDatabaseReplicaLargeDataDirectories() {
+		return databaseReplicaLargeDataDirectories;
+	}
+
+	public void setDatabaseReplicaLargeDataDirectories(Map<String, String> databaseReplicaLargeDataDirectories) {
+		this.databaseReplicaLargeDataDirectories = databaseReplicaLargeDataDirectories;
+	}
 }

src/main/java/com/marklogic/appdeployer/DefaultAppConfigFactory.java

@@ -6,9 +6,7 @@
 import com.marklogic.mgmt.util.PropertySourceFactory;
 
 import java.io.File;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
+import java.util.*;
 import java.util.regex.Pattern;
 
 public class DefaultAppConfigFactory extends PropertySourceFactory implements AppConfigFactory {
@@ -266,24 +264,78 @@ public AppConfig newAppConfig() {
 			c.setDatabaseNamesAndReplicaCounts(prop);
 		}
 
+		prop = getProperty("mlForestDataDirectory");
+		if (prop != null) {
+			logger.info("Default forest data directory for all databases: " + prop);
+			c.setForestDataDirectory(prop);
+		}
+
+		prop = getProperty("mlForestFastDataDirectory");
+		if (prop != null) {
+			logger.info("Default forest fast data directory for all databases: " + prop);
+			c.setForestFastDataDirectory(prop);
+		}
+
+		prop = getProperty("mlForestLargeDataDirectory");
+		if (prop != null) {
+			logger.info("Default forest large data directory for all databases: " + prop);
+			c.setForestLargeDataDirectory(prop);
+		}
+
 		prop = getProperty("mlReplicaForestDataDirectory");
 		if (prop != null) {
-			logger.info("Replica forest data directory " + prop);
+			logger.info("Default replica forest data directory for all databases: " + prop);
 			c.setReplicaForestDataDirectory(prop);
 		}
 
 		prop = getProperty("mlReplicaForestLargeDataDirectory");
 		if (prop != null) {
-			logger.info("Replica forest large data directory " + prop);
+			logger.info("Default replica forest large data directory for all databases: " + prop);
 			c.setReplicaForestLargeDataDirectory(prop);
 		}
 
 		prop = getProperty("mlReplicaForestFastDataDirectory");
 		if (prop != null) {
-			logger.info("Replica forest fast data directory " + prop);
+			logger.info("Default replica forest fast data directory for all databases: " + prop);
 			c.setReplicaForestFastDataDirectory(prop);
 		}
 
+		prop = getProperty("mlDatabaseDataDirectories");
+		if (prop != null) {
+			logger.info("Databases and forest data directories: " + prop);
+			c.setDatabaseDataDirectories(buildMapFromCommaDelimitedString(prop));
+		}
+
+		prop = getProperty("mlDatabaseFastDataDirectories");
+		if (prop != null) {
+			logger.info("Databases and forest fast data directories: " + prop);
+			c.setDatabaseFastDataDirectories(buildMapFromCommaDelimitedString(prop));
+		}
+
+		prop = getProperty("mlDatabaseLargeDataDirectories");
+		if (prop != null) {
+			logger.info("Databases and forest large data directories: " + prop);
+			c.setDatabaseLargeDataDirectories(buildMapFromCommaDelimitedString(prop));
+		}
+
+		prop = getProperty("mlDatabaseReplicaDataDirectories");
+		if (prop != null) {
+			logger.info("Databases and replica forest data directories: " + prop);
+			c.setDatabaseReplicaDataDirectories(buildMapFromCommaDelimitedString(prop));
+		}
+
+		prop = getProperty("mlDatabaseReplicaFastDataDirectories");
+		if (prop != null) {
+			logger.info("Databases and replica forest fast data directories: " + prop);
+			c.setDatabaseReplicaFastDataDirectories(buildMapFromCommaDelimitedString(prop));
+		}
+
+		prop = getProperty("mlDatabaseReplicaLargeDataDirectories");
+		if (prop != null) {
+			logger.info("Databases and replica forest large data directories: " + prop);
+			c.setDatabaseReplicaLargeDataDirectories(buildMapFromCommaDelimitedString(prop));
+		}
+
 		/**
 		 * When undo is invoked on DeployDatabaseCommand (such as via mlUndeploy in ml-gradle), this controls whether
 		 * or not forests are deleted, or just their configuration is deleted. If mlDeleteReplicas is set to true, this
@@ -544,4 +596,12 @@ public AppConfig newAppConfig() {
 		return c;
 	}
 
+	protected Map<String, String> buildMapFromCommaDelimitedString(String str) {
+		Map<String, String> map = new HashMap<>();
+		String[] tokens = str.split(",");
+		for (int i = 0; i < tokens.length; i += 2) {
+			map.put(tokens[i], tokens[i + 1]);
+		}
+		return map;
+	}
 }

src/main/java/com/marklogic/appdeployer/command/databases/DeployDatabaseCommand.java

@@ -8,7 +8,9 @@
 import com.marklogic.appdeployer.command.forests.DeployForestsCommand;
 import com.marklogic.mgmt.PayloadParser;
 import com.marklogic.mgmt.SaveReceipt;
+import com.marklogic.mgmt.api.forest.Forest;
 import com.marklogic.mgmt.resource.databases.DatabaseManager;
+import com.marklogic.mgmt.util.ObjectMapperFactory;
 
 import java.io.File;
 import java.util.ArrayList;
@@ -289,15 +291,54 @@ protected boolean customForestsExist(CommandContext context, String dbName) {
      */
     protected DeployForestsCommand buildDeployForestsCommand(String dbPayload, SaveReceipt receipt,
             CommandContext context) {
+    	final String databaseName = receipt.getResourceId();
+
         DeployForestsCommand c = new DeployForestsCommand();
         c.setCreateForestsOnEachHost(createForestsOnEachHost);
         c.setForestsPerHost(determineForestCountPerHost(dbPayload, context));
         c.setForestFilename(forestFilename);
-        c.setDatabaseName(receipt.getResourceId());
-        c.setForestPayload(DeployForestsCommand.DEFAULT_FOREST_PAYLOAD);
+        c.setDatabaseName(databaseName);
+
+        Forest forest = buildForest(context.getAppConfig());
+        forest.setObjectMapper(ObjectMapperFactory.getObjectMapper());
+	    c.setForestPayload(forest.getJson());
         return c;
     }
 
+    protected Forest buildForest(AppConfig config) {
+	    Forest forest = new Forest();
+	    forest.setForestName("%%FOREST_NAME%%");
+	    forest.setHost("%%FOREST_HOST%%");
+	    forest.setDatabase("%%FOREST_DATABASE%%");
+
+	    // First see if we have any database-agnostic forest directories
+	    if (config.getForestDataDirectory() != null) {
+	    	forest.setDataDirectory(config.getForestDataDirectory());
+	    }
+	    if (config.getForestFastDataDirectory() != null) {
+	    	forest.setFastDataDirectory(config.getForestFastDataDirectory());
+	    }
+	    if (config.getForestLargeDataDirectory() != null) {
+	    	forest.setLargeDataDirectory(config.getForestLargeDataDirectory());
+	    }
+
+	    // Now check for database-specific forest directories
+	    Map<String, String> map = config.getDatabaseDataDirectories();
+	    if (map != null && map.containsKey(databaseName)) {
+		    forest.setDataDirectory(map.get(databaseName));
+	    }
+	    map = config.getDatabaseFastDataDirectories();
+	    if (map != null && map.containsKey(databaseName)) {
+		    forest.setFastDataDirectory(map.get(databaseName));
+	    }
+	    map = config.getDatabaseLargeDataDirectories();
+	    if (map != null && map.containsKey(databaseName)) {
+		    forest.setLargeDataDirectory(map.get(databaseName));
+	    }
+
+	    return forest;
+    }
+
     /**
      * Checks the forestCounts map in AppConfig to see if the client has specified a number of forests per host for this
      * database.

src/main/java/com/marklogic/appdeployer/command/forests/ConfigureForestReplicasCommand.java

@@ -89,7 +89,7 @@ public void execute(CommandContext context) {
 		for (String forestName : forestNamesAndReplicaCounts.keySet()) {
 			int replicaCount = forestNamesAndReplicaCounts.get(forestName);
 			if (replicaCount > 0) {
-				configureReplicaForests(forestName, replicaCount, hostIds, context, forestMgr);
+				configureReplicaForests(null, forestName, replicaCount, hostIds, context, forestMgr);
 			}
 		}
 	}
@@ -148,21 +148,22 @@ protected void configureDatabaseReplicaForests(String databaseName, int replicaC
 		DatabaseManager dbMgr = new DatabaseManager(context.getManageClient());
 		List<String> forestNames = dbMgr.getForestNames(databaseName);
 		for (String name : forestNames) {
-			configureReplicaForests(name, replicaCount, hostIds, context, forestMgr);
+			configureReplicaForests(databaseName, name, replicaCount, hostIds, context, forestMgr);
 		}
 	}
 
 	/**
 	 * Creates forests as needed (they may already exists) and then sets those forests as the replicas for the given
 	 * primaryForestName.
 	 *
+	 * @param databaseName
 	 * @param forestIdOrName
 	 * @param replicaCount
 	 * @param hostIds
 	 * @param context
 	 * @param forestMgr
 	 */
-	protected void configureReplicaForests(String forestIdOrName, int replicaCount, List<String> hostIds,
+	protected void configureReplicaForests(String databaseName, String forestIdOrName, int replicaCount, List<String> hostIds,
 	                                       CommandContext context, ForestManager forestMgr) {
 		ForestStatus status = forestMgr.getForestStatus(forestIdOrName);
 		if (!status.isPrimary()) {
@@ -175,14 +176,15 @@ protected void configureReplicaForests(String forestIdOrName, int replicaCount,
 		}
 
 		logger.info(format("Creating forest replicas for primary forest %s", forestIdOrName));
-		createReplicaForests(forestIdOrName, replicaCount, hostIds, context, forestMgr);
+		createReplicaForests(databaseName, forestIdOrName, replicaCount, hostIds, context, forestMgr);
 		logger.info(format("Finished creating forest replicas for primary forest %s", forestIdOrName));
 	}
 
 	/**
 	 * Finds the host that the forest is on, and then starting with the next host in the list of host IDs,
 	 * creates N replicas.
 	 *
+	 * @param databaseName
 	 * @param forestIdOrName
 	 * @param replicaCount
 	 * @param hostIds
@@ -191,7 +193,7 @@ protected void configureReplicaForests(String forestIdOrName, int replicaCount,
 	 * @return a map where the keys are replica forest names, and the value of each key is the ID of the host that
 	 * the replica was created on
 	 */
-	protected Map<String, String> createReplicaForests(String forestIdOrName, int replicaCount, List<String> hostIds,
+	protected Map<String, String> createReplicaForests(String databaseName, String forestIdOrName, int replicaCount, List<String> hostIds,
 	                                                   CommandContext context, ForestManager forestMgr) {
 
 		// Using the Forest class to generate JSON
@@ -213,7 +215,7 @@ protected Map<String, String> createReplicaForests(String forestIdOrName, int re
 					}
 					String replicaHostId = hostIds.get(nextReplicaHostIndex);
 					String name = forestIdOrName + "-replica-" + j;
-					replicas.add(buildForestReplica(name, replicaHostId, context.getAppConfig()));
+					replicas.add(buildForestReplica(databaseName, name, replicaHostId, context.getAppConfig()));
 					replicaNamesAndHostIds.put(name, replicaHostId);
 					nextReplicaHostIndex++;
 				}
@@ -225,13 +227,40 @@ protected Map<String, String> createReplicaForests(String forestIdOrName, int re
 		return replicaNamesAndHostIds;
 	}
 
-	protected ForestReplica buildForestReplica(String name, String replicaHostId, AppConfig appConfig) {
+	/**
+	 * Return a ForestReplica instance with its properties configured based on what's in AppConfig.
+	 *
+	 * @param databaseName
+	 * @param name
+	 * @param replicaHostId
+	 * @param appConfig
+	 * @return
+	 */
+	protected ForestReplica buildForestReplica(String databaseName, String name, String replicaHostId, AppConfig appConfig) {
 		ForestReplica replica = new ForestReplica();
 		replica.setHost(replicaHostId);
 		replica.setReplicaName(name);
 		replica.setDataDirectory(appConfig.getReplicaForestDataDirectory());
 		replica.setLargeDataDirectory(appConfig.getReplicaForestLargeDataDirectory());
 		replica.setFastDataDirectory(appConfig.getReplicaForestFastDataDirectory());
+
+		if (databaseName != null) {
+			Map<String, String> map = appConfig.getDatabaseReplicaDataDirectories();
+			if (map != null && map.containsKey(databaseName)) {
+				replica.setDataDirectory(map.get(databaseName));
+			}
+
+			map = appConfig.getDatabaseReplicaFastDataDirectories();
+			if (map != null && map.containsKey(databaseName)) {
+				replica.setFastDataDirectory(map.get(databaseName));
+			}
+
+			map = appConfig.getDatabaseReplicaLargeDataDirectories();
+			if (map != null && map.containsKey(databaseName)) {
+				replica.setLargeDataDirectory(map.get(databaseName));
+			}
+		}
+
 		return replica;
 	}