Synchronize Secret Format and Usage

CreateSnapshot/src/main/java/org/opensearch/migrations/CreateSnapshot.java

@@ -1,7 +1,10 @@
 package org.opensearch.migrations;
 
+import java.util.ArrayList;
 import java.util.List;
 
+import org.opensearch.migrations.arguments.ArgLogUtils;
+import org.opensearch.migrations.arguments.ArgNameConstants;
 import org.opensearch.migrations.bulkload.common.FileSystemSnapshotCreator;
 import org.opensearch.migrations.bulkload.common.OpenSearchClientFactory;
 import org.opensearch.migrations.bulkload.common.S3SnapshotCreator;
@@ -99,6 +102,28 @@ public static class Args {
         String otelCollectorEndpoint;
     }
 
+    public static class EnvParameters {
+
+        private EnvParameters() {
+            throw new IllegalStateException("EnvParameters utility class should not instantiated");
+        }
+
+        public static void injectFromEnv(Args args) {
+            List<String> addedEnvParams = new ArrayList<>();
+            if (args.sourceArgs.username == null && System.getenv(ArgNameConstants.SOURCE_USERNAME_ENV_ARG) != null) {
+                args.sourceArgs.username = System.getenv(ArgNameConstants.SOURCE_USERNAME_ENV_ARG);
+                addedEnvParams.add(ArgNameConstants.SOURCE_USERNAME_ENV_ARG);
+            }
+            if (args.sourceArgs.password == null && System.getenv(ArgNameConstants.SOURCE_PASSWORD_ENV_ARG) != null) {
+                args.sourceArgs.password = System.getenv(ArgNameConstants.SOURCE_PASSWORD_ENV_ARG);
+                addedEnvParams.add(ArgNameConstants.SOURCE_PASSWORD_ENV_ARG);
+            }
+            if (!addedEnvParams.isEmpty()) {
+                log.info("Adding parameters from the following expected environment variables: {}", addedEnvParams);
+            }
+        }
+    }
+
     @Getter
     @AllArgsConstructor
     public static class S3RepoInfo {
@@ -107,10 +132,11 @@ public static class S3RepoInfo {
     }
 
     public static void main(String[] args) throws Exception {
-        // TODO: Add back arg printing after not consuming plaintext password MIGRATIONS-1915
+        System.err.println("Starting program with: " + String.join(" ", ArgLogUtils.getRedactedArgs(args, ArgNameConstants.CENSORED_SOURCE_ARGS)));
         Args arguments = new Args();
         JCommander jCommander = JCommander.newBuilder().addObject(arguments).build();
         jCommander.parse(args);
+        EnvParameters.injectFromEnv(arguments);
 
         if (arguments.help) {
             jCommander.usage();

DocumentsFromSnapshotMigration/docker/entrypoint.sh

@@ -41,14 +41,6 @@ if [[ "$RFS_COMMAND" == *"--target-password"* || "$RFS_COMMAND" == *"--targetPas
 fi
 echo "Executing RFS_COMMAND: $SAFE_PRINT_COMMAND"
 
-if [ -n "$TARGET_USERNAME" ]; then
-  echo "TARGET_USERNAME is set by the environment."
-fi
-
-if [ -n "$TARGET_PASSWORD" ]; then
-  echo "TARGET_PASSWORD is set by the environment."
-fi
-
 # Extract the value passed after --s3-local-dir or --s3LocalDir
 S3_LOCAL_DIR=$(echo "$RFS_COMMAND" | sed -n 's/.*--\(s3-local-dir\|s3LocalDir\)\s\+\("[^"]\+"\|[^ ]\+\).*/\2/p' | tr -d '"')
 # Extract the value passed after --lucene-dir or --luceneDir

DocumentsFromSnapshotMigration/src/main/java/org/opensearch/migrations/RfsMigrateDocuments.java

@@ -15,6 +15,8 @@
 import java.util.function.Supplier;
 import java.util.regex.Pattern;
 
+import org.opensearch.migrations.arguments.ArgLogUtils;
+import org.opensearch.migrations.arguments.ArgNameConstants;
 import org.opensearch.migrations.bulkload.common.DefaultSourceRepoAccessor;
 import org.opensearch.migrations.bulkload.common.DocumentReindexer;
 import org.opensearch.migrations.bulkload.common.FileSystemRepo;
@@ -48,7 +50,6 @@
 import org.opensearch.migrations.transform.TransformationLoader;
 import org.opensearch.migrations.transform.TransformerConfigUtils;
 import org.opensearch.migrations.transform.TransformerParams;
-import org.opensearch.migrations.utils.ArgLogUtils;
 import org.opensearch.migrations.utils.ProcessHelpers;
 
 import com.beust.jcommander.IStringConverter;
@@ -69,8 +70,6 @@ public class RfsMigrateDocuments {
     public static final int NO_WORK_LEFT_EXIT_CODE = 3;
     public static final int TOLERABLE_CLIENT_SERVER_CLOCK_DIFFERENCE_SECONDS = 5;
     public static final String LOGGING_MDC_WORKER_ID = "workerId";
-    public static final String TARGET_USERNAME_ENV_VAR = "TARGET_USERNAME";
-    public static final String TARGET_PASSWORD_ENV_VAR = "TARGET_PASSWORD";
 
     // Decrease successor nextAcquisitionLeaseExponent if shard setup takes less than 2.5% of total lease time
     // Increase successor nextAcquisitionLeaseExponent if shard setup takes more than 10% of lease total time
@@ -265,13 +264,13 @@ private EnvParameters() {
 
         public static void injectFromEnv(Args args) {
             List<String> addedEnvParams = new ArrayList<>();
-            if (args.targetArgs.username == null && System.getenv(TARGET_USERNAME_ENV_VAR) != null) {
-                args.targetArgs.username = System.getenv(TARGET_USERNAME_ENV_VAR);
-                addedEnvParams.add(TARGET_USERNAME_ENV_VAR);
+            if (args.targetArgs.username == null && System.getenv(ArgNameConstants.TARGET_USERNAME_ENV_ARG) != null) {
+                args.targetArgs.username = System.getenv(ArgNameConstants.TARGET_USERNAME_ENV_ARG);
+                addedEnvParams.add(ArgNameConstants.TARGET_USERNAME_ENV_ARG);
             }
-            if (args.targetArgs.password == null && System.getenv(TARGET_PASSWORD_ENV_VAR) != null) {
-                args.targetArgs.password = System.getenv(TARGET_PASSWORD_ENV_VAR);
-                addedEnvParams.add(TARGET_PASSWORD_ENV_VAR);
+            if (args.targetArgs.password == null && System.getenv(ArgNameConstants.TARGET_PASSWORD_ENV_ARG) != null) {
+                args.targetArgs.password = System.getenv(ArgNameConstants.TARGET_PASSWORD_ENV_ARG);
+                addedEnvParams.add(ArgNameConstants.TARGET_PASSWORD_ENV_ARG);
             }
             if (!addedEnvParams.isEmpty()) {
                 log.info("Adding parameters from the following expected environment variables: {}", addedEnvParams);
@@ -312,7 +311,7 @@ public static void validateArgs(Args args) {
 
     public static void main(String[] args) throws Exception {
         var workerId = ProcessHelpers.getNodeInstanceName();
-        System.err.println("Starting program with: " + String.join(" ", ArgLogUtils.getRedactedArgs(args)));
+        System.err.println("Starting program with: " + String.join(" ", ArgLogUtils.getRedactedArgs(args, ArgNameConstants.CENSORED_TARGET_ARGS)));
         // Ensure that log4j2 doesn't execute shutdown hooks until ours have completed. This means that we need to take
         // responsibility for calling `LogManager.shutdown()` in our own shutdown hook..
         System.setProperty("log4j2.shutdownHookEnabled", "false");

MetadataMigration/src/main/java/org/opensearch/migrations/MetadataMigration.java

@@ -2,12 +2,9 @@
 
 import java.util.Optional;
 
-import org.opensearch.migrations.commands.Configure;
-import org.opensearch.migrations.commands.Evaluate;
-import org.opensearch.migrations.commands.EvaluateArgs;
-import org.opensearch.migrations.commands.Migrate;
-import org.opensearch.migrations.commands.MigrateArgs;
-import org.opensearch.migrations.commands.Result;
+import org.opensearch.migrations.arguments.ArgLogUtils;
+import org.opensearch.migrations.arguments.ArgNameConstants;
+import org.opensearch.migrations.commands.*;
 import org.opensearch.migrations.metadata.tracing.RootMetadataMigrationContext;
 import org.opensearch.migrations.tracing.ActiveContextTracker;
 import org.opensearch.migrations.tracing.ActiveContextTrackerByActivityType;
@@ -22,6 +19,10 @@
 public class MetadataMigration {
 
     public static void main(String[] args) throws Exception {
+        System.err.println("Starting program with: " + String.join(" ", ArgLogUtils.getRedactedArgs(
+                args,
+                ArgNameConstants.joinLists(ArgNameConstants.CENSORED_SOURCE_ARGS, ArgNameConstants.CENSORED_TARGET_ARGS)
+        )));
         var metadataArgs = new MetadataArgs();
         var migrateArgs = new MigrateArgs();
         var evaluateArgs = new EvaluateArgs();
@@ -31,6 +32,8 @@ public static void main(String[] args) throws Exception {
             .addCommand(evaluateArgs)
             .build();
         jCommander.parse(args);
+        EnvArgs.injectFromEnv(migrateArgs);
+        EnvArgs.injectFromEnv(evaluateArgs);
 
         var context = new RootMetadataMigrationContext(
             RootOtelContext.initializeOpenTelemetryWithCollectorOrAsNoop(metadataArgs.otelCollectorEndpoint, "metadata",
@@ -40,8 +43,6 @@ public static void main(String[] args) throws Exception {
 
         var meta = new MetadataMigration();
 
-        // TODO: Add back arg printing after not consuming plaintext password MIGRATIONS-1915
-
         if (metadataArgs.help || jCommander.getParsedCommand() == null) {
             printTopLevelHelp(jCommander);
             return;

MetadataMigration/src/main/java/org/opensearch/migrations/commands/EnvArgs.java

@@ -0,0 +1,40 @@
+package org.opensearch.migrations.commands;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.opensearch.migrations.MigrateOrEvaluateArgs;
+import org.opensearch.migrations.arguments.ArgNameConstants;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class EnvArgs {
+
+    private EnvArgs() {
+        throw new IllegalStateException("EnvArgs utility class should not instantiated");
+    }
+
+    public static void injectFromEnv(MigrateOrEvaluateArgs args) {
+        List<String> addedEnvParams = new ArrayList<>();
+        if (args.sourceArgs.username == null && System.getenv(ArgNameConstants.SOURCE_USERNAME_ENV_ARG) != null) {
+            args.sourceArgs.username = System.getenv(ArgNameConstants.SOURCE_USERNAME_ENV_ARG);
+            addedEnvParams.add(ArgNameConstants.SOURCE_USERNAME_ENV_ARG);
+        }
+        if (args.sourceArgs.password == null && System.getenv(ArgNameConstants.SOURCE_PASSWORD_ENV_ARG) != null) {
+            args.sourceArgs.password = System.getenv(ArgNameConstants.SOURCE_PASSWORD_ENV_ARG);
+            addedEnvParams.add(ArgNameConstants.SOURCE_PASSWORD_ENV_ARG);
+        }
+        if (args.targetArgs.username == null && System.getenv(ArgNameConstants.TARGET_USERNAME_ENV_ARG) != null) {
+            args.targetArgs.username = System.getenv(ArgNameConstants.TARGET_USERNAME_ENV_ARG);
+            addedEnvParams.add(ArgNameConstants.TARGET_USERNAME_ENV_ARG);
+        }
+        if (args.targetArgs.password == null && System.getenv(ArgNameConstants.TARGET_PASSWORD_ENV_ARG) != null) {
+            args.targetArgs.password = System.getenv(ArgNameConstants.TARGET_PASSWORD_ENV_ARG);
+            addedEnvParams.add(ArgNameConstants.TARGET_PASSWORD_ENV_ARG);
+        }
+        if (!addedEnvParams.isEmpty()) {
+            log.info("Adding parameters from the following expected environment variables: {}", addedEnvParams);
+        }
+    }
+}

RFS/src/main/java/org/opensearch/migrations/bulkload/common/http/ConnectionContext.java

@@ -5,6 +5,8 @@
 import java.nio.file.Path;
 import java.time.Clock;
 
+import org.opensearch.migrations.arguments.ArgNameConstants;
+
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.ParametersDelegate;
 import com.beust.jcommander.converters.PathConverter;
@@ -137,13 +139,13 @@ public static class TargetArgs implements IParams {
         public String host;
 
         @Parameter(
-            names = {"--target-username", "--targetUsername" },
+            names = {ArgNameConstants.TARGET_USERNAME_ARG_CAMEL_CASE, ArgNameConstants.TARGET_USERNAME_ARG_KEBAB_CASE },
             description = "Optional.  The target username; if not provided, will assume no auth on target",
             required = false)
         public String username = null;
 
         @Parameter(
-            names = {"--target-password", "--targetPassword" },
+            names = {ArgNameConstants.TARGET_PASSWORD_ARG_CAMEL_CASE, ArgNameConstants.TARGET_PASSWORD_ARG_KEBAB_CASE },
             description = "Optional.  The target password; if not provided, will assume no auth on target",
             required = false)
         public String password = null;
@@ -215,13 +217,13 @@ public static class SourceArgs implements IParams {
         public String host = null;
 
         @Parameter(
-            names = {"--source-username", "--sourceUsername" },
+            names = {ArgNameConstants.SOURCE_USERNAME_ARG_CAMEL_CASE, ArgNameConstants.SOURCE_USERNAME_ARG_KEBAB_CASE },
             description = "The source username; if not provided, will assume no auth on source",
             required = false)
         public String username = null;
 
         @Parameter(
-            names = {"--source-password", "--sourcePassword" },
+            names = {ArgNameConstants.SOURCE_PASSWORD_ARG_CAMEL_CASE, ArgNameConstants.SOURCE_PASSWORD_ARG_KEBAB_CASE },
             description = "The source password; if not provided, will assume no auth on source",
             required = false)
         public String password = null;

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/cluster.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, Generator, Optional
+from typing import Any, Dict, Generator, NamedTuple, Optional
 from enum import Enum
 import json
 import logging
@@ -86,6 +86,11 @@ def validate_basic_auth_options(field, value, error):
 }
 
 
+class AuthDetails(NamedTuple):
+    username: str
+    password: str
+
+
 class Cluster:
     """
     An elasticcsearch or opensearch cluster.
@@ -119,14 +124,14 @@ def __init__(self, config: Dict, client_options: Optional[ClientOptions] = None)
             self.auth_details = config["sigv4"] if config["sigv4"] is not None else {}
         self.client_options = client_options
 
-    def get_basic_auth_details(self) -> tuple[str, str]:
+    def get_basic_auth_details(self) -> AuthDetails:
         """Return a tuple of (username, password) for basic auth. Will use username/password if provided in plaintext,
         otherwise will pull both username/password as keys in the specified secrets manager secret.
         """
         assert self.auth_type == AuthMethod.BASIC_AUTH
         assert self.auth_details is not None  # for mypy's sake
         if "username" in self.auth_details and "password" in self.auth_details:
-            return (self.auth_details["username"], self.auth_details["password"])
+            return AuthDetails(username=self.auth_details["username"], password=self.auth_details["password"])
         # Pull password from AWS Secrets Manager
         assert "user_secret_arn" in self.auth_details  # for mypy's sake
         client = create_boto3_client(aws_service_name="secretsmanager", client_options=self.client_options)
@@ -143,7 +148,7 @@ def get_basic_auth_details(self) -> tuple[str, str]:
                 f"Secret is missing required key(s): {', '.join(missing_keys)}"
             )
 
-        return (secret_dict["username"], secret_dict["password"])
+        return AuthDetails(username=secret_dict["username"], password=secret_dict["password"])
 
     def _get_sigv4_details(self, force_region=False) -> tuple[str, Optional[str]]:
         """Return the service signing name and region name. If force_region is true,
@@ -159,8 +164,8 @@ def _get_sigv4_details(self, force_region=False) -> tuple[str, Optional[str]]:
     def _generate_auth_object(self) -> requests.auth.AuthBase | None:
         if self.auth_type == AuthMethod.BASIC_AUTH:
             assert self.auth_details is not None  # for mypy's sake
-            username, password = self.get_basic_auth_details()
-            return HTTPBasicAuth(username, password)
+            auth_details = self.get_basic_auth_details()
+            return HTTPBasicAuth(auth_details.username, auth_details.password)
         elif self.auth_type == AuthMethod.SIGV4:
             service_name, region_name = self._get_sigv4_details(force_region=True)
             return SigV4AuthPlugin(service_name, region_name)
@@ -209,7 +214,9 @@ def execute_benchmark_workload(self, workload: str,
             client_options += ",use_ssl:true"
         password_to_censor = ""
         if self.auth_type == AuthMethod.BASIC_AUTH:
-            username, password_to_censor = self.get_basic_auth_details()
+            auth_details = self.get_basic_auth_details()
+            username = auth_details.username
+            password_to_censor = auth_details.password
             client_options += (f",basic_auth_user:{username},"
                                f"basic_auth_password:{password_to_censor}")
         elif self.auth_type == AuthMethod.SIGV4:

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/metadata.py

@@ -201,10 +201,10 @@ def migrate_or_evaluate(self, command: str, extra_args=None) -> CommandResult:
 
         if self._target_cluster.auth_type == AuthMethod.BASIC_AUTH:
             try:
-                username, password = self._target_cluster.get_basic_auth_details()
+                auth_details = self._target_cluster.get_basic_auth_details()
                 command_args.update({
-                    "--target-username": username,
-                    "--target-password": password
+                    "--target-username": auth_details.username,
+                    "--target-password": auth_details.password
                 })
                 logger.info("Using basic auth for target cluster")
             except KeyError as e:

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/snapshot.py

@@ -91,10 +91,10 @@ def _collect_universal_command_args(self) -> Dict:
 
         if self.source_cluster.auth_type == AuthMethod.BASIC_AUTH:
             try:
-                username, password = self.source_cluster.get_basic_auth_details()
+                auth_details = self.source_cluster.get_basic_auth_details()
                 command_args.update({
-                    "--source-username": username,
-                    "--source-password": password
+                    "--source-username": auth_details.username,
+                    "--source-password": auth_details.password
                 })
                 logger.info("Using basic auth for source cluster")
             except KeyError as e: