allow config of connection timeout and error limit

common/config/environment.go

@@ -34,6 +34,8 @@ const (
 	DDInfraExtraResourcesTags               = "extraResourcesTags"
 	DDInfraSSHUser                          = "sshUser"
 	DDInfraInitOnly                         = "initOnly"
+	DDInfraDialErrorLimit                   = "dialErrorLimit"
+	DDInfraPerDialTimeoutSeconds            = "perDialTimeoutSeconds"
 
 	// Agent Namespace
 	DDAgentDeployParamName               = "deploy"
@@ -244,6 +246,14 @@ func (e *CommonEnvironment) InfraSSHUser() string {
 	return e.GetStringWithDefault(e.InfraConfig, DDInfraSSHUser, "")
 }
 
+func (e *CommonEnvironment) InfraDialErrorLimit() int {
+	return e.GetIntWithDefault(e.InfraConfig, DDInfraDialErrorLimit, 0)
+}
+
+func (e *CommonEnvironment) InfraPerDialTimeoutSeconds() int {
+	return e.GetIntWithDefault(e.InfraConfig, DDInfraPerDialTimeoutSeconds, 0)
+}
+
 func EnvVariableResourceTags() map[string]string {
 	tags := map[string]string{}
 	lookupVars := []string{"TEAM", "PIPELINE_ID", "CI_PIPELINE_ID"}

components/remote/connection.go

@@ -7,12 +7,6 @@ import (
 	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
 )
 
-const (
-	// We retry every 5s maximum 100 times (~8 minutes)
-	dialTimeoutSeconds int = 5
-	dialErrorLimit     int = 100
-)
-
 // NewConnection creates a remote connection to a host.
 // Host and user are mandatory.
 func NewConnection(host pulumi.StringInput, user string, options ...ConnectionOption) (*remote.ConnectionArgs, error) {
@@ -24,8 +18,8 @@ func NewConnection(host pulumi.StringInput, user string, options ...ConnectionOp
 	conn := &remote.ConnectionArgs{
 		Host:           args.host,
 		User:           pulumi.String(args.user),
-		PerDialTimeout: pulumi.IntPtr(dialTimeoutSeconds),
-		DialErrorLimit: pulumi.IntPtr(dialErrorLimit),
+		PerDialTimeout: pulumi.IntPtr(args.perDialTimeoutSeconds),
+		DialErrorLimit: pulumi.IntPtr(args.dialErrorLimit),
 		Port:           pulumi.Float64Ptr(float64(args.port)),
 	}
 

components/remote/connectionargs.go

@@ -1,28 +1,33 @@
 package remote
 
 import (
-	"github.com/DataDog/test-infra-definitions/common"
 	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
+
+	"github.com/DataDog/test-infra-definitions/common"
 )
 
 type connectionArgs struct {
 	host pulumi.StringInput
 	user string
 
 	// ==== Optional ====
-	privateKeyPath     string
-	privateKeyPassword string
-	sshAgentPath       string
-	port               int
+	privateKeyPath        string
+	privateKeyPassword    string
+	sshAgentPath          string
+	port                  int
+	dialErrorLimit        int
+	perDialTimeoutSeconds int
 }
 
 type ConnectionOption = func(*connectionArgs) error
 
 func buildConnectionArgs(host pulumi.StringInput, user string, options ...ConnectionOption) (*connectionArgs, error) {
 	args := &connectionArgs{
-		host: host,
-		user: user,
-		port: 22,
+		host:                  host,
+		user:                  user,
+		port:                  22,
+		dialErrorLimit:        100,
+		perDialTimeoutSeconds: 5,
 	}
 	return common.ApplyOption(args, options)
 }
@@ -58,3 +63,23 @@ func WithPort(port int) ConnectionOption {
 		return nil
 	}
 }
+
+// WithDialErrorLimit [optional] sets the maximum dial attempts for the connection. Defaults to 100.
+func WithDialErrorLimit(limit int) ConnectionOption {
+	return func(args *connectionArgs) error {
+		if limit > 0 {
+			args.dialErrorLimit = limit
+		}
+		return nil
+	}
+}
+
+// WithPerDialTimeoutSeconds [optional] sets the per dial timeout in seconds for the connection. Defaults to 5.
+func WithPerDialTimeoutSeconds(seconds int) ConnectionOption {
+	return func(args *connectionArgs) error {
+		if seconds > 0 {
+			args.perDialTimeoutSeconds = seconds
+		}
+		return nil
+	}
+}

resources/hyperv/vm.go

@@ -37,6 +37,8 @@ func NewVM(e Environment, args VMArgs, opts ...pulumi.ResourceOption) (*remote.H
 			"<SSH_USER_NAME>",
 			remote.WithPrivateKeyPath(e.DefaultPrivateKeyPath()),
 			remote.WithPrivateKeyPassword(e.DefaultPrivateKeyPassword()),
+			remote.WithDialErrorLimit(e.InfraDialErrorLimit()),
+			remote.WithPerDialTimeoutSeconds(e.InfraPerDialTimeoutSeconds()),
 		)
 		if err != nil {
 			return err

scenarios/aws/ec2/vm.go

@@ -80,6 +80,8 @@ func NewVM(e aws.Environment, name string, params ...VMOption) (*remote.Host, er
 			sshUser,
 			remote.WithPrivateKeyPath(e.DefaultPrivateKeyPath()),
 			remote.WithPrivateKeyPassword(e.DefaultPrivateKeyPassword()),
+			remote.WithDialErrorLimit(e.InfraDialErrorLimit()),
+			remote.WithPerDialTimeoutSeconds(e.InfraPerDialTimeoutSeconds()),
 		)
 		if err != nil {
 			return err

scenarios/aws/microVMs/microvms/provision.go

@@ -9,6 +9,7 @@ import (
 	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
 	"github.com/pulumi/pulumi/sdk/v3/go/pulumi/config"
 
+	commonConfig "github.com/DataDog/test-infra-definitions/common/config"
 	"github.com/DataDog/test-infra-definitions/common/namer"
 	"github.com/DataDog/test-infra-definitions/components/command"
 	remoteComp "github.com/DataDog/test-infra-definitions/components/remote"
@@ -232,6 +233,8 @@ func provisionRemoteMicroVMs(vmCollections []*VMCollection, instanceEnv *Instanc
 					"ubuntu",
 					remoteComp.WithPrivateKeyPath(instanceEnv.DefaultPrivateKeyPath()),
 					remoteComp.WithPrivateKeyPassword(instanceEnv.DefaultPrivateKeyPassword()),
+					remoteComp.WithDialErrorLimit(instanceEnv.InfraDialErrorLimit()),
+					remoteComp.WithPerDialTimeoutSeconds(instanceEnv.InfraPerDialTimeoutSeconds()),
 				)
 				if err != nil {
 					return nil, err
@@ -276,7 +279,7 @@ func provisionRemoteMicroVMs(vmCollections []*VMCollection, instanceEnv *Instanc
 	return waitFor, nil
 }
 
-func provisionLocalMicroVMs(vmCollections []*VMCollection) ([]pulumi.Resource, error) {
+func provisionLocalMicroVMs(vmCollections []*VMCollection, e commonConfig.CommonEnvironment) ([]pulumi.Resource, error) {
 	var waitFor []pulumi.Resource
 	for _, collection := range vmCollections {
 		if collection.instance.Arch != LocalVMSet {
@@ -294,6 +297,8 @@ func provisionLocalMicroVMs(vmCollections []*VMCollection) ([]pulumi.Resource, e
 					domain.ip,
 					"root",
 					remoteComp.WithPrivateKeyPath(filepath.Join(GetWorkingDirectory(domain.vmset.Arch), "ddvm_rsa")),
+					remoteComp.WithDialErrorLimit(e.InfraDialErrorLimit()),
+					remoteComp.WithPerDialTimeoutSeconds(e.InfraPerDialTimeoutSeconds()),
 				)
 				if err != nil {
 					return nil, err

scenarios/aws/microVMs/microvms/run.go

@@ -365,7 +365,7 @@ func run(e commonConfig.CommonEnvironment) (*ScenarioDone, error) {
 			return nil, err
 		}
 
-		if _, err := provisionLocalMicroVMs(vmCollections); err != nil {
+		if _, err := provisionLocalMicroVMs(vmCollections, e); err != nil {
 			return nil, err
 		}
 	}

scenarios/azure/compute/vm.go

@@ -63,6 +63,8 @@ func NewVM(e azure.Environment, name string, params ...VMOption) (*remote.Host,
 			compute.AdminUsername,
 			remote.WithPrivateKeyPath(e.DefaultPrivateKeyPath()),
 			remote.WithPrivateKeyPassword(e.DefaultPrivateKeyPassword()),
+			remote.WithDialErrorLimit(e.InfraDialErrorLimit()),
+			remote.WithPerDialTimeoutSeconds(e.InfraPerDialTimeoutSeconds()),
 		)
 		if err != nil {
 			return err

scenarios/gcp/compute/vm.go

@@ -3,12 +3,13 @@ package compute
 import (
 	"fmt"
 
+	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
+
 	"github.com/DataDog/test-infra-definitions/components"
 	"github.com/DataDog/test-infra-definitions/components/command"
 	"github.com/DataDog/test-infra-definitions/components/os"
 	"github.com/DataDog/test-infra-definitions/components/remote"
 	"github.com/DataDog/test-infra-definitions/resources/gcp/compute"
-	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
 
 	"github.com/DataDog/test-infra-definitions/resources/gcp"
 )
@@ -40,6 +41,8 @@ func NewVM(e gcp.Environment, name string, option ...VMOption) (*remote.Host, er
 			"gce",
 			remote.WithPrivateKeyPath(e.DefaultPrivateKeyPath()),
 			remote.WithPrivateKeyPassword(e.DefaultPrivateKeyPassword()),
+			remote.WithDialErrorLimit(e.InfraDialErrorLimit()),
+			remote.WithPerDialTimeoutSeconds(e.InfraPerDialTimeoutSeconds()),
 		)
 		if err != nil {
 			return err