feat: allow multiple environments per VPC via new service discovery namespace (#2515)

This PR modifies the default service discovery namespace for new environments to the form {env}.{app}.local. This change allows customers to place multiple environments in the same VPC.

The PR also creates an upgrade path for older environments. Those upgraded environments will retain the default app.local namespace to avoid a backward breaking change for services deployed there, but new environments will use env.app.local.

<img width="761" alt="Screen Shot 2021-06-24 at 3 56 02 PM" src="https://user-images.githubusercontent.com/25392995/123342814-50146a00-d505-11eb-8749-41e409150999.png">


In the image above, I have deployed the same service to 3 environments.
legacy was created at environment version 1.4.0. and upgraded to 1.5.0.
new was created at version 1.5.0
shared-with-legacy was created at 1.5.0 and shares a vpc and subnets with legacy.

There currently is not an upgrade path for existing environments to switch to new service discovery. This should not be an issue in most cases, however, as the environment variable points to the correct namespace in all cases and new+old environments can coexist in the same VPC.

This change does not disrupt any existing environments. If we eventually add an "environment v2.0.0," we can force migration to the new discovery service by removing the UseLegacyServiceDiscoveryIfBlank parameter and updating our workloads to remove the LegacyServiceDiscovery bool from the template package.

Updated integ tests to reflect new endpoints. Manually tested the above by running `copilot svc exec -n fe -e $ENV` for each of the three environments, then running `curl fe.$COPILOT_SVC_DISCOVERY_ENDPOINT` from within the container. Each time, I received the expected HTML content. 

Related #2377

By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.

Author: bvtujo

e2e/customized-env/customized_env_test.go

@@ -76,8 +76,9 @@ var _ = Describe("Customized Env", func() {
 
 	Context("when adding cross account environments", func() {
 		var (
-			testEnvInitErr error
-			prodEnvInitErr error
+			testEnvInitErr   error
+			prodEnvInitErr   error
+			sharedEnvInitErr error
 		)
 		BeforeAll(func() {
 			_, testEnvInitErr = cli.EnvInit(&client.EnvInitRequest{
@@ -96,17 +97,26 @@ var _ = Describe("Customized Env", func() {
 				VPCConfig:     vpcConfig,
 				CustomizedEnv: true,
 			})
+			_, sharedEnvInitErr = cli.EnvInit(&client.EnvInitRequest{
+				AppName:       appName,
+				EnvName:       "shared",
+				Profile:       "default",
+				Prod:          false,
+				VPCImport:     vpcImport,
+				CustomizedEnv: true,
+			})
 		})
 
-		It("env init should succeed for test and prod envs", func() {
+		It("env init should succeed for test, prod and shared envs", func() {
 			Expect(testEnvInitErr).NotTo(HaveOccurred())
 			Expect(prodEnvInitErr).NotTo(HaveOccurred())
+			Expect(sharedEnvInitErr).NotTo(HaveOccurred())
 		})
 
-		It("env ls should list both envs", func() {
+		It("env ls should list all three envs", func() {
 			envListOutput, err := cli.EnvList(appName)
 			Expect(err).NotTo(HaveOccurred())
-			Expect(len(envListOutput.Envs)).To(Equal(2))
+			Expect(len(envListOutput.Envs)).To(Equal(3))
 			envs := map[string]client.EnvDescription{}
 			for _, env := range envListOutput.Envs {
 				envs[env.Name] = env
@@ -117,6 +127,9 @@ var _ = Describe("Customized Env", func() {
 			Expect(envs["test"]).NotTo(BeNil())
 			Expect(envs["test"].Prod).To(BeFalse())
 
+			Expect(envs["shared"]).NotTo(BeNil())
+			Expect(envs["shared"].Prod).To(BeFalse())
+
 			Expect(envs["prod"]).NotTo(BeNil())
 			Expect(envs["prod"].Prod).To(BeTrue())
 		})
@@ -169,10 +182,11 @@ environments:
 		})
 	})
 
-	Context("when deploying a svc to test and prod envs", func() {
+	Context("when deploying a svc to test, prod, and shared envs", func() {
 		var (
 			testDeployErr    error
 			prodEndDeployErr error
+			sharedDeployErr  error
 			svcName          string
 		)
 		BeforeAll(func() {
@@ -188,32 +202,44 @@ environments:
 				EnvName:  "prod",
 				ImageTag: "gallopinggurdey",
 			})
+			_, sharedDeployErr = cli.SvcDeploy(&client.SvcDeployInput{
+				Name:     svcName,
+				EnvName:  "shared",
+				ImageTag: "gallopinggurdey",
+			})
 		})
 
-		It("svc deploy should succeed to both environments", func() {
+		It("svc deploy should succeed to all three environments", func() {
 			Expect(testDeployErr).NotTo(HaveOccurred())
 			Expect(prodEndDeployErr).NotTo(HaveOccurred())
+			Expect(sharedDeployErr).NotTo(HaveOccurred())
 		})
 
-		It("svc show should include a valid URL and description for test and prod envs", func() {
+		It("svc show should include a valid URL and description for test, prod and shared envs", func() {
 			svc, svcShowErr := cli.SvcShow(&client.SvcShowRequest{
 				AppName: appName,
 				Name:    svcName,
 			})
 			Expect(svcShowErr).NotTo(HaveOccurred())
-			Expect(len(svc.Routes)).To(Equal(2))
+			Expect(len(svc.Routes)).To(Equal(3))
 			// Group routes by environment
 			envRoutes := map[string]client.SvcShowRoutes{}
 			for _, route := range svc.Routes {
 				envRoutes[route.Environment] = route
 			}
 
-			Expect(len(svc.ServiceDiscoveries)).To(Equal(1))
-			Expect(svc.ServiceDiscoveries[0].Environment).To(ConsistOf("test", "prod"))
-			Expect(svc.ServiceDiscoveries[0].Namespace).To(Equal(fmt.Sprintf("%s.%s.local:80", svc.SvcName, appName)))
+			Expect(len(svc.ServiceDiscoveries)).To(Equal(3))
+			var envs, namespaces, wantedNamespaces []string
+			for _, sd := range svc.ServiceDiscoveries {
+				envs = append(envs, sd.Environment[0])
+				namespaces = append(namespaces, sd.Namespace)
+				wantedNamespaces = append(wantedNamespaces, fmt.Sprintf("%s.%s.%s.local:80", svc.SvcName, sd.Environment, appName))
+			}
+			Expect(envs).To(ConsistOf("test", "prod", "shared"))
+			Expect(namespaces).To(ConsistOf(wantedNamespaces))
 
 			// Call each environment's endpoint and ensure it returns a 200
-			for _, env := range []string{"test", "prod"} {
+			for _, env := range []string{"test", "prod", "shared"} {
 				route := envRoutes[env]
 				Expect(route.Environment).To(Equal(env))
 				Eventually(func() (int, error) {
@@ -224,7 +250,7 @@ environments:
 		})
 
 		It("svc logs should display logs", func() {
-			for _, envName := range []string{"test", "prod"} {
+			for _, envName := range []string{"test", "prod", "shared"} {
 				var svcLogs []client.SvcLogsOutput
 				var svcLogsErr error
 				Eventually(func() ([]client.SvcLogsOutput, error) {
@@ -246,9 +272,9 @@ environments:
 			}
 		})
 
-		It("env show should display info for test and prod envs", func() {
+		It("env show should display info for test, prod, and shared envs", func() {
 			envs := map[string]client.EnvDescription{}
-			for _, envName := range []string{"test", "prod"} {
+			for _, envName := range []string{"test", "prod", "shared"} {
 				envShowOutput, envShowErr := cli.EnvShow(&client.EnvShowRequest{
 					AppName: appName,
 					EnvName: envName,
@@ -272,6 +298,8 @@ environments:
 			Expect(envs["test"].Prod).To(BeFalse())
 			Expect(envs["prod"]).NotTo(BeNil())
 			Expect(envs["prod"].Prod).To(BeTrue())
+			Expect(envs["shared"]).NotTo(BeNil())
+			Expect(envs["shared"].Prod).To(BeFalse())
 		})
 	})
 })

e2e/init/init_suite_test.go

@@ -15,6 +15,7 @@ import (
 
 var cli *client.CLI
 var appName string
+var envName string
 
 /**
 The Init Suite runs through the copilot init workflow for a brand new
@@ -31,6 +32,7 @@ var _ = BeforeSuite(func() {
 	cli = ecsCli
 	Expect(err).NotTo(HaveOccurred())
 	appName = fmt.Sprintf("e2e-init-%d", time.Now().Unix())
+	envName = "test"
 })
 
 var _ = AfterSuite(func() {

e2e/init/init_test.go

@@ -133,7 +133,7 @@ var _ = Describe("init flow", func() {
 		It("should return a valid service discovery namespace", func() {
 			Expect(len(svc.ServiceDiscoveries)).To(Equal(1))
 			Expect(svc.ServiceDiscoveries[0].Environment).To(Equal([]string{"test"}))
-			Expect(svc.ServiceDiscoveries[0].Namespace).To(Equal(fmt.Sprintf("%s.%s.local:80", svcName, appName)))
+			Expect(svc.ServiceDiscoveries[0].Namespace).To(Equal(fmt.Sprintf("%s.%s.%s.local:80", svcName, envName, appName)))
 		})
 
 		It("should return the correct environment variables", func() {
@@ -143,7 +143,7 @@ var _ = Describe("init flow", func() {
 				"COPILOT_ENVIRONMENT_NAME":           "test",
 				"COPILOT_LB_DNS":                     strings.TrimPrefix(svc.Routes[0].URL, "http://"),
 				"COPILOT_SERVICE_NAME":               svcName,
-				"COPILOT_SERVICE_DISCOVERY_ENDPOINT": fmt.Sprintf("%s.local", appName),
+				"COPILOT_SERVICE_DISCOVERY_ENDPOINT": fmt.Sprintf("%s.%s.local", envName, appName),
 			}
 			for _, variable := range svc.Variables {
 				Expect(variable.Value).To(Equal(expectedVars[variable.Name]))

internal/pkg/cli/env_upgrade.go

@@ -79,6 +79,7 @@ func newEnvUpgradeOpts(vars envUpgradeVars) (*envUpgradeOpts, error) {
 		sel:   selector.NewSelect(prompt.New(), store),
 		legacyEnvTemplater: stack.NewEnvStackConfig(&deploy.CreateEnvironmentInput{
 			Version: deploy.LegacyEnvTemplateVersion,
+			AppName: vars.appName,
 		}),
 		prog:     termprogress.NewSpinner(log.DiagnosticWriter),
 		uploader: template.New(),

internal/pkg/cli/interfaces.go

@@ -408,6 +408,10 @@ type versionGetter interface {
 	Version() (string, error)
 }
 
+type endpointGetter interface {
+	ServiceDiscoveryEndpoint() (string, error)
+}
+
 type envTemplater interface {
 	EnvironmentTemplate(appName, envName string) (string, error)
 }

internal/pkg/cli/job_deploy.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 
 	"github.com/aws/copilot-cli/internal/pkg/deploy"
+	"github.com/aws/copilot-cli/internal/pkg/describe"
 
 	"github.com/aws/copilot-cli/internal/pkg/addon"
 	"github.com/aws/copilot-cli/internal/pkg/exec"
@@ -47,6 +48,7 @@ type deployJobOpts struct {
 	sessProvider       sessionProvider
 	s3                 artifactUploader
 	envUpgradeCmd      actionCommand
+	endpointGetter     endpointGetter
 
 	spinner progress
 	sel     wsSelector
@@ -206,6 +208,14 @@ func (o *deployJobOpts) configureClients() error {
 
 	// CF client against env account profile AND target environment region
 	o.jobCFN = cloudformation.New(envSession)
+	o.endpointGetter, err = describe.NewEnvDescriber(describe.NewEnvDescriberConfig{
+		App:         o.appName,
+		Env:         o.envName,
+		ConfigStore: o.store,
+	})
+	if err != nil {
+		return fmt.Errorf("initiate environment describer: %w", err)
+	}
 
 	addonsSvc, err := addon.New(o.name)
 	if err != nil {
@@ -300,10 +310,15 @@ func (o *deployJobOpts) stackConfiguration(addonsURL string) (cloudformation.Sta
 }
 
 func (o *deployJobOpts) runtimeConfig(addonsURL string) (*stack.RuntimeConfig, error) {
+	endpoint, err := o.endpointGetter.ServiceDiscoveryEndpoint()
+	if err != nil {
+		return nil, err
+	}
 	if !o.buildRequired {
 		return &stack.RuntimeConfig{
-			AddonsTemplateURL: addonsURL,
-			AdditionalTags:    tags.Merge(o.targetApp.Tags, o.resourceTags),
+			AddonsTemplateURL:        addonsURL,
+			AdditionalTags:           tags.Merge(o.targetApp.Tags, o.resourceTags),
+			ServiceDiscoveryEndpoint: endpoint,
 		}, nil
 	}
 	resources, err := o.appCFN.GetAppResourcesByRegion(o.targetApp, o.targetEnvironment.Region)
@@ -324,8 +339,9 @@ func (o *deployJobOpts) runtimeConfig(addonsURL string) (*stack.RuntimeConfig, e
 			ImageTag: o.imageTag,
 			Digest:   o.imageDigest,
 		},
-		AddonsTemplateURL: addonsURL,
-		AdditionalTags:    tags.Merge(o.targetApp.Tags, o.resourceTags),
+		AddonsTemplateURL:        addonsURL,
+		AdditionalTags:           tags.Merge(o.targetApp.Tags, o.resourceTags),
+		ServiceDiscoveryEndpoint: endpoint,
 	}, nil
 }
 

internal/pkg/cli/job_package.go

@@ -8,6 +8,7 @@ import (
 	"io/ioutil"
 	"os"
 
+	"github.com/aws/copilot-cli/internal/pkg/describe"
 	"github.com/aws/copilot-cli/internal/pkg/exec"
 
 	"github.com/aws/copilot-cli/internal/pkg/aws/sessions"
@@ -104,6 +105,17 @@ func newPackageJobOpts(vars packageJobVars) (*packageJobOpts, error) {
 			addonsWriter:     ioutil.Discard,
 			fs:               &afero.Afero{Fs: afero.NewOsFs()},
 			stackSerializer:  o.stackSerializer,
+			newEndpointGetter: func(app, env string) (endpointGetter, error) {
+				d, err := describe.NewEnvDescriber(describe.NewEnvDescriberConfig{
+					App:         app,
+					Env:         env,
+					ConfigStore: store,
+				})
+				if err != nil {
+					return nil, fmt.Errorf("new env describer for environment %s in app %s: %v", env, app, err)
+				}
+				return d, nil
+			},
 		}
 	}
 	return opts, nil