feat: added schematics unit test support (#307)

* feat: new package testschematic with create tar function

* feat: add TestSchematicOptions

Changes to support adding a TestSchematicOptions and factory, including small changes to testhelper private methods to public to avoid duplication

* feat: initial working schematic creation

initial working schematic workspace creation, including env and input variables, and tar upload of terraform

* feat: added wait for job completion and status

* refactor: moved all schematics code to own file

* feat: full end-to-end schematics test working

* feat: full working schematics testing

* refactor: fixed some error messages

* docs: added function header comments

* fix: pointer bug and interface cleanup

* fix: add test service for options, for mocking

* test: added full unit tests for schematics service

* feat: add schematic options for working folder and tf version

* fix: tar subdirectories and optional tf version for workspace

* fix: schematic template folder not supported with tar upload

* feat: schematics support for netrc and env settings

* fix: added parent directory to schematic tar file

* fix: schematic GetRefreshToken always new token

Changed GetRefreshToken for schematics api to always request a new token when called.
Resulted in not needing to persist token at service level, and instead opt for IamAuthenticator mocking for tests

* refactor: remove api version from file names

* refactor: changed references of schematicsv1 remove version number

Dropped the version number notation in all schematics packages by using alias

* refactor: move common helper functions to new package

located helper functions to a new common package to be shared with all other packages

Author: toddgiguere

.secrets.baseline

@@ -3,7 +3,7 @@
     "files": "go.sum|package-lock.json|^.secrets.baseline$",
     "lines": null
   },
-  "generated_at": "2023-01-13T13:18:01Z",
+  "generated_at": "2023-01-17T15:29:43Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -82,15 +82,15 @@
         "hashed_secret": "892bd503fb45f6fcafb1c7003d88291fc0b20208",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 104,
+        "line_number": 115,
         "type": "Secret Keyword",
         "verified_result": null
       },
       {
         "hashed_secret": "d4c3d66fd0c38547a3c7a4c6bdc29c36911bc030",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 165,
+        "line_number": 176,
         "type": "Secret Keyword",
         "verified_result": null
       }

cloudinfo/service.go

@@ -26,6 +26,17 @@ type CloudInfoService struct {
 	lock                      sync.Mutex
 }
 
+// interface for the cloudinfo service (can be mocked in tests)
+type CloudInfoServiceI interface {
+	GetLeastVpcTestRegion() (string, error)
+	GetLeastVpcTestRegionWithoutActivityTracker() (string, error)
+	GetLeastPowerConnectionZone() (string, error)
+	LoadRegionPrefsFromFile(string) error
+	HasRegionData() bool
+	RemoveRegionForTest(string)
+	GetThreadLock() *sync.Mutex
+}
+
 // CloudInfoServiceOptions structure used as input params for service constructor.
 type CloudInfoServiceOptions struct {
 	ApiKey                    string

common/general.go

@@ -0,0 +1,166 @@
+package common
+
+import (
+	"encoding/json"
+	"os"
+	"os/exec"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+// GetRequiredEnvVars returns a map containing required environment variables and their values
+// Fails the test if any are missing
+func GetRequiredEnvVars(t *testing.T, variableNames []string) map[string]string {
+	var missingVariables []string
+	envVars := make(map[string]string)
+
+	for _, variableName := range variableNames {
+		val, present := os.LookupEnv(variableName)
+		if present {
+			envVars[variableName] = val
+		} else {
+			missingVariables = append(missingVariables, variableName)
+		}
+	}
+	require.Empty(t, missingVariables, "The following environment variables must be set: %v", missingVariables)
+
+	return envVars
+}
+
+// GitRootPath gets the path to the current git repos root directory
+func GitRootPath(fromPath string) (string, error) {
+	cmd := exec.Command("git", "rev-parse", "--show-toplevel")
+	cmd.Dir = fromPath
+	path, err := cmd.Output()
+
+	if err != nil {
+		return "", err
+	}
+	return strings.TrimSpace(string(path)), nil
+}
+
+// GetBeforeAfterDiff takes a JSON string as input and returns a string with the differences
+// between the "before" and "after" objects in the JSON.
+//
+// For example, given the JSON string:
+//
+//	{"before": {"a": 1, "b": 2}, "after": {"a": 2, "b": 3}}
+//
+// the function would return the string:
+//
+//	"Before: {"b": 2}\nAfter: {"a": 2, "b": 3}"
+func GetBeforeAfterDiff(jsonString string) string {
+	// Parse the JSON string into a map
+	var jsonMap map[string]interface{}
+	err := json.Unmarshal([]byte(jsonString), &jsonMap)
+	if err != nil {
+		return "Error: unable to parse JSON string"
+	}
+
+	// Get the "before" and "after" values from the map
+	before, beforeOk := jsonMap["before"]
+	after, afterOk := jsonMap["after"]
+	if !beforeOk || !afterOk {
+		return "Error: missing 'before' or 'after' key in JSON"
+	}
+
+	// Check if the "before" and "after" values are objects
+	beforeObject, beforeOk := before.(map[string]interface{})
+	if !beforeOk {
+		return "Error: 'before' value is not an object"
+	}
+	afterObject, afterOk := after.(map[string]interface{})
+	if !afterOk {
+		return "Error: 'after' value is not an object"
+	}
+
+	// Find the differences between the two objects
+	diffsBefore := make(map[string]interface{})
+	for key, value := range beforeObject {
+		if !reflect.DeepEqual(afterObject[key], value) {
+			diffsBefore[key] = value
+		}
+	}
+
+	// Convert the diffs map to a JSON string
+	diffsJson, err := json.Marshal(diffsBefore)
+	if err != nil {
+		return "Error: unable to convert diffs to JSON"
+	}
+
+	// Find the differences between the two objects
+	diffsAfter := make(map[string]interface{})
+	for key, value := range afterObject {
+		if !reflect.DeepEqual(beforeObject[key], value) {
+			diffsAfter[key] = value
+		}
+	}
+
+	// Convert the diffs map to a JSON string
+	diffsJson2, err := json.Marshal(diffsAfter)
+	if err != nil {
+		return "Error: unable to convert diffs2 to JSON"
+	}
+
+	return "Before: " + string(diffsJson) + "\nAfter: " + string(diffsJson2)
+}
+
+// overwriting duplicate keys
+func MergeMaps(maps ...map[string]interface{}) map[string]interface{} {
+	result := make(map[string]interface{})
+	for _, m := range maps {
+		for k, v := range m {
+			result[k] = v
+		}
+	}
+	return result
+}
+
+// Adds value to map[key] only if value != compareValue
+func ConditionalAdd(amap map[string]interface{}, key string, value string, compareValue string) {
+	if value != compareValue {
+		amap[key] = value
+	}
+}
+
+// ConvertArrayToJsonString is a helper function that will take an array of Golang data types, and return a string
+// of the array formatted as a JSON array.
+// Helpful to convert Golang arrays into a format that Terraform can consume.
+func ConvertArrayToJsonString(arr interface{}) (string, error) {
+	// first marshal array into json compatible
+	json, jsonErr := json.Marshal(arr)
+	if jsonErr != nil {
+		return "", jsonErr
+	}
+
+	// take json array, wrap as one string, and escape any double quotes inside
+	s := string(json)
+
+	return s, nil
+}
+
+// IsArray is a simple helper function that will determine if a given Golang value is a slice or array.
+func IsArray(v interface{}) bool {
+
+	theType := reflect.TypeOf(v).Kind()
+
+	if (theType == reflect.Slice) || (theType == reflect.Array) {
+		return true
+	}
+
+	return false
+}
+
+// StrArrayContains is a helper function that will check an array and see if a value is already present
+func StrArrayContains(arr []string, val string) bool {
+	for _, arrVal := range arr {
+		if arrVal == val {
+			return true
+		}
+	}
+
+	return false
+}

common/general_test.go

@@ -0,0 +1,143 @@
+package common
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetRequiredEnvVarsSuccess(t *testing.T) {
+	t.Setenv("A_REQUIRED_VARIABLE", "The Value")
+	t.Setenv("ANOTHER_VARIABLE", "Another Value")
+
+	expected := make(map[string]string)
+	expected["A_REQUIRED_VARIABLE"] = "The Value"
+	expected["ANOTHER_VARIABLE"] = "Another Value"
+
+	assert.Equal(t, expected, GetRequiredEnvVars(t, []string{"A_REQUIRED_VARIABLE", "ANOTHER_VARIABLE"}))
+}
+
+func TestGetRequiredEnvVarsEmptyInput(t *testing.T) {
+
+	expected := make(map[string]string)
+	assert.Equal(t, expected, GetRequiredEnvVars(t, []string{}))
+}
+
+func TestGetBeforeAfterDiffValidInput(t *testing.T) {
+	jsonString := `{"before": {"a": 1, "b": 2}, "after": {"a": 2, "b": 3}}`
+	expected := "Before: {\"a\":1,\"b\":2}\nAfter: {\"a\":2,\"b\":3}"
+	result := GetBeforeAfterDiff(jsonString)
+	if result != expected {
+		t.Errorf("TestGetBeforeAfterDiffValidInput(%q) returned %q, expected %q", jsonString, result, expected)
+	}
+}
+
+func TestGetBeforeAfterDiffMissingBeforeKey(t *testing.T) {
+	jsonString := `{"after": {"a": 1, "b": 2}}`
+	expected := "Error: missing 'before' or 'after' key in JSON"
+	result := GetBeforeAfterDiff(jsonString)
+	if result != expected {
+		t.Errorf("TestGetBeforeAfterDiffMissingBeforeKey(%q) returned %q, expected %q", jsonString, result, expected)
+	}
+}
+
+func TestGetBeforeAfterDiffNonObjectBeforeValue(t *testing.T) {
+	jsonString := `{"before": ["a", "b"], "after": {"a": 1, "b": 2}}`
+	expected := "Error: 'before' value is not an object"
+	result := GetBeforeAfterDiff(jsonString)
+	if result != expected {
+		t.Errorf("TestGetBeforeAfterDiffNonObjectBeforeValue(%q) returned %q, expected %q", jsonString, result, expected)
+	}
+}
+
+func TestGetBeforeAfterDiffNonObjectAfterValue(t *testing.T) {
+	jsonString := `{"before": {"a": 1, "b": 2}, "after": ["a", "b"]}`
+	expected := "Error: 'after' value is not an object"
+	result := GetBeforeAfterDiff(jsonString)
+	if result != expected {
+		t.Errorf("TestGetBeforeAfterDiffNonObjectAfterValue(%q) returned %q, expected %q", jsonString, result, expected)
+	}
+}
+
+func TestGetBeforeAfterDiffInvalidJSON(t *testing.T) {
+	jsonString := `{"before": {"a": 1, "b": 2}, "after": {"a": 1, "b": 2}`
+	expected := "Error: unable to parse JSON string"
+	result := GetBeforeAfterDiff(jsonString)
+	if result != expected {
+		t.Errorf("TestGetBeforeAfterDiffInvalidJSON(%q) returned %q, expected %q", jsonString, result, expected)
+	}
+}
+
+func TestConvertArrayJson(t *testing.T) {
+
+	t.Run("GoodArray", func(t *testing.T) {
+		goodArr := []interface{}{
+			"hello",
+			true,
+			99,
+			"bye",
+		}
+		goodStr, goodErr := ConvertArrayToJsonString(goodArr)
+		if assert.NoError(t, goodErr, "error converting array") {
+			assert.NotEmpty(t, goodStr)
+		}
+	})
+
+	t.Run("NilValue", func(t *testing.T) {
+		nullVal, nullErr := ConvertArrayToJsonString(nil)
+		if assert.NoError(t, nullErr) {
+			assert.Equal(t, "null", nullVal)
+		}
+	})
+
+	t.Run("NullPointer", func(t *testing.T) {
+		var intPtr *int
+		ptrVal, ptrErr := ConvertArrayToJsonString(intPtr)
+		if assert.NoError(t, ptrErr) {
+			assert.Equal(t, "null", ptrVal)
+		}
+	})
+}
+
+func TestIsArray(t *testing.T) {
+
+	t.Run("IsSlice", func(t *testing.T) {
+		slice := []int{1, 2, 3}
+		isSlice := IsArray(slice)
+		assert.True(t, isSlice)
+	})
+
+	t.Run("IsArray", func(t *testing.T) {
+		arr := [3]int{1, 2, 3}
+		isArr := IsArray(arr)
+		assert.True(t, isArr)
+	})
+
+	t.Run("TryString", func(t *testing.T) {
+		val := "hello"
+		is := IsArray(val)
+		assert.False(t, is)
+	})
+
+	t.Run("TryBool", func(t *testing.T) {
+		bval := true
+		bis := IsArray(bval)
+		assert.False(t, bis)
+	})
+
+	t.Run("TryNumber", func(t *testing.T) {
+		nval := 99.99
+		nis := IsArray(nval)
+		assert.False(t, nis)
+	})
+
+	t.Run("TryStruct", func(t *testing.T) {
+		type TestObject struct {
+			prop1 string
+			prop2 int
+		}
+		obj := &TestObject{"hello", 99}
+		sis := IsArray(*obj)
+		assert.False(t, sis)
+	})
+}

testhelper/travis.go renamed to common/travis.go

@@ -1,4 +1,4 @@
-package testhelper
+package common
 
 import (
 	"os"

testhelper/travis_test.go renamed to common/travis_test.go

@@ -1,9 +1,10 @@
-package testhelper
+package common
 
 import (
-	"github.com/stretchr/testify/assert"
 	"strings"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 type travisVariables struct {