Aws logs sync state (#353)

* Add enable_logs_sync to signalfx_aws_integration
* Bump signalfx-go

Author: bgola-signalfx

CHANGELOG.md

@@ -1,3 +1,8 @@
+## 6.9.0
+
+IMPROVEMENTS:
+* resource/signalfx_aws_integration: Add support for AWS metric streams and AWS logs synchronization [#351](https://github.com/splunk-terraform/terraform-provider-signalfx/pull/351) [#353](https://github.com/splunk-terraform/terraform-provider-signalfx/pull/353)
+
 ## 6.8.1
 
 BUGFIXES:

README.md

@@ -65,23 +65,31 @@ $ export SFX_AUTH_TOKEN=XXXXXX
 $ make testacc
 ```
 
-To also run the AWS integration tests for Cloudwatch Metric Streams synchronization, you must create an actual AWS IAM user with an access key and secret that SignalFx can use to manage AWS Cloudwatch Metric Streams resources, and define the `SFX_TEST_AWS_ACCESS_KEY_ID` and `SFX_TEST_AWS_SECRET_ACCESS_KEY` environment variables:
+To also run the AWS integration tests for CloudWatch Metric Streams and AWS logs synchronization, you must create an actual AWS IAM user with an access key and secret that SignalFx can use to manage AWS resources, and define the `SFX_TEST_AWS_ACCESS_KEY_ID` and `SFX_TEST_AWS_SECRET_ACCESS_KEY` environment variables:
 
 ```sh
 export SFX_TEST_AWS_ACCESS_KEY_ID=AKIAXXXXXX
 export SFX_TEST_AWS_SECRET_ACCESS_KEY=XXXXXX
 ```
 
-The policies that this user must be granted are:
+The following permissions must be granted. Additional permissions may be required to capture logs from specific AWS services.
 
 ```
-"cloudwatch:ListMetricStreams",
+"cloudwatch:DeleteMetricStream",
 "cloudwatch:GetMetricStream",
+"cloudwatch:ListMetricStreams",
 "cloudwatch:PutMetricStream",
-"cloudwatch:DeleteMetricStream",
 "cloudwatch:StartMetricStreams",
 "cloudwatch:StopMetricStreams",
-"iam:PassRole"
+"iam:PassRole",
+
+"logs:DeleteSubscriptionFilter",
+"logs:DescribeLogGroups",
+"logs:DescribeSubscriptionFilters",
+"logs:PutSubscriptionFilter",
+"s3:GetBucketLogging",
+"s3:GetBucketNotification",
+"s3:PutBucketNotification"
 ```
 
 See [Connect to AWS using the guided setup in Splunk Observability Cloud](https://docs.splunk.com/Observability/gdi/get-data-in/connect/aws/aws-wizardconfig.html) and [Enable CloudWatch Metric Streams](https://docs.splunk.com/Observability/gdi/get-data-in/connect/aws/aws-apiconfig.html#enable-cloudwatch-metric-streams) in Splunk documentation for more details about creating that IAM policy.

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d
 	github.com/hashicorp/terraform-plugin-sdk v1.15.0
 	github.com/mitchellh/go-homedir v1.1.0
-	github.com/signalfx/signalfx-go v1.8.7
+	github.com/signalfx/signalfx-go v1.8.8
 	github.com/stretchr/testify v1.6.1
 )
 

go.sum

@@ -217,8 +217,8 @@ github.com/signalfx/golib/v3 v3.3.37 h1:CvL6Q8hySjm7D82P039qH2lFWJvBa18IJdAwa4LH
 github.com/signalfx/golib/v3 v3.3.37/go.mod h1:zH70SVnrzVUqDmUFEwDk1HSwS71Pi2w3bSsNpnOtYuQ=
 github.com/signalfx/gomemcache v0.0.0-20180823214636-4f7ef64c72a9/go.mod h1:Ytb8KfCSyuwy/VILnROdgCvbQLA5ch0nkbG7lKT0BXw=
 github.com/signalfx/sapm-proto v0.4.0/go.mod h1:x3gtwJ1GRejtkghB4nYpwixh2zqJrLbPU959ZNhM0Fk=
-github.com/signalfx/signalfx-go v1.8.7 h1:05lfikNYb1ef8+RxmxZS1JnS5DgRoBK1a0bgRzJoA+s=
-github.com/signalfx/signalfx-go v1.8.7/go.mod h1:YhPTMdQJDfphcRBdk+9acbbAw1gYY7z5BIHUWzLmGlA=
+github.com/signalfx/signalfx-go v1.8.8 h1:Da0r1WVlXSRL6q2MhIXXkGmRYVWuh46NKm63Gn6Z2Go=
+github.com/signalfx/signalfx-go v1.8.8/go.mod h1:YhPTMdQJDfphcRBdk+9acbbAw1gYY7z5BIHUWzLmGlA=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/assertions v0.0.0-20190215210624-980c5ac6f3ac h1:wbW+Bybf9pXxnCFAOWZTqkRjAc7rAIwo2e1ArUhiHxg=
 github.com/smartystreets/assertions v0.0.0-20190215210624-980c5ac6f3ac/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=

signalfx/resource_signalfx_aws_external_integration.go

@@ -4,13 +4,10 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"log"
-	"strings"
-	"time"
-
-	"github.com/hashicorp/terraform-plugin-sdk/helper/resource"
 	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
 	"github.com/signalfx/signalfx-go/integration"
+	"log"
+	"strings"
 )
 
 func integrationAWSExternalResource() *schema.Resource {
@@ -126,66 +123,3 @@ func integrationAWSExternalCreate(d *schema.ResourceData, meta interface{}) erro
 func integrationAWSExternalDelete(d *schema.ResourceData, meta interface{}) error {
 	return DoIntegrationAWSDelete(d, meta)
 }
-
-// DoIntegrationAWSDelete is public because it is also used by integrationAWSTokenDelete
-func DoIntegrationAWSDelete(d *schema.ResourceData, meta interface{}) error {
-	config := meta.(*signalfxConfig)
-
-	// Retrieve current integration state
-	int, err := config.Client.GetAWSCloudWatchIntegration(context.TODO(), d.Id())
-	if err != nil {
-		return fmt.Errorf("Error fetching existing integration %s, %s", d.Id(), err.Error())
-	}
-
-	// Only disable the Cloudwatch Metric Stream synchronization if needed
-	if int.MetricStreamsSyncState != "" && int.MetricStreamsSyncState != "DISABLED" {
-		int.MetricStreamsSyncState = "CANCELLING"
-		_, err := config.Client.UpdateAWSCloudWatchIntegration(context.TODO(), d.Id(), int)
-		if err != nil {
-			if strings.Contains(err.Error(), "40") {
-				err = fmt.Errorf("%s\nPlease verify you are using an admin token when working with integrations", err.Error())
-			}
-			return err
-		}
-		// Wait for expected Cloudwatch Metric Streams sync state to be disabled
-		if _, err = waitForAWSIntegrationMetricStreamsSyncStateDelete(d, config, int.Id); err != nil {
-			return err
-		}
-	}
-
-	return config.Client.DeleteAWSCloudWatchIntegration(context.TODO(), d.Id())
-}
-
-func waitForAWSIntegrationMetricStreamsSyncStateDelete(d *schema.ResourceData, config *signalfxConfig, id string) (*integration.AwsCloudWatchIntegration, error) {
-	pending := []string{
-		"ENABLED",
-		"CANCELLING",
-	}
-	target := []string{
-		"",
-		"DISABLED",
-	}
-
-	stateConf := &resource.StateChangeConf{
-		Pending: pending,
-		Target:  target,
-		Refresh: func() (interface{}, string, error) {
-			int, err := config.Client.GetAWSCloudWatchIntegration(context.TODO(), id)
-			if err != nil {
-				return 0, "", err
-			}
-			return int, int.MetricStreamsSyncState, nil
-		},
-		Timeout:    d.Timeout(schema.TimeoutUpdate) - time.Minute,
-		Delay:      10 * time.Second,
-		MinTimeout: 5 * time.Second,
-	}
-
-	int, err := stateConf.WaitForState()
-	if err != nil {
-		return nil, fmt.Errorf(
-			"Error waiting for integration (%s) Cloudwatch Metric Streams sync state to become disabled: %s",
-			id, err)
-	}
-	return int.(*integration.AwsCloudWatchIntegration), nil
-}

signalfx/resource_signalfx_aws_integration.go

@@ -14,6 +14,16 @@ import (
 	"github.com/signalfx/signalfx-go/integration"
 )
 
+type stateSupplierFunc = func(*integration.AwsCloudWatchIntegration) string
+
+func metricStreamsStateSupplier(int *integration.AwsCloudWatchIntegration) string {
+	return int.MetricStreamsSyncState
+}
+
+func logsSyncStateSupplier(int *integration.AwsCloudWatchIntegration) string {
+	return int.LogsSyncState
+}
+
 func integrationAWSResource() *schema.Resource {
 	return &schema.Resource{
 		Schema: map[string]*schema.Schema{
@@ -185,6 +195,12 @@ func integrationAWSResource() *schema.Resource {
 				Computed:    true,
 				Description: "Enables the use of Cloudwatch Metric Streams for metrics synchronization.",
 			},
+			"enable_logs_sync": &schema.Schema{
+				Type:        schema.TypeBool,
+				Optional:    true,
+				Computed:    true,
+				Description: "Enables AWS logs synchronization.",
+			},
 			"named_token": &schema.Schema{
 				Type:        schema.TypeString,
 				Optional:    true,
@@ -274,6 +290,9 @@ func awsIntegrationAPIToTF(d *schema.ResourceData, aws *integration.AwsCloudWatc
 	if err := d.Set("use_metric_streams_sync", aws.MetricStreamsSyncState == "ENABLED"); err != nil {
 		return err
 	}
+	if err := d.Set("enable_logs_sync", aws.LogsSyncState == "ENABLED"); err != nil {
+		return err
+	}
 	if err := d.Set("enable_check_large_volume", aws.EnableCheckLargeVolume); err != nil {
 		return err
 	}
@@ -382,8 +401,13 @@ func getPayloadAWSIntegration(d *schema.ResourceData) (*integration.AwsCloudWatc
 	if d.Get("use_metric_streams_sync").(bool) {
 		aws.MetricStreamsSyncState = "ENABLED"
 	} else if d.HasChange("use_metric_streams_sync") {
-		// Only set to CANCELLING state if the use_metric_streams_sync is false and it has changed, meaning it was ENABLED before
-		aws.MetricStreamsSyncState = "CANCELLING"
+		aws.MetricStreamsSyncState = "CANCELLING" // use_metric_streams_sync is false, and it has changed, meaning it was ENABLED before
+	}
+
+	if d.Get("enable_logs_sync").(bool) {
+		aws.LogsSyncState = "ENABLED"
+	} else if d.HasChange("enable_logs_sync") {
+		aws.LogsSyncState = "CANCELLING" // enable_logs_sync is false, and it has changed, meaning it was ENABLED before
 	}
 
 	if d.Get("external_id").(string) != "" {
@@ -553,50 +577,88 @@ func integrationAWSUpdate(d *schema.ResourceData, meta interface{}) error {
 	}
 
 	if d.HasChange("use_metric_streams_sync") {
-		// Wait for expected Cloudwatch Metric Streams sync state to be reached
-		if int, err = waitForAWSIntegrationMetricStreamsSyncState(d, config, int.Id); err != nil {
+		if int, err = waitForIntegrationStateToSettle(d, config, int.Id, "use_metric_streams_sync", metricStreamsStateSupplier); err != nil {
+			return err
+		}
+	}
+	if d.HasChange("enable_logs_sync") {
+		if int, err = waitForIntegrationStateToSettle(d, config, int.Id, "enable_logs_sync", logsSyncStateSupplier); err != nil {
 			return err
 		}
 	}
 
 	return awsIntegrationAPIToTF(d, int)
 }
 
-func waitForAWSIntegrationMetricStreamsSyncState(d *schema.ResourceData, config *signalfxConfig, id string) (*integration.AwsCloudWatchIntegration, error) {
+func DoIntegrationAWSDelete(d *schema.ResourceData, meta interface{}) error {
+	config := meta.(*signalfxConfig)
+
+	// Retrieve current integration state
+	int, err := config.Client.GetAWSCloudWatchIntegration(context.TODO(), d.Id())
+	if err != nil {
+		return fmt.Errorf("Error fetching existing integration %s, %s", d.Id(), err.Error())
+	}
+
+	// Disable the AWS logs sync and/or CloudWatch metric streams sync if needed
+	needToDisableMetricStreams := int.Enabled && int.MetricStreamsSyncState != "" && int.MetricStreamsSyncState != "DISABLED"
+	needToDisableLogsSync := int.Enabled && int.LogsSyncState != "" && int.LogsSyncState != "DISABLED"
+	if needToDisableMetricStreams || needToDisableLogsSync {
+		if needToDisableMetricStreams {
+			int.MetricStreamsSyncState = "CANCELLING"
+		}
+		if needToDisableLogsSync {
+			int.LogsSyncState = "CANCELLING"
+		}
+		_, err := config.Client.UpdateAWSCloudWatchIntegration(context.TODO(), d.Id(), int)
+		if err != nil {
+			if strings.Contains(err.Error(), "40") {
+				err = fmt.Errorf("%s\nPlease verify you are using an admin token when working with integrations", err.Error())
+			}
+			return err
+		}
+		if needToDisableMetricStreams {
+			if _, err = waitForIntegrationSpecificSyncStateToSettle(d, false, config, int.Id, "use_metric_streams_sync", metricStreamsStateSupplier); err != nil {
+				return err
+			}
+		}
+		if needToDisableLogsSync {
+			if _, err = waitForIntegrationSpecificSyncStateToSettle(d, false, config, int.Id, "enable_logs_sync", logsSyncStateSupplier); err != nil {
+				return err
+			}
+		}
+	}
+
+	return config.Client.DeleteAWSCloudWatchIntegration(context.TODO(), d.Id())
+}
+
+func waitForIntegrationStateToSettle(d *schema.ResourceData, config *signalfxConfig, intId string, syncStateField string,
+	stateSupplier stateSupplierFunc) (*integration.AwsCloudWatchIntegration, error) {
+	return waitForIntegrationSpecificSyncStateToSettle(d, d.Get(syncStateField).(bool), config, intId, syncStateField, stateSupplier)
+}
+
+func waitForIntegrationSpecificSyncStateToSettle(d *schema.ResourceData, syncState bool, config *signalfxConfig, intId string, syncStateField string,
+	stateSupplier stateSupplierFunc) (*integration.AwsCloudWatchIntegration, error) {
 	var pending, target []string
-	useMetricStreamsSync := d.Get("use_metric_streams_sync").(bool)
 	var expectedState string
-	if useMetricStreamsSync {
+	if syncState {
 		expectedState = "enabled"
-		pending = []string{
-			"DISABLED",
-			"CANCELLING",
-			"CANCELLATION_FAILED",
-		}
-		target = []string{
-			"ENABLED",
-		}
+		pending = []string{"DISABLED", "CANCELLING", "CANCELLATION_FAILED"}
+		target = []string{"ENABLED"}
 	} else {
 		expectedState = "disabled"
-		pending = []string{
-			"ENABLED",
-			"CANCELLING",
-		}
-		target = []string{
-			"",
-			"DISABLED",
-		}
+		pending = []string{"ENABLED", "CANCELLING"}
+		target = []string{"", "DISABLED"}
 	}
 
 	stateConf := &resource.StateChangeConf{
 		Pending: pending,
 		Target:  target,
 		Refresh: func() (interface{}, string, error) {
-			int, err := config.Client.GetAWSCloudWatchIntegration(context.TODO(), id)
+			int, err := config.Client.GetAWSCloudWatchIntegration(context.TODO(), intId)
 			if err != nil {
 				return 0, "", err
 			}
-			return int, int.MetricStreamsSyncState, nil
+			return int, stateSupplier(int), nil
 		},
 		Timeout:    d.Timeout(schema.TimeoutUpdate) - time.Minute,
 		Delay:      10 * time.Second,
@@ -605,9 +667,7 @@ func waitForAWSIntegrationMetricStreamsSyncState(d *schema.ResourceData, config
 
 	int, err := stateConf.WaitForState()
 	if err != nil {
-		return nil, fmt.Errorf(
-			"Error waiting for integration (%s) Cloudwatch Metric Streams sync state to become %s: %s",
-			id, expectedState, err)
+		return nil, fmt.Errorf("Error waiting for integration %s state for %s to become %s: %s", intId, syncStateField, expectedState, err)
 	}
 	return int.(*integration.AwsCloudWatchIntegration), nil
 }