From e340a286a6261df2cfca1579076ca585ad5d16b8 Mon Sep 17 00:00:00 2001 From: Shaun Remekie Date: Thu, 16 May 2024 15:59:09 +0200 Subject: [PATCH 1/4] updated drop to send for recombine added route to validate logs from docker log drive --- opentelemetry/ecs-ec2/template.yaml | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/opentelemetry/ecs-ec2/template.yaml b/opentelemetry/ecs-ec2/template.yaml index 39d96a94..f8be4aa5 100644 --- a/opentelemetry/ecs-ec2/template.yaml +++ b/opentelemetry/ecs-ec2/template.yaml @@ -120,24 +120,35 @@ Mappings: include_file_path: true # add log.file.path to resource attributes operators: + - type: router + id: docker_log_json_parser + routes: + - output: json_parser + expr: 'body matches "^\\{\"log\".*\\}xxx"' + default: move_log_file_path + - type: json_parser parse_from: body parse_to: body + output: recombine timestamp: parse_from: body.time layout: '%Y-%m-%dT%H:%M:%S.%fZ' + default: move_log_file_path # handle logs split by docker - type: recombine id: recombine + output: move_log_file_path combine_field: body.log source_identifier: attributes["log.file.path"] is_last_entry: body.log endsWith "\n" force_flush_period: 10s - on_error: drop + on_error: send combine_with: "" - type: move + id: move_log_file_path from: attributes["log.file.path"] to: resource["log.file.path"] @@ -261,28 +272,38 @@ Mappings: include_file_path: true # add log.file.path to resource attributes operators: + - type: router + id: docker_log_json_parser + routes: + - output: json_parser + expr: 'body matches "^\\{\"log\".*\\}xxx"' + default: move_log_file_path + - type: json_parser parse_from: body parse_to: body + output: recombine timestamp: parse_from: body.time layout: '%Y-%m-%dT%H:%M:%S.%fZ' + default: move_log_file_path # handle logs split by docker - type: recombine id: recombine + output: move_log_file_path combine_field: body.log source_identifier: attributes["log.file.path"] is_last_entry: body.log endsWith "\n" force_flush_period: 10s - on_error: drop + on_error: send combine_with: "" - type: move + id: move_log_file_path from: attributes["log.file.path"] to: resource["log.file.path"] - - # add metrics + awsecscontainermetricsd: processors: From 71154fc902b8f81452881dcd206e6ce1b6c35fdb Mon Sep 17 00:00:00 2001 From: Shaun Remekie Date: Thu, 16 May 2024 16:03:58 +0200 Subject: [PATCH 2/4] changelogs --- opentelemetry/ecs-ec2/CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/opentelemetry/ecs-ec2/CHANGELOG.md b/opentelemetry/ecs-ec2/CHANGELOG.md index 444b9cc3..c59cd36b 100644 --- a/opentelemetry/ecs-ec2/CHANGELOG.md +++ b/opentelemetry/ecs-ec2/CHANGELOG.md @@ -1,4 +1,7 @@ # Changelog +### 0.0.12 / 2024-05-16 +- Added validation using operator route to default otel config for ecs-ec2 config + ### 0.0.11 / 2024-04-05 - Remove deprecated PrivateKey Param from ecs-ec2 otel deployment From 9032e952275b4c6ab1ee95d15cca939456ec5441 Mon Sep 17 00:00:00 2001 From: Shaun Remekie Date: Thu, 16 May 2024 16:29:16 +0200 Subject: [PATCH 3/4] remove typo --- opentelemetry/ecs-ec2/template.yaml | 4 +- opentelemetry/lambda/script.py | 45 +++++++++++++++ opentelemetry/lambda/template.yaml | 87 +++++++++++++++++++++++++++++ 3 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 opentelemetry/lambda/script.py create mode 100644 opentelemetry/lambda/template.yaml diff --git a/opentelemetry/ecs-ec2/template.yaml b/opentelemetry/ecs-ec2/template.yaml index f8be4aa5..1e8d6c4a 100644 --- a/opentelemetry/ecs-ec2/template.yaml +++ b/opentelemetry/ecs-ec2/template.yaml @@ -124,7 +124,7 @@ Mappings: id: docker_log_json_parser routes: - output: json_parser - expr: 'body matches "^\\{\"log\".*\\}xxx"' + expr: 'body matches "^\\{\"log\".*\\}"' default: move_log_file_path - type: json_parser @@ -276,7 +276,7 @@ Mappings: id: docker_log_json_parser routes: - output: json_parser - expr: 'body matches "^\\{\"log\".*\\}xxx"' + expr: 'body matches "^\\{\"log\".*\\}"' default: move_log_file_path - type: json_parser diff --git a/opentelemetry/lambda/script.py b/opentelemetry/lambda/script.py new file mode 100644 index 00000000..7a3f2af6 --- /dev/null +++ b/opentelemetry/lambda/script.py @@ -0,0 +1,45 @@ +import boto3 +import cfnresponse +import json + +def handler(event, context): + print("received event: " + json.dumps(event, indent=2)) + + try: + status = cfnresponse.SUCCESS + + if event['RequestType'] in ['Create', 'Update']: + lambda_arn = event['ResourceProperties']['LambdaArn'] + lambda_client = boto3.client('lambda') + + # Get the current environment variables of the Lambda function + function_configuration = lambda_client.get_function_configuration(FunctionName=lambda_arn) + current_env = function_configuration['Environment']['Variables'] + + # Merge the current environment variables with any new ones specified in the CloudFormation event + # new_environment = current_environment.copy() + current_env.update(event['ResourceProperties']['env']) + + # Get the list of layers to add to the Lambda function + layers = event['ResourceProperties'].get('LayerArns', []) + + # Add the specified layers and environment variables to the Lambda function + lambda_client.update_function_configuration( + FunctionName=lambda_arn, + Environment={'Variables': current_env}, + Layers=layers) + + if event['RequestType'] == 'Delete': + # TODO: implement delete logic + pass + + + except Exception as e: + print(f"custom resource failed to update layers and env vars: {e}") + status = cfnresponse.FAILED + + finally: + # Send a status response to CloudFormation + response_data = {} + cfnresponse.send(event, context, status, response_data) + return diff --git a/opentelemetry/lambda/template.yaml b/opentelemetry/lambda/template.yaml new file mode 100644 index 00000000..e9ae4623 --- /dev/null +++ b/opentelemetry/lambda/template.yaml @@ -0,0 +1,87 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 + +Parameters: + LambdaFunctionARN: + Type: String + Description: The ARN of the Lambda function to invoke + + WrapperHandler: + Type: String + Description: The path to the otel instrumentation executable + Default: /opt/python/otel-instrument + + Runtime: + Type: String + Description: The language of the Lambda function + Default: python + AllowedValues: + - python + - nodejs + - go + + Arch: + Type: String + Description: The architecture of the Lambda function + Default: amd64 + AllowedValues: + - amd64 + - arm64 + +Mappings: + layers: + python: + amd64: arn:aws:lambda:eu-north-1:035955823196:layer:coralogix-aws-lambda-telemetry-exporter-python-x86_64:1 + arm64: arn:aws:lambda:eu-north-1:035955823196:layer:coralogix-aws-lambda-telemetry-exporter-python-arm64:1 + + RegionMap: + us-east-1: + LambdaLayerARN: arn:aws:lambda:us-east-1:901920570463:layer:aws-otel-python38-ver-1-0-0:1 + us-east-2: + LambdaLayerARN: arn:aws:lambda:us-east-2:901920570463:layer:aws-otel-python38-ver-1-0-0:1 + us-west-2: + LambdaLayerARN: arn:aws:lambda:us-west-2:901920570463:layer:aws-otel-python38-ver-1-0-0:1 + +Resources: + CustomResourceConfigureLambdaFunction: + Type: 'AWS::CloudFormation::CustomResource' + Properties: + ServiceToken: !GetAtt AddConfigLambdaFunction.Arn + LambdaArn: !Ref LambdaFunctionARN + layers: + - !FindInMap [layers, !Ref Runtime, !Ref Arch] + env: + AWS_LAMBDA_EXEC_WRAPPER: !Ref WrapperHandler + + + AddConfigLambdaFunction: + Type: AWS::Serverless::Function + Properties: + FunctionName: coralogix-otel-lambda-configure + Handler: index.lambda_handler + Runtime: python3.9 + Policies: + - Statement: + - Sid: AllowLambdaConfigUpdatePolicy + Effect: Allow + Action: + - lambda:UpdateFunctionConfiguration + Resource: "*" + + InlineCode: | + import boto3 + import json + import cfnresponse + + def handler(event, context): + print("Received event: " + json.dumps(event, indent=2)) + lambda_client = boto3.client('lambda') + lambda_client.update_function_configuration( + FunctionName=event['ResourceProperties']['LambdaArn'], + Layers=event['ResourceProperties']['layers'], + Environment={ + 'Variables': event['ResourceProperties']['env'] + } + ) + return cfnresponse.SUCCESS + From ea46856b01c7e3efb1606b1177c31d868a3088fb Mon Sep 17 00:00:00 2001 From: Shaun Remekie Date: Thu, 16 May 2024 16:31:07 +0200 Subject: [PATCH 4/4] removed unnecessary files --- opentelemetry/lambda/script.py | 45 ---------------- opentelemetry/lambda/template.yaml | 87 ------------------------------ 2 files changed, 132 deletions(-) delete mode 100644 opentelemetry/lambda/script.py delete mode 100644 opentelemetry/lambda/template.yaml diff --git a/opentelemetry/lambda/script.py b/opentelemetry/lambda/script.py deleted file mode 100644 index 7a3f2af6..00000000 --- a/opentelemetry/lambda/script.py +++ /dev/null @@ -1,45 +0,0 @@ -import boto3 -import cfnresponse -import json - -def handler(event, context): - print("received event: " + json.dumps(event, indent=2)) - - try: - status = cfnresponse.SUCCESS - - if event['RequestType'] in ['Create', 'Update']: - lambda_arn = event['ResourceProperties']['LambdaArn'] - lambda_client = boto3.client('lambda') - - # Get the current environment variables of the Lambda function - function_configuration = lambda_client.get_function_configuration(FunctionName=lambda_arn) - current_env = function_configuration['Environment']['Variables'] - - # Merge the current environment variables with any new ones specified in the CloudFormation event - # new_environment = current_environment.copy() - current_env.update(event['ResourceProperties']['env']) - - # Get the list of layers to add to the Lambda function - layers = event['ResourceProperties'].get('LayerArns', []) - - # Add the specified layers and environment variables to the Lambda function - lambda_client.update_function_configuration( - FunctionName=lambda_arn, - Environment={'Variables': current_env}, - Layers=layers) - - if event['RequestType'] == 'Delete': - # TODO: implement delete logic - pass - - - except Exception as e: - print(f"custom resource failed to update layers and env vars: {e}") - status = cfnresponse.FAILED - - finally: - # Send a status response to CloudFormation - response_data = {} - cfnresponse.send(event, context, status, response_data) - return diff --git a/opentelemetry/lambda/template.yaml b/opentelemetry/lambda/template.yaml deleted file mode 100644 index e9ae4623..00000000 --- a/opentelemetry/lambda/template.yaml +++ /dev/null @@ -1,87 +0,0 @@ -AWSTemplateFormatVersion: '2010-09-09' -Transform: AWS::Serverless-2016-10-31 - -Parameters: - LambdaFunctionARN: - Type: String - Description: The ARN of the Lambda function to invoke - - WrapperHandler: - Type: String - Description: The path to the otel instrumentation executable - Default: /opt/python/otel-instrument - - Runtime: - Type: String - Description: The language of the Lambda function - Default: python - AllowedValues: - - python - - nodejs - - go - - Arch: - Type: String - Description: The architecture of the Lambda function - Default: amd64 - AllowedValues: - - amd64 - - arm64 - -Mappings: - layers: - python: - amd64: arn:aws:lambda:eu-north-1:035955823196:layer:coralogix-aws-lambda-telemetry-exporter-python-x86_64:1 - arm64: arn:aws:lambda:eu-north-1:035955823196:layer:coralogix-aws-lambda-telemetry-exporter-python-arm64:1 - - RegionMap: - us-east-1: - LambdaLayerARN: arn:aws:lambda:us-east-1:901920570463:layer:aws-otel-python38-ver-1-0-0:1 - us-east-2: - LambdaLayerARN: arn:aws:lambda:us-east-2:901920570463:layer:aws-otel-python38-ver-1-0-0:1 - us-west-2: - LambdaLayerARN: arn:aws:lambda:us-west-2:901920570463:layer:aws-otel-python38-ver-1-0-0:1 - -Resources: - CustomResourceConfigureLambdaFunction: - Type: 'AWS::CloudFormation::CustomResource' - Properties: - ServiceToken: !GetAtt AddConfigLambdaFunction.Arn - LambdaArn: !Ref LambdaFunctionARN - layers: - - !FindInMap [layers, !Ref Runtime, !Ref Arch] - env: - AWS_LAMBDA_EXEC_WRAPPER: !Ref WrapperHandler - - - AddConfigLambdaFunction: - Type: AWS::Serverless::Function - Properties: - FunctionName: coralogix-otel-lambda-configure - Handler: index.lambda_handler - Runtime: python3.9 - Policies: - - Statement: - - Sid: AllowLambdaConfigUpdatePolicy - Effect: Allow - Action: - - lambda:UpdateFunctionConfiguration - Resource: "*" - - InlineCode: | - import boto3 - import json - import cfnresponse - - def handler(event, context): - print("Received event: " + json.dumps(event, indent=2)) - lambda_client = boto3.client('lambda') - lambda_client.update_function_configuration( - FunctionName=event['ResourceProperties']['LambdaArn'], - Layers=event['ResourceProperties']['layers'], - Environment={ - 'Variables': event['ResourceProperties']['env'] - } - ) - return cfnresponse.SUCCESS -