Mc 1578 add event bridge rule

.aws/src/elasticache.ts

@@ -7,19 +7,22 @@ import {
 
 export class Elasticache extends Construct {
   public readonly nodeList: string[];
+  public readonly clusterArn: string;
 
   constructor(scope: Construct, name: string) {
     super(scope, name);
 
-    this.nodeList = Elasticache.createElasticache(scope);
+    const { nodeList, clusterArn } = Elasticache.createElasticache(scope);
+    this.nodeList = nodeList;
+    this.clusterArn = clusterArn;
   }
 
   /**
-   * Creates the elasticache and returns the node address list
+   * Creates the Elasticache cluster and returns the node list and cluster ARN.
    * @param scope
    * @private
    */
-  private static createElasticache(scope: Construct): string[] {
+  private static createElasticache(scope: Construct): { nodeList: string[]; clusterArn: string } {
     const pocketVPC = new PocketVPC(scope, 'pocket-shared-vpc');
 
     const elasticache = new ApplicationMemcache(scope, 'memcached', {
@@ -44,11 +47,14 @@ export class Elasticache extends Construct {
       // its rendering -1.8881545897087503e+289 for some weird reason...
       // For now we just hardcode to 11211 which is the default memcache port.
       nodeList.push(
-        `${
-            elasticache.elasticacheCluster.cacheNodes.get(i).address
-        }:11211`
+          `${
+              elasticache.elasticacheCluster.cacheNodes.get(i).address
+          }:11211`
       );
     }
-    return nodeList;
+
+    const clusterArn = elasticache.elasticacheCluster.arn;
+
+    return { nodeList, clusterArn };
   }
 }

.aws/src/main.ts

@@ -6,6 +6,7 @@ import {DynamoDB} from "./dynamodb";
 import {PocketALBApplication, PocketECSCodePipeline} from "@pocket-tools/terraform-modules";
 import {SqsLambda} from "./sqsLambda";
 import {Elasticache} from "./elasticache";
+import {RemoveItemLambda} from "./removeItemLambda";
 import {RecommendationApiSynthetics} from './monitoring';
 
 import {ArchiveProvider} from '@cdktf/provider-archive/lib/provider';
@@ -40,12 +41,13 @@ class RecommendationAPI extends TerraformStack {
         const caller = new DataAwsCallerIdentity(this, 'caller');
 
         const dynamodb = new DynamoDB(this, 'dynamodb');
+        const elasticache = new Elasticache(this, 'elasticache');
 
         const pocketApp = this.createPocketAlbApplication({
             secretsManagerKmsAlias: this.getSecretsManagerKmsAlias(),
             region,
             caller,
-            elasticache: new Elasticache(this, 'elasticache'),
+            elasticache,
             dynamodb: dynamodb
         });
 
@@ -54,7 +56,11 @@ class RecommendationAPI extends TerraformStack {
         const synthetic = new RecommendationApiSynthetics(this, 'synthetics');
         synthetic.createSyntheticCheck([]);
 
+        // Lambda for storing candidate sets for Pocket Explore & Topic pages in DynamoDB.
         new SqsLambda(this, 'sqs-lambda', dynamodb.candidateSetsTable);
+
+        // Lambda for caching removed items, to avoid sending them to Pocket Home.
+        new RemoveItemLambda(this, 'remove-item-lambda', elasticache);
     }
 
 

.aws/src/removeItemLambda.ts

@@ -0,0 +1,103 @@
+import {Construct} from 'constructs';
+import {config} from './config';
+import {
+  LAMBDA_RUNTIMES,
+  PocketSQSWithLambdaTarget,
+  PocketVPC
+} from '@pocket-tools/terraform-modules';
+import {CloudwatchEventRule} from '@cdktf/provider-aws/lib/cloudwatch-event-rule';
+import {CloudwatchEventTarget} from '@cdktf/provider-aws/lib/cloudwatch-event-target';
+import {DataAwsSsmParameter} from '@cdktf/provider-aws/lib/data-aws-ssm-parameter';
+import {Elasticache} from './elasticache';
+import {SqsQueuePolicy} from '@cdktf/provider-aws/lib/sqs-queue-policy';
+
+export class RemoveItemLambda extends Construct {
+  constructor(scope: Construct, name: string, elasticache: Elasticache) {
+    super(scope, name);
+
+    const vpc = new PocketVPC(this, 'pocket-shared-vpc');
+
+    const { sentryDsn, gitSha } = this.getEnvVariableValues();
+
+    const sqsWithLambda = new PocketSQSWithLambdaTarget(this, 'remove-sqs-lambda', {
+      name: `${config.prefix}-Sqs-RemoveEventHandler`,
+      batchSize: 1,
+      sqsQueue: {
+        maxReceiveCount: 3, // 2 retries
+        visibilityTimeoutSeconds: 300,
+      },
+      lambda: {
+        runtime: 'python3.9' as LAMBDA_RUNTIMES,
+        handler: 'remove_item_lambda.sqs_handler.handler',
+        timeout: 120,
+        executionPolicyStatements: [
+          {
+            effect: 'Allow',
+            actions: ['elasticache:ModifyCacheCluster', 'elasticache:DescribeCacheClusters'],
+            resources: [elasticache.clusterArn],
+          },
+        ],
+        environment: {
+          SENTRY_DSN: sentryDsn,
+          GIT_SHA: gitSha,
+          ENVIRONMENT: config.environment === 'Prod' ? 'production' : 'development',
+          ELASTICACHE_SERVERS: elasticache.nodeList.join(','), // Pass node list to Lambda
+        },
+        vpcConfig: {
+          securityGroupIds: vpc.internalSecurityGroups.ids,
+          subnetIds: vpc.privateSubnetIds,
+        },
+        codeDeploy: {
+          region: 'us-east-1',
+          accountId: '410318598490',
+        },
+      },
+    });
+
+    const eventRule = new CloudwatchEventRule(this, 'remove-event-rule', {
+      name: `${config.prefix}-RemoveEventRule`,
+      description: 'Event rule for REMOVE_ITEM events to SQS',
+      // source and detail-type are defined in:
+      // https://github.com/Pocket/content-monorepo/blob/main/servers/curated-corpus-api/src/config/index.ts
+      eventPattern: JSON.stringify({
+        source: ['curation-migration-datasync'],
+        'detail-type': ['remove-approved-item'],
+      }),
+      eventBusName: 'default',
+    });
+
+    new CloudwatchEventTarget(this, 'remove-event-target', {
+      rule: eventRule.name,
+      arn: sqsWithLambda.sqsQueueResource.arn,
+    });
+
+    // Allow the EventRule to invoke the SQS Queue
+    new SqsQueuePolicy(this, 'sqs-queue-policy', {
+      queueUrl: sqsWithLambda.sqsQueueResource.url,
+      policy: JSON.stringify({
+        Version: '2012-10-17',
+        Statement: [
+          {
+            Effect: 'Allow',
+            Principal: { Service: 'events.amazonaws.com' },
+            Action: 'sqs:SendMessage',
+            Resource: sqsWithLambda.sqsQueueResource.arn,
+            Condition: { ArnEquals: { 'aws:SourceArn': eventRule.arn } },
+          },
+        ],
+      }),
+    });
+  }
+
+  private getEnvVariableValues() {
+    const sentryDsn = new DataAwsSsmParameter(this, 'sentry-dsn', {
+      name: `/${config.name}/${config.environment}/SENTRY_DSN`,
+    });
+
+    const serviceHash = new DataAwsSsmParameter(this, 'service-hash', {
+      name: `${config.circleCIPrefix}/SERVICE_HASH`,
+    });
+
+    return { sentryDsn: sentryDsn.value, gitSha: serviceHash.value };
+  }
+}

.circleci/config.yml

@@ -166,15 +166,24 @@ jobs:
           path: test-reports
 
   lambda:
-    description: Builds and Optionaly deploys all the associated lambdas
+    description: Builds and Optionally deploys associated lambdas
     parameters:
       env_lower_name:
         type: string
-        description: The lower case env name
+        description: The lower case environment name (e.g., dev or prod)
       env_capital_name:
-        default: Env Name
-        description: The env capital name
         type: string
+        description: The capitalized environment name (e.g., Dev or Prod)
+      lambda_lower_name:
+        type: string
+        description: Lowercase part of the Lambda name (e.g., sqs-removeeventhandler or sqs-translation)
+      lambda_capital_name:
+        type: string
+        description: Capitalized part of the Lambda name (e.g., SqsRemoveeventhandler or SqsTranslation)
+      lambda_path:
+        type: string
+        description: Path to the Lambda source code (e.g., aws_lambda/ or remove_item_lambda/)
+        default: "aws_lambda/"
       deploy:
         type: boolean
         default: true
@@ -204,26 +213,27 @@ jobs:
                 command: |
                   apt-get update && apt-get install zip
                   pip install pipenv==2022.8.15
-                  cd aws_lambda
+                  cd <<parameters.lambda_path>>
                   pipenv requirements > requirements.txt
                   pip install -r requirements.txt --no-deps -t package
                   cd package
                   mkdir -p /tmp
                   zip -r9 "/tmp/$CIRCLE_SHA1.zip" . -x \*__pycache__\* \.git\*
                   cd .. && rm -rf package __pycache__ requirements.txt && cd ..
-                  zip -gr "/tmp/$CIRCLE_SHA1.zip" aws_lambda -x aws_lambda/Pipfile*
+                  zip -gr "/tmp/$CIRCLE_SHA1.zip" <<parameters.lambda_path>> -x <<parameters.lambda_path>>/Pipfile*
                   cp "/tmp/$CIRCLE_SHA1.zip" /tmp/build.zip
             - run:
-                name: Upload Package
-                command: aws s3 cp "/tmp/$CIRCLE_SHA1.zip" s3://pocket-recommendationapi-<< parameters.env_lower_name >>-sqs-translation/
+                name: Upload Package to S3
+                command: |
+                  aws s3 cp "/tmp/$CIRCLE_SHA1.zip" s3://pocket-recommendationapi-<< parameters.env_lower_name >>-<< parameters.lambda_lower_name >>/
             - pocket/deploy_lambda:
-                s3-bucket: pocket-recommendationapi-<< parameters.env_lower_name >>-sqs-translation
+                s3-bucket: pocket-recommendationapi-<< parameters.env_lower_name >>-<< parameters.lambda_lower_name >>
                 aws-access-key-id: << parameters.env_capital_name >>_AWS_ACCESS_KEY
                 aws-secret-access-key: << parameters.env_capital_name >>_AWS_SECRET_ACCESS_KEY
                 aws-region: << parameters.env_capital_name >>_AWS_DEFAULT_REGION
-                codedeploy-application-name: RecommendationAPI-<< parameters.env_capital_name >>-Sqs-Translation-Lambda
-                codedeploy-deployment-group-name: RecommendationAPI-<< parameters.env_capital_name >>-Sqs-Translation-Lambda
-                function-name: RecommendationAPI-<< parameters.env_capital_name >>-Sqs-Translation-Function
+                codedeploy-application-name: RecommendationAPI-<< parameters.env_capital_name >>-<< parameters.lambda_capital_name >>-Lambda
+                codedeploy-deployment-group-name: RecommendationAPI-<< parameters.env_capital_name >>-<< parameters.lambda_capital_name >>-Lambda
+                function-name: RecommendationAPI-<< parameters.env_capital_name >>-<< parameters.lambda_capital_name >>-Function
 
       - store_artifacts:
           path: /tmp/build.zip
@@ -243,24 +253,52 @@ workflows:
 
       - build
 
-      # Build & Deploy Development Lambdas
+      # Deploy Dev Lambdas
+      - lambda:
+          <<: *only_dev
+          context: pocket
+          name: deploy_lambda_dev_remove_item
+          env_lower_name: dev
+          env_capital_name: Dev
+          lambda_lower_name: sqs-removeeventhandler
+          lambda_capital_name: Sqs-RemoveEventHandler
+          lambda_path: "remove_item_lambda/"
+          deploy: true
+
       - lambda:
           <<: *only_dev
           context: pocket
-          name: deploy_lambdas_dev
+          name: deploy_lambda_dev_translation
           env_lower_name: dev
           env_capital_name: Dev
+          lambda_lower_name: sqs-translation
+          lambda_capital_name: Sqs-Translation
+          lambda_path: "aws_lambda/"
+          deploy: true
+
+      # Deploy Prod Lambdas
+      - lambda:
+          <<: *only_main
+          context: pocket
+          name: deploy_lambda_prod_remove_item
+          env_lower_name: prod
+          env_capital_name: Prod
+          lambda_lower_name: sqs-removeeventhandler
+          lambda_capital_name: Sqs-RemoveEventHandler
+          lambda_path: "remove_item_lambda/"
           deploy: true
           requires:
-            - setup-deploy-params-dev
+            - setup-deploy-params-prod
 
-      # Build & Deploy Production Lambdas
       - lambda:
           <<: *only_main
           context: pocket
-          name: deploy_lambdas_prod
+          name: deploy_lambda_prod_translation
           env_lower_name: prod
           env_capital_name: Prod
+          lambda_lower_name: sqs-translation
+          lambda_capital_name: Sqs-Translation
+          lambda_path: "aws_lambda/"
           deploy: true
           requires:
             - setup-deploy-params-prod

remove_item_lambda/Pipfile

@@ -0,0 +1,22 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+pytest = "*"
+pytest-cov = "*"
+coverage = "*"
+requests = "*"
+moto = "*"
+mypy_boto3_dynamodb = "*"
+pytest-mock = "*"
+
+[packages]
+boto3 = "*"
+# We define the essential stubs
+boto3-stubs = {extras = ["essential"], version = "*"}
+sentry-sdk = "*"
+
+[requires]
+python_version = "3.9"