From 291aba2d93c1460ebe04f611dc9a8ac47c7750ad Mon Sep 17 00:00:00 2001 From: Subramanian Venkateswaran Date: Sun, 18 Jun 2017 23:06:01 -0700 Subject: [PATCH 1/4] Removed streambot dependency --- index.js | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/index.js b/index.js index 4c3b298..c40fac2 100644 --- a/index.js +++ b/index.js @@ -3,16 +3,11 @@ var Dyno = require('dyno'); var queue = require('queue-async'); var crypto = require('crypto'); var https = require('https'); -var streambot = require('streambot'); module.exports.replicate = replicate; -module.exports.streambotReplicate = streambot(function(event, callback) { - replicate(event, {}, callback); -}); + module.exports.backup = incrementalBackup; -module.exports.streambotBackup = streambot(function(event, callback) { - incrementalBackup(event, {}, callback); -}); + module.exports.snapshot = require('./s3-snapshot'); module.exports.agent = new https.Agent({ keepAlive: true, @@ -68,6 +63,7 @@ function replicate(event, context, callback) { var params = { RequestItems: {} }; params.RequestItems[process.env.ReplicaTable] = Object.keys(allRecords).map(function(key) { var change = allRecords[key].pop(); + console.log('change = ', change); if (change.eventName === 'INSERT' || change.eventName === 'MODIFY') { return { PutRequest: { Item: Dyno.deserialize(JSON.stringify(change.dynamodb.NewImage)) } @@ -173,7 +169,7 @@ function incrementalBackup(event, context, callback) { q.awaitAll(function(err) { if (err) throw err; - callback(); + callback(null, "Sucessful back up"); }); function backupRecord(changes, callback) { @@ -193,6 +189,8 @@ function incrementalBackup(event, context, callback) { }; var req = change.eventName === 'REMOVE' ? 'deleteObject' : 'putObject'; + console.log('Event = ', change.eventName); + console.log('Change = ', JSON.stringify(change.dynamodb)); if (req === 'putObject') params.Body = JSON.stringify(change.dynamodb.NewImage); s3[req](params, function(err) { @@ -223,4 +221,4 @@ function incrementalBackup(event, context, callback) { callback(); }); } -} +} \ No newline at end of file From 34cb0819efc45c2ff561f6d2018823261e0099a2 Mon Sep 17 00:00:00 2001 From: Subramanian Venkateswaran Date: Mon, 19 Jun 2017 00:45:32 -0700 Subject: [PATCH 2/4] Updated readme with instructions to run --- README.md | 98 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/README.md b/README.md index 0898963..61219b4 100644 --- a/README.md +++ b/README.md @@ -185,3 +185,101 @@ Usage: incremental-record-history # Read the history of a single record $ incremental-record-history us-east-1/my-table s3://dynamodb-backups/incremental '{"id":"abc"}' ``` + +### Instructions to run: +##### Streambot dependency removed since it is deprecated + +The code could be deployed on a lambda for incremental backups to S3 and for replication across DynamoDB tables across regions and accounts. + +Initial setup: +``` +$ git clone https://github.com/subbuvenk94/dynamodb-replicator.git +$ cd dynamodb-replicator +$ npm install +``` +After the dependent packages are installed, zip the contents of the root directory. Deploy the zip on to AWS Lambda console with the handler as ```index.backup``` or ```index.replicate``` based on your required functionality. + +The Lambda console lets us set the environment variables. Refer to this: +``` +## Backup Functionality +# Target bucket on S3 +BackupBucket= +# Prefix for the data inside the bucket (Optional) +BackupPrefix= +# Region for the bucket (Optional) +BackupRegion= + +## Replication Functionality +# Key/Secret for a different AWS account write (Optional) +ReplicaAccessKeyId= +ReplicaSecretAccesseKey= +# Name of replica table +ReplicaTable= +# Replica table region (NOTE: We have to manually create this table for the same key) +ReplicaRegion= +#Endpoint for replica table (Optional) +ReplicaEndpoint= +``` +Enable triggers for the tables on which these operations are made with view type ```New and old images - both the new and the old images of the item```. Note that the 'Triggers' option on DynamoDB tables are available based on region. Create the trigger for the existing Lambda function that we just created. + +For tables in which data already exists, the existing data has to be first replicated before we could do the incremental replication. Refer to [diff-tables](https://github.com/subbuvenk94/dynamodb-replicator#incremental-diff-record) for replication backfill. Similarly, for we can perform [incremental backfill](https://github.com/subbuvenk94/dynamodb-replicator#incremental-backfill). + +The Lambda function would need permissions to perform the incremental backup or replication tasks. The IAM policy that should be created to award this permission could be set as: +``` +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Resource": "arn:aws:logs:*:*:*" + }, + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject" + ], + "Resource": [ + "arn:aws:s3:::/*" + ] + }, + { + "Effect": "Allow", + "Action": "lambda:InvokeFunction", + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "dynamodb:DescribeStream", + "dynamodb:GetRecords", + "dynamodb:GetShardIterator", + "dynamodb:ListStreams", + "dynamodb:DescribeTable", + "dynamodb:PutItem", + "dynamodb:BatchWriteItem", + "dynamodb:Scan" + ], + "Resource": "*" + } + ] +} +``` +NOTE: Edit the BACKUP_BUCKET_NAME to the name of the backup bucket. + +After this setup, any changes to the tables should trigger the Lambda which would perform the incremental backup or the table repication. Your changes would be visible in the respective table or bucket. From 6eba17dbdf11cffa515085ac111a02da83ea0bc2 Mon Sep 17 00:00:00 2001 From: Subramanian Venkateswaran Date: Mon, 19 Jun 2017 00:48:21 -0700 Subject: [PATCH 3/4] Updated readme with instructions to run --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 61219b4..dba5bff 100644 --- a/README.md +++ b/README.md @@ -222,7 +222,7 @@ ReplicaEndpoint= ``` Enable triggers for the tables on which these operations are made with view type ```New and old images - both the new and the old images of the item```. Note that the 'Triggers' option on DynamoDB tables are available based on region. Create the trigger for the existing Lambda function that we just created. -For tables in which data already exists, the existing data has to be first replicated before we could do the incremental replication. Refer to [diff-tables](https://github.com/subbuvenk94/dynamodb-replicator#incremental-diff-record) for replication backfill. Similarly, for we can perform [incremental backfill](https://github.com/subbuvenk94/dynamodb-replicator#incremental-backfill). +For tables in which data already exists, the existing data has to be first replicated before we could do the incremental replication. Refer to [diff-tables](#incremental-diff-record) for replication backfill. Similarly, for we can perform [incremental backfill](#incremental-backfill). The Lambda function would need permissions to perform the incremental backup or replication tasks. The IAM policy that should be created to award this permission could be set as: ``` From 601c64c0902e1667ba49c4b827430fce9669f8c4 Mon Sep 17 00:00:00 2001 From: Subramanian Venkateswaran Date: Mon, 19 Jun 2017 00:49:10 -0700 Subject: [PATCH 4/4] Updated readme with instructions to run --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dba5bff..23f0d51 100644 --- a/README.md +++ b/README.md @@ -222,7 +222,7 @@ ReplicaEndpoint= ``` Enable triggers for the tables on which these operations are made with view type ```New and old images - both the new and the old images of the item```. Note that the 'Triggers' option on DynamoDB tables are available based on region. Create the trigger for the existing Lambda function that we just created. -For tables in which data already exists, the existing data has to be first replicated before we could do the incremental replication. Refer to [diff-tables](#incremental-diff-record) for replication backfill. Similarly, for we can perform [incremental backfill](#incremental-backfill). +For tables in which data already exists, the existing data has to be first replicated before we could do the incremental replication. Refer to [diff-tables](#diff-tables) for replication backfill. Similarly, for we can perform [incremental backfill](#incremental-backfill). The Lambda function would need permissions to perform the incremental backup or replication tasks. The IAM policy that should be created to award this permission could be set as: ```