From 951e863e395d9ccbeff86c28101c31ee8f2fe515 Mon Sep 17 00:00:00 2001 From: Kelvin Luck Date: Mon, 4 Jul 2022 21:53:36 +0100 Subject: [PATCH 1/5] feat: Introduce an optional escape hatch for `listAllObjects` There are cases where we might want to page through the list of objects available on S3 but we don't need to keep going once we've found what we are looking for. This commit updates `listAllObjects` to accept an `until` argument - if this is provided then when a new page of results is loaded we check if `some` of them match the `until` - if so we don't bother loading another page. A concrete use-case for this is in #120 - sometimes we just want to find the currently active revision so we only need to loop through the "pages" on S3 until we do. --- lib/s3.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/s3.js b/lib/s3.js index c821371..1c6fca6 100644 --- a/lib/s3.js +++ b/lib/s3.js @@ -174,7 +174,7 @@ module.exports = CoreObject.extend({ }); }, - listAllObjects: function(options) { + listAllObjects: function(options, until) { var client = this._client; var listObjects = RSVP.denodeify(client.listObjects.bind(client)); var allRevisions = []; @@ -183,13 +183,14 @@ module.exports = CoreObject.extend({ return listObjects(options).then(function(response) { [].push.apply(allRevisions, response.Contents); - if (response.IsTruncated) { - var nextMarker = response.Contents[response.Contents.length - 1].Key; - options.Marker = nextMarker; - return listObjectRecursively(options); - } else { + var isComplete = !response.IsTruncated || (until && response.Contents.some(until)); + + if (isComplete) { return allRevisions; } + var nextMarker = response.Contents[response.Contents.length - 1].Key; + options.Marker = nextMarker; + return listObjectRecursively(options); }); } From ccfd4270fdfc6386639b23cf01cf9b61d1d9774a Mon Sep 17 00:00:00 2001 From: Kelvin Luck Date: Mon, 4 Jul 2022 21:55:46 +0100 Subject: [PATCH 2/5] feat: Introduce a `getActiveRevision` method This loads the head object and then calls `listAllObjects` with the new `until` parameter and stops as soon as it has found an object which matches the `ETag` of the active revision. It returns the active revision in the same format that `fetchRevisions` would - for backward compatibility with use-cases which were depending on that. --- lib/s3.js | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/lib/s3.js b/lib/s3.js index 1c6fca6..cc50f90 100644 --- a/lib/s3.js +++ b/lib/s3.js @@ -152,6 +152,31 @@ module.exports = CoreObject.extend({ .then((response) => response.Contents.find((element) => element.Key === revisionPrefix)); }, + getActiveRevision(options) { + var client = this._client; + var bucket = options.bucket; + var prefix = options.prefix; + var revisionPrefix = joinUriSegments(prefix, options.filePattern + ":"); + var indexKey = joinUriSegments(prefix, options.filePattern); + + return headObject(client, { Bucket: bucket, Key: indexKey }) + .then((current) => this.listAllObjects( + { Bucket: bucket, Prefix: revisionPrefix }, + (element) => element.ETag === current.ETag + ) + .then((elements) => { + let activeRevision = elements.find((element) => element.ETag === current.ETag); + if (!activeRevision) { + return; + } + + var revision = activeRevision.Key.substr(revisionPrefix.length); + return { active: true, revision, timestamp: activeRevision.LastModified }; + }) + ); + + }, + fetchRevisions: function(options) { var client = this._client; var bucket = options.bucket; From 8000d88a7a7f2770d875a52cb4742fc68cf71585 Mon Sep 17 00:00:00 2001 From: Kelvin Luck Date: Mon, 4 Jul 2022 21:59:59 +0100 Subject: [PATCH 3/5] feat: Add `fetchOnlyRelevantRevisions` config option when activating If you set this option in your S3 config then we will only load enough revisions from S3 to find the currently active version and then we will stop. The assumption here is that the main use-case for `fetchInitialRevisions` is to be able to do some sort of changelog or audit log from another plugin where we would want to know the details of the previously active revision. Since looping over revisions from S3 can be slow when you have too many of them stored (see #120) we provide an option to short circuit that. --- index.js | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/index.js b/index.js index bb1edaa..0cd5b66 100644 --- a/index.js +++ b/index.js @@ -116,6 +116,12 @@ module.exports = { }, fetchInitialRevisions: function(context) { + + if (this.readConfig('fetchOnlyRelevantRevisions')) { + this.log('fetchInitialRevisions - fetching only enough revisions to find the active revision', { verbose: true }); + return this._fetchActiveRevision(); + } + return this._list(context) .then(function(revisions) { return { @@ -124,6 +130,24 @@ module.exports = { }); }, + _fetchActiveRevision: function() { + var bucket = this.readConfig('bucket'); + var prefix = this.readConfig('prefix'); + var filePattern = this.readConfig('filePattern'); + + var options = { + bucket: bucket, + prefix: prefix, + filePattern: filePattern, + }; + + var s3 = new this.S3({ plugin: this }); + return s3.getActiveRevision(options) + .then((activeRevision) => { + return { initialRevisions: [activeRevision] }; + }); + }, + _list: function(/* context */) { var bucket = this.readConfig('bucket'); var prefix = this.readConfig('prefix'); From cc321efcbacd9df05b020a6c6d2acaf9dec9b7a1 Mon Sep 17 00:00:00 2001 From: Kelvin Luck Date: Mon, 4 Jul 2022 22:07:26 +0100 Subject: [PATCH 4/5] feat: Introduce `disableFetchRevisions` config flag If you pass this flag then `fetchRevisions` doesn't attempt to fetch _any_ revisions from S3. This is useful because fetching the revisions from S3 can take a long time (see #120) and if you don't have any reason to use them then this time is wasted. In our use case I have hooked this flag up to an environment variable. e.g. - in `deploy.js`: ``` disableFetchRevisions: process.env.DISABLE_FETCH_REVISIONS ``` Then we can set that environment variable when we are calling the `activate` command and leave it off at any other time (so e.g. `ember deploy:list` works as expected - if you have the time to wait) --- index.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/index.js b/index.js index 0cd5b66..e2110da 100644 --- a/index.js +++ b/index.js @@ -107,6 +107,13 @@ module.exports = { }, fetchRevisions: function(context) { + if (this.readConfig('disableFetchRevisions')) { + this.log('fetchRevisions - not fetching any revisions', { verbose: true }); + return { + activations: [] + }; + } + return this._list(context) .then(function(revisions) { return { From 0149aa69af2738558078046e9872c832012fd151 Mon Sep 17 00:00:00 2001 From: Kelvin Luck Date: Tue, 5 Jul 2022 10:04:19 +0100 Subject: [PATCH 5/5] fix: Handle the case where there is no active revision It's possible you are deploying or activating an app which hasn't previously been activated. In these cases we won't be able to find a currently active revision and we shouldn't error --- lib/s3.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/s3.js b/lib/s3.js index cc50f90..1d10d91 100644 --- a/lib/s3.js +++ b/lib/s3.js @@ -160,7 +160,12 @@ module.exports = CoreObject.extend({ var indexKey = joinUriSegments(prefix, options.filePattern); return headObject(client, { Bucket: bucket, Key: indexKey }) - .then((current) => this.listAllObjects( + .then((current) => { + if (current === undefined) { + return; + } + + return this.listAllObjects( { Bucket: bucket, Prefix: revisionPrefix }, (element) => element.ETag === current.ETag ) @@ -173,7 +178,7 @@ module.exports = CoreObject.extend({ var revision = activeRevision.Key.substr(revisionPrefix.length); return { active: true, revision, timestamp: activeRevision.LastModified }; }) - ); + }); },