added concurrency settings

Author: dave

README.md

@@ -24,20 +24,21 @@ $ ucdn upload -c /path/to/config/file.yaml -C production
 
 ## Options
 
-All following options except `config` and `config-key` can be configured using the config file.
+All following options except `config`, `config-key` and `concurrencyLimit` can be configured using the config file.
 
 Available AWS regions can be viewed at https://docs.aws.amazon.com/sns/latest/dg/sns-supported-regions-countries.html.
 
 ```
---config, -c                            path to config file           [default: "./ucdn.yml"]
---config-key, -C                        root config key               [default: null]
---region, -r                            AWS geographical area         [default: "eu-west-1"]
---dir, -d                               assets directory              [default: "dist/"]
---bucket, -b                            AWS bucket for upload         [required]
---exclude, -e                           excluded extenstions          [default: ["html","gz"]]
---accessKeyId, --access-key-id          AWS access key ID             [required]
---secretAccessKey, --secret-access-key  AWS secret access key         [required]
---targetDir, --target-dir               AWS bucket target directory   [default: ""]
+--config, -c                            path to config file                    [default: "./ucdn.yml"]
+--config-key, -C                        root config key                        [default: null]
+--region, -r                            AWS geographical area                  [default: "eu-west-1"]
+--dir, -d                               assets directory                       [default: "dist/"]
+--bucket, -b                            AWS bucket for upload                  [required]
+--exclude, -e                           excluded extenstions                   [default: ["html","gz"]]
+--accessKeyId, --access-key-id          AWS access key ID                      [required]
+--secretAccessKey, --secret-access-key  AWS secret access key                  [required]
+--targetDir, --target-dir               AWS bucket target directory            [default: ""]
+--concurrencyLimit, --processes         Limit of concurrent upload processes   [default: 100]
 ```
 
 ## Contributing

package.json

@@ -14,6 +14,7 @@
     "@aws-sdk/client-s3": "^3.421.0",
     "js-yaml": "^4.1.0",
     "mime-types": "^2.1.35",
+    "p-limit": "^6.2.0",
     "yargs": "^17.7.2"
   },
   "devDependencies": {

ucdn

@@ -59,6 +59,12 @@ const yargsOptions = {
     type: "string",
     default: "",
   },
+  concurrencyLimit: {
+    alias: "processes",
+    describe: "Limit of concurrent upload processes",
+    type: "integer",
+    default: 100,
+  },
 }
 
 yargs

upload.js

@@ -2,6 +2,7 @@ const fs = require("fs")
 const path = require("path")
 const mime = require("mime-types")
 const { S3Client, PutObjectCommand } = require("@aws-sdk/client-s3")
+const pLimit = require("p-limit")
 
 const configDefaults = {
   region: null,
@@ -11,6 +12,7 @@ const configDefaults = {
   accessKeyId: null,
   secretAccessKey: null,
   targetDir: null,
+  concurrencyLimit: 100,
 }
 
 const getConfig = passedConfig => {
@@ -61,10 +63,12 @@ const upload = argv => {
     return !exclude.includes(ext)
   })
 
+  const limit = pLimit(config.concurrencyLimit);
+
   const promises = files.map(file => {
     const key = getKey(file, directory, targetDir)
-    return uploadFile(s3, config, file, key)
-      .then(() => console.log("Uploaded", file))
+    return limit(uploadFile(s3, config, file, key)
+      .then(() => console.log("Uploaded", file)))
   })
 
   return Promise.all(promises)

yarn.lock

@@ -2511,6 +2511,13 @@ p-limit@^3.0.2:
   dependencies:
     yocto-queue "^0.1.0"
 
+p-limit@^6.2.0:
+  version "6.2.0"
+  resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-6.2.0.tgz#c254d22ba6aeef441a3564c5e6c2f2da59268a0f"
+  integrity sha512-kuUqqHNUqoIWp/c467RI4X6mmyuojY5jGutNU0wVTmEOOfcuwLqyMVoAi9MKi2Ak+5i9+nhmrK4ufZE8069kHA==
+  dependencies:
+    yocto-queue "^1.1.1"
+
 p-locate@^5.0.0:
   version "5.0.0"
   resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-5.0.0.tgz#83c8315c6785005e3bd021839411c9e110e6d834"
@@ -2968,3 +2975,8 @@ yocto-queue@^0.1.0:
   version "0.1.0"
   resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b"
   integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==
+
+yocto-queue@^1.1.1:
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-1.2.1.tgz#36d7c4739f775b3cbc28e6136e21aa057adec418"
+  integrity sha512-AyeEbWOu/TAXdxlV9wmGcR0+yh2j3vYPGOECcIj2S7MkrLyC7ne+oye2BKTItt0ii2PHk4cDy+95+LshzbXnGg==