add ITEMS_MAX_LIMIT configuration to override hard-coded default of 10000 (#942)

* add @types/compression dependency so 'npm run test:unit' will run

* add ITEMS_MAX_LIMIT variable to configure max items in a search response

* handle invalid value for ITEMS_MAX_LIMIT better

Author: philvarner

CHANGELOG.md

@@ -13,6 +13,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
   `ENABLE_RESPONSE_COMPRESSION` to `false`. If using post-hooks, you must update
   to hooks to handle compression or disable compression.
 
+### Added
+
+- The maximum limit for the number of items returned from the /search and
+  /collections/{collection_id}/items endpoints can now be configured with the
+  `ITEMS_MAX_LIMIT` variable. It is recommended that this be set to 100.
+
 ## [4.2.0] - 2025-05-05
 
 ### Added

README.md

@@ -613,6 +613,7 @@ There are some settings that should be reviewed and updated as needeed in the se
 | ENABLE_THUMBNAILS                | Enables support for presigned thumbnails.                                                                                                                                                                                                                                       | none (not enabled)                                                                   |
 | ENABLE_INGEST_ACTION_TRUNCATE    | Enables support for ingest action "truncate".                                                                                                                                                                                                                                   | none (not enabled)                                                                   |
 | ENABLE_RESPONSE_COMPRESSION      | Enables response compression. Set to 'false' to disable.                                                                                                                                                                                                                        | enabled                                                                              |
+| ITEMS_MAX_LIMIT                  | The maximum limit for the number of items returned from the /search and /collections/{collection_id}/items endpoints. It is recommended that this be set to 100. There is an absolute max limit of 10000 for this.                                                              | 10000                                                                                |
 
 Additionally, the credential for OpenSearch must be configured, as decribed in the
 section [Populating and accessing credentials](#populating-and-accessing-credentials).

package-lock.json

@@ -80,6 +80,7 @@
     "@stoplight/spectral-cli": "^6.15.0",
     "@tsconfig/node22": "^22.0.1",
     "@types/aws-lambda": "^8.10.149",
+    "@types/compression": "^1.8.1",
     "@types/cors": "^2.8.17",
     "@types/express": "^4.17.21",
     "@types/http-errors": "^2.0.4",

package.json

@@ -95,12 +95,20 @@ export const extractLimit = function (params) {
 
     if (Number.isNaN(limit) || limit <= 0) {
       throw new ValidationError(
-        'Invalid limit value, must be a number between 1 and 10000 inclusive'
+        'Invalid limit value, must be a positive number'
       )
     }
-    if (limit > 10000) {
-      limit = 10000
+
+    let itemsMaxLimit = Number(process.env['ITEMS_MAX_LIMIT'])
+    if (Number.isNaN(itemsMaxLimit) || itemsMaxLimit <= 0) {
+      itemsMaxLimit = Number.MAX_SAFE_INTEGER
     }
+    limit = Math.min(
+      itemsMaxLimit,
+      limit || Number.MAX_SAFE_INTEGER,
+      10000
+    )
+
     return limit
   }
   return undefined

src/lib/api.js

@@ -4,6 +4,10 @@ import test from 'ava'
 import { extractLimit } from '../../src/lib/api.js'
 import { ValidationError } from '../../src/lib/errors.js'
 
+test.beforeEach(() => {
+  delete process.env['ITEMS_MAX_LIMIT']
+})
+
 test('extractLimit undefined', (t) => {
   t.falsy(extractLimit({}), 'Returns undefined when no limit parameter')
 })
@@ -16,6 +20,21 @@ test('extractLimit when over max limit', (t) => {
   t.is(extractLimit({ limit: '10001' }), 10000)
 })
 
+test('extractLimit when over max limit and ITEMS_MAX_LIMIT set', (t) => {
+  process.env['ITEMS_MAX_LIMIT'] = '100'
+  t.is(extractLimit({ limit: '8374' }), 100)
+})
+
+test('extractLimit when over max limit and ITEMS_MAX_LIMIT set over absolute max limit', (t) => {
+  process.env['ITEMS_MAX_LIMIT'] = '10001'
+  t.is(extractLimit({ limit: '10002' }), 10000)
+})
+
+test('extractLimit when ITEMS_MAX_LIMIT is invalid', (t) => {
+  process.env['ITEMS_MAX_LIMIT'] = 'foo'
+  t.is(extractLimit({ limit: '10002' }), 10000)
+})
+
 test('extractLimit invalid values', (t) => {
   const invalidLimits = ['', '-1', '0', 'a', -1, 0]