feat: read metadata directly from source if required

src/html-pipe.js

@@ -36,6 +36,7 @@ import { PipelineResponse } from './PipelineResponse.js';
 import { validatePathInfo } from './utils/path.js';
 import fetchMappedMetadata from './steps/fetch-mapped-metadata.js';
 import { applyMetaLastModified, setLastModified } from './utils/last-modified.js';
+import fetchSourcedMetadata from './steps/fetch-sourced-metadata.js';
 
 /**
  * Fetches the content and if not found, fetches the 404.html
@@ -132,6 +133,7 @@ export async function htmlPipe(state, req) {
     state.timer?.update('metadata-fetch');
     await Promise.all([
       contentPromise,
+      fetchSourcedMetadata(state, res),
       fetchMappedMetadata(state, res),
     ]);
 

src/steps/fetch-sourced-metadata.js

@@ -0,0 +1,71 @@
+/*
+ * Copyright 2021 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import { PipelineStatusError } from '../PipelineStatusError.js';
+import { Modifiers } from '../utils/modifiers.js';
+import { extractLastModified, recordLastModified } from '../utils/last-modified.js';
+
+/**
+ * Loads metadata from the metadata sources if required. this happens when the amount of metadata
+ * was too large to include in the config response. the metadata is loaded if
+ * `state.metadata` is empty and `config.metadata.source` is not.
+ *
+ * @type PipelineStep
+ * @param {PipelineState} state
+ * @param {PipelineResponse} res
+ * @returns {Promise<void>}
+ */
+// eslint-disable-next-line no-unused-vars
+export default async function fetchSourcedMetadata(state, res) {
+  if (!state.metadata.isEmpty()) {
+    return;
+  }
+  const sources = state.config.metadata?.source || [];
+  if (sources.length === 0) {
+    return;
+  }
+
+  const { contentBusId, partition } = state;
+  const metadatas = {};
+  await Promise.all(sources.map(async (src) => {
+    metadatas[src] = [];
+    const key = `${contentBusId}/${partition}/${src}`;
+    const ret = await state.s3Loader.getObject('helix-content-bus', key);
+    if (ret.status === 200) {
+      let json;
+      try {
+        json = JSON.parse(ret.body);
+      } catch (e) {
+        throw new PipelineStatusError(500, `failed parsing of ${key}: ${e.message}`);
+      }
+      const { data } = json.default ?? json;
+      if (!data) {
+        state.log.info(`default sheet missing in ${key}`);
+        return;
+      }
+
+      if (!Array.isArray(data)) {
+        throw new PipelineStatusError(500, `failed loading of ${key}: data must be an array`);
+      }
+      metadatas[src] = data;
+      recordLastModified(state, res, 'content', extractLastModified(ret.headers));
+    } else if (ret.status !== 404) {
+      throw new PipelineStatusError(502, `failed to load ${key}: ${ret.status}`);
+    }
+  }));
+  // aggregate the metadata in the same order as specified
+  const metadata = [];
+  for (const src of sources) {
+    metadata.push(...metadatas[src]);
+  }
+  state.metadata = Modifiers.fromModifierSheet(metadata);
+}

src/utils/modifiers.js

@@ -124,6 +124,14 @@ export class Modifiers {
     return new Modifiers(res);
   }
 
+  /**
+   * Returns true if there are no modifiers.
+   * @returns {boolean}
+   */
+  isEmpty() {
+    return this.modifiers.length === 0;
+  }
+
   constructor(config) {
     this.modifiers = Object.entries(config).map(([url, mods]) => {
       const pat = url.indexOf('*') >= 0 ? globToRegExp(url) : url;

test/FileS3Loader.js

@@ -25,6 +25,7 @@ export class FileS3Loader {
       statusCodeOverrides: {},
       rewrites: [],
       headerOverride: {},
+      contentOverrides: {},
     });
   }
 
@@ -38,6 +39,11 @@ export class FileS3Loader {
     return this;
   }
 
+  override(fileName, data) {
+    this.contentOverrides[fileName] = data;
+    return this;
+  }
+
   headers(fileName, name, value) {
     let headers = this.headerOverride[fileName];
     if (!headers) {
@@ -57,20 +63,21 @@ export class FileS3Loader {
 
     fileName = this.rewrites.reduce((result, rewrite) => rewrite(key) || result, null) || fileName;
     const status = this.statusCodeOverrides[fileName];
+    let body = this.contentOverrides[fileName] || '';
     const headers = this.headerOverride[fileName] ?? new Map();
-    if (status) {
+    if (status || body) {
       // eslint-disable-next-line no-console
-      console.log(`FileS3Loader: loading ${bucketId}/${fileName} -> ${status}`);
+      console.log(`FileS3Loader: loading ${bucketId}/${fileName} -> ${status || 200}`);
       return {
-        status,
-        body: '',
+        status: status || 200,
+        body,
         headers,
       };
     }
 
     const file = path.resolve(dir, fileName);
     try {
-      const body = await readFile(file, 'utf-8');
+      body = await readFile(file, 'utf-8');
       // eslint-disable-next-line no-console
       console.log(`FileS3Loader: loading ${bucketId}/${fileName} -> 200`);
       return {

test/html-pipe.test.js

@@ -198,6 +198,181 @@ describe('HTML Pipe Test', () => {
     });
   });
 
+  it('loads sourced metadata', async () => {
+    const s3Loader = new FileS3Loader();
+    s3Loader.override('metadata.json', JSON.stringify(
+      {
+        data: [
+          { URL: '/**', key: 'category', value: 'news' },
+          { URL: '/**', key: 'template', value: 'page' },
+        ],
+      },
+    ));
+    s3Loader.override('metadata-seo.json', JSON.stringify(
+      {
+        data: [
+          { URL: '/**', key: 'template', value: 'blog' },
+        ],
+      },
+    ));
+    const state = DEFAULT_STATE({
+      ...DEFAULT_CONFIG,
+      metadata: {
+        source: [
+          'metadata.json',
+          'metadata-seo.json',
+          'metadata-missing.json',
+        ],
+      },
+    }, {
+      log: console,
+      s3Loader,
+      ref: 'super-test',
+      partition: 'live',
+      path: '/articles',
+      timer: {
+        update: () => { },
+      },
+    });
+    const resp = await htmlPipe(
+      state,
+      new PipelineRequest(new URL('https://www.hlx.live/')),
+    );
+    assert.strictEqual(resp.status, 200);
+    assert.ok(resp.body.includes('<meta name="category" content="news">'));
+    assert.ok(resp.body.includes('<meta name="template" content="blog">'));
+    assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), {
+      'content-type': 'text/html; charset=utf-8',
+      'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT',
+      'x-surrogate-key': 'iQzO-EvK0WKNO_o0 foo-id_metadata super-test--helix-pages--adobe_head foo-id',
+    });
+  });
+
+  it('rejects invalid sourced metadata (json error)', async () => {
+    const s3Loader = new FileS3Loader();
+    s3Loader.override('metadata.json', 'kaputt');
+    const state = DEFAULT_STATE({
+      ...DEFAULT_CONFIG,
+      metadata: {
+        source: [
+          'metadata.json',
+        ],
+      },
+    }, {
+      log: console,
+      s3Loader,
+      ref: 'super-test',
+      partition: 'live',
+      path: '/articles',
+      timer: {
+        update: () => { },
+      },
+    });
+    const resp = await htmlPipe(
+      state,
+      new PipelineRequest(new URL('https://www.hlx.live/')),
+    );
+    assert.strictEqual(resp.status, 500);
+    assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), {
+      'content-type': 'text/html; charset=utf-8',
+      'x-error': 'failed parsing of foo-id/live/metadata.json: Unexpected token \'k\', "kaputt" is not valid JSON',
+    });
+  });
+
+  it('rejects invalid sourced metadata (invalid sheet)', async () => {
+    const s3Loader = new FileS3Loader();
+    s3Loader.override('metadata.json', '{ "data": "foo" }');
+    const state = DEFAULT_STATE({
+      ...DEFAULT_CONFIG,
+      metadata: {
+        source: [
+          'metadata.json',
+        ],
+      },
+    }, {
+      log: console,
+      s3Loader,
+      ref: 'super-test',
+      partition: 'live',
+      path: '/articles',
+      timer: {
+        update: () => { },
+      },
+    });
+    const resp = await htmlPipe(
+      state,
+      new PipelineRequest(new URL('https://www.hlx.live/')),
+    );
+    assert.strictEqual(resp.status, 500);
+    assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), {
+      'content-type': 'text/html; charset=utf-8',
+      'x-error': 'failed loading of foo-id/live/metadata.json: data must be an array',
+    });
+  });
+
+  it('ignores invalid sourced metadata (missing sheet)', async () => {
+    const s3Loader = new FileS3Loader();
+    s3Loader.override('metadata.json', '{}');
+    const state = DEFAULT_STATE({
+      ...DEFAULT_CONFIG,
+      metadata: {
+        source: [
+          'metadata.json',
+        ],
+      },
+    }, {
+      log: console,
+      s3Loader,
+      ref: 'super-test',
+      partition: 'live',
+      path: '/articles',
+      timer: {
+        update: () => { },
+      },
+    });
+    const resp = await htmlPipe(
+      state,
+      new PipelineRequest(new URL('https://www.hlx.live/')),
+    );
+    assert.strictEqual(resp.status, 200);
+    assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), {
+      'content-type': 'text/html; charset=utf-8',
+      'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT',
+      'x-surrogate-key': 'iQzO-EvK0WKNO_o0 foo-id_metadata super-test--helix-pages--adobe_head foo-id',
+    });
+  });
+
+  it('rejects invalid sourced metadata (status code)', async () => {
+    const s3Loader = new FileS3Loader();
+    s3Loader.status('metadata.json', 401);
+    const state = DEFAULT_STATE({
+      ...DEFAULT_CONFIG,
+      metadata: {
+        source: [
+          'metadata.json',
+        ],
+      },
+    }, {
+      log: console,
+      s3Loader,
+      ref: 'super-test',
+      partition: 'live',
+      path: '/articles',
+      timer: {
+        update: () => { },
+      },
+    });
+    const resp = await htmlPipe(
+      state,
+      new PipelineRequest(new URL('https://www.hlx.live/')),
+    );
+    assert.strictEqual(resp.status, 502);
+    assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), {
+      'content-type': 'text/html; charset=utf-8',
+      'x-error': 'failed to load foo-id/live/metadata.json: 401',
+    });
+  });
+
   it('renders static html with selector my-block.selector.html', async () => {
     const s3Loader = new FileS3Loader();
     const state = DEFAULT_STATE(DEFAULT_CONFIG, {

test/modifiers.test.js

@@ -145,4 +145,13 @@ describe('Metadata', () => {
     const actual = Modifiers.fromModifierSheet(data).getModifiers('/nope');
     assert.deepEqual(actual, {});
   });
+
+  it('isEmpty returns true, if empty', async () => {
+    assert.strictEqual(new Modifiers({}).isEmpty(), true);
+  });
+
+  it('isEmpty returns false, if not empty', async () => {
+    const { default: { data } } = await readTestJSON('metadata.json');
+    assert.strictEqual(Modifiers.fromModifierSheet(data).isEmpty(), false);
+  });
 });