metafix

src/html-pipe.js

@@ -131,7 +131,7 @@ export async function htmlPipe(state, req) {
     state.timer?.update('metadata-fetch');
     await Promise.all([
       contentPromise,
-      fetchMappedMetadata(state),
+      fetchMappedMetadata(state, res),
     ]);
 
     if (res.error) {

src/steps/fetch-mapped-metadata.js

@@ -12,6 +12,7 @@
 
 import { PipelineStatusError } from '../PipelineStatusError.js';
 import { Modifiers } from '../utils/modifiers.js';
+import { extractLastModified, updateLastModified } from '../utils/last-modified.js';
 
 /**
  * Loads metadata for a mapped path and puts it into `state.mappedMetadata`. If path is not
@@ -20,9 +21,10 @@ import { Modifiers } from '../utils/modifiers.js';
  *
  * @type PipelineStep
  * @param {PipelineState} state
+ * @param {PipelineResponse} res
  * @returns {Promise<void>}
  */
-export default async function fetchMappedMetadata(state) {
+export default async function fetchMappedMetadata(state, res) {
   state.mappedMetadata = Modifiers.EMPTY;
   if (!state.mapped) {
     return;
@@ -51,6 +53,7 @@ export default async function fetchMappedMetadata(state) {
     state.mappedMetadata = Modifiers.fromModifierSheet(
       data,
     );
+    updateLastModified(state, res, extractLastModified(ret.headers));
     return;
   }
   if (ret.status !== 404) {

test/FileS3Loader.js

@@ -53,15 +53,14 @@ export class FileS3Loader {
     if (!dir) {
       throw Error(`unknown bucketId: ${bucketId}`);
     }
-    // eslint-disable-next-line no-console
-    let fileName = key.split('/').pop();
+    let fileName = key.split('/').slice(2).join('/');
 
     fileName = this.rewrites.reduce((result, rewrite) => rewrite(key) || result, null) || fileName;
     const status = this.statusCodeOverrides[fileName];
     const headers = this.headerOverride[fileName] ?? new Map();
     if (status) {
       // eslint-disable-next-line no-console
-      console.log(`FileS3Loader: loading ${bucketId}/${key} -> ${status}`);
+      console.log(`FileS3Loader: loading ${bucketId}/${fileName} -> ${status}`);
       return {
         status,
         body: '',
@@ -73,7 +72,7 @@ export class FileS3Loader {
     try {
       const body = await readFile(file, 'utf-8');
       // eslint-disable-next-line no-console
-      console.log(`FileS3Loader: loading ${bucketId}/${key} -> 200`);
+      console.log(`FileS3Loader: loading ${bucketId}/${fileName} -> 200`);
       return {
         status: 200,
         body,
@@ -86,7 +85,7 @@ export class FileS3Loader {
       };
     } catch (e) {
       // eslint-disable-next-line no-console
-      console.log(`FileS3Loader: loading ${bucketId}/${key} -> 404 (${e.message})`);
+      console.log(`FileS3Loader: loading ${bucketId}/${fileName} -> 404 (${e.message})`);
       return {
         status: 404,
         body: '',

test/fixtures/content/blog/index.md

@@ -0,0 +1,3 @@
+<!-- this is a test document -->
+# Hello
+

test/fixtures/content/generic-product/metadata.json

@@ -0,0 +1,66 @@
+{
+  ":version": 3,
+  ":type": "multi-sheet",
+  ":names": [
+    "default"
+  ],
+  "default": {
+    "total": 2,
+    "offset": 0,
+    "limit": 2,
+    "data": [
+      {
+        "URL": "/page-*",
+        "Category": "rendering-test"
+      },
+      {
+        "url": "**/page-*-blocks",
+        "Glob Test": "match ** and * combo"
+      },
+      {
+        "Url": "**/marketing/**",
+        "Category": "Marketing"
+      },
+      {
+        "URL": "/page-metadata-json.html",
+        "Image": "/media_cf867e391c0b433ec3d416c979aafa1f8e4aae9c.png",
+        "Keywords": "Baz, Bar, Foo",
+        "og:publisher": "Adobe"
+      },
+      {
+        "URL": "/page-metadata-json",
+        "Image": "/media_cf867e391c0b433ec3d416c979aafa1f8e4aae9c.png",
+        "Keywords": "Baz, Bar, Foo",
+        "og:publisher": "Adobe"
+      },
+      {
+        "URL": "/exact-match.html",
+        "Keywords": "Exactomento",
+        "og:publisher": "Adobe",
+        "Short Title": "E"
+      },
+      {
+        "URL": "/page-metadata-block",
+        "Short Title": "global-meta"
+      },
+      {
+        "URL": "/exact-match",
+        "Keywords": "Exactomento",
+        "og:publisher": "Adobe",
+        "Short Title": "E"
+      },
+      {
+        "URL": "/exact-folder/",
+        "Keywords": "Exactomento Folder",
+        "og:publisher": "Adobe",
+        "Short Title": "E"
+      },
+      {
+        "URL": "/products**",
+        "Keywords": "Exactomento Mapped Folder",
+        "og:publisher": "Adobe",
+        "Short Title": "E"
+      }
+    ]
+  }
+}

test/fixtures/content/one-section/index.md

@@ -0,0 +1,3 @@
+<!-- this is a test document -->
+# Hello
+

test/rendering.test.js

@@ -179,8 +179,7 @@
       // eslint-disable-next-line no-param-reassign
       url = new URL(`https://helix-pages.com/${url}`);
     }
-    const spec = url.pathname.split('/').pop();
-    const expFile = path.resolve(__testdir, 'fixtures', 'content', `${spec}.html`);
+    const expFile = path.resolve(__testdir, 'fixtures', 'content', `${url.pathname.substring(1)}.html`);
     let expHtml = null;
     try {
       expHtml = await readFile(expFile, 'utf-8');
@@ -193,7 +192,7 @@
     }
     const response = await render(url, '', expStatus, partition);
     const actHtml = response.body;
     console.log(actHtml);
     if (expStatus === 200) {
       const $actMain = new JSDOM(actHtml).window.document.querySelector(domSelector);
       const $expMain = new JSDOM(expHtml).window.document.querySelector(domSelector);
@@ -209,7 +208,7 @@
     }
     if (!spec) {
       // eslint-disable-next-line no-param-reassign
-      spec = url.pathname.split('/').pop();
+      spec = url.pathname.substring(1);
     }
     const response = await render(url, '.plain');
     const actHtml = response.body;
@@ -526,8 +525,8 @@
 
     it('renders 404.html if content not found', async () => {
       loader
-        .rewrite('404.html', '404-test.html')
-        .headers('404-test.html', 'x-amz-meta-x-source-last-modified', 'Wed, 12 Oct 2009 17:50:00 GMT');
+        .rewrite('404.html', 'super-test/404-test.html')
+        .headers('super-test/404-test.html', 'x-amz-meta-x-source-last-modified', 'Wed, 12 Oct 2009 17:50:00 GMT');
       const { body, headers } = await testRender('not-found-with-handler', 'html', 404);
       assert.deepStrictEqual(Object.fromEntries(headers.entries()), {
         'content-type': 'text/html; charset=utf-8',
@@ -542,8 +541,8 @@
 
     it('renders 404.html if content not found for .plain.html', async () => {
       loader
-        .rewrite('404.html', '404-test.html')
-        .headers('404-test.html', 'x-amz-meta-x-source-last-modified', 'Wed, 12 Oct 2009 17:50:00 GMT');
+        .rewrite('super-test/404.html', 'super-test/404-test.html')
+        .headers('super-test/404-test.html', 'x-amz-meta-x-source-last-modified', 'Wed, 12 Oct 2009 17:50:00 GMT');
       const { body, headers } = await testRender('not-found-with-handler.plain.html', 'html', 404);
       assert.deepStrictEqual(Object.fromEntries(headers.entries()), {
         'content-type': 'text/html; charset=utf-8',
@@ -557,8 +556,8 @@
 
     it('renders 404.html if content not found for static html', async () => {
       loader
-        .rewrite('404.html', '404-test.html')
-        .headers('404-test.html', 'x-amz-meta-x-source-last-modified', 'Wed, 12 Oct 2009 17:50:00 GMT');
+        .rewrite('super-test/404.html', 'super-test/404-test.html')
+        .headers('super-test/404-test.html', 'x-amz-meta-x-source-last-modified', 'Wed, 12 Oct 2009 17:50:00 GMT');
       const { body, headers } = await testRender('not-found-with-handler.html', 'html', 404);
       assert.deepStrictEqual(Object.fromEntries(headers.entries()), {
         'content-type': 'text/html; charset=utf-8',
@@ -631,7 +630,7 @@
     });
 
     it('renders redirect for static html (code)', async () => {
-      loader.headers('static.html', 'x-amz-meta-redirect-location', '/foo');
+      loader.headers('super-test/static.html', 'x-amz-meta-redirect-location', '/foo');
       const ret = await render(new URL('https://localhost/static.html'), '', 301);
       assert.strictEqual(ret.headers.get('location'), '/foo');
     });
@@ -671,6 +670,7 @@
       assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), {
         'access-control-allow-origin': '*',
         'content-type': 'text/html; charset=utf-8',
+        'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT',
         'x-surrogate-key': 'AkcHu8fRFT7HarTR foo-id_metadata super-test--helix-pages--adobe_head foo-id AkcHu8fRFT7HarTR_metadata z8NGXvKB0X5Fzcnd',
         link: '</scripts/scripts.js>; rel=modulepreload; as=script; crossorigin=use-credentials',
       });
@@ -696,8 +696,8 @@
     it('respect folder mapping: render 404 if mapped missing', async () => {
       loader.status('document1.md', 404);
       loader.status('articles/document1.md', 404);
-      loader.status('default-article.md', 404);
-      loader.rewrite('404.html', '404-test.html');
+      loader.status('special/default-article.md', 404);
+      loader.rewrite('super-test/404.html', 'super-test/404-test.html');
 
       const resp = await render(new URL('https://helix-pipeline.com/articles/document1'), '', 404);
       assert.strictEqual(resp.body, '<html><body>There might be dragons.</body></html>\n');
@@ -717,6 +717,9 @@
     });
 
     it('respect metadata with folder mapping: self and descendents', async () => {
+      loader
+        .headers('generic-product/metadata.json', 'last-modified', 'Thu Nov 07 2024 00:00:00 GMT+0000');
+
       let resp = await render(new URL('https://helix-pipeline.com/products'));
       assert.strictEqual(resp.status, 200);
       assert.match(resp.body, /<meta name="short-title" content="E">/);
@@ -731,14 +734,14 @@
       assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), {
         'access-control-allow-origin': '*',
         'content-type': 'text/html; charset=utf-8',
-        'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT',
+        'last-modified': 'Thu Nov 07 2024 00:00:00 GMT+0000',
         'x-surrogate-key': 'AkcHu8fRFT7HarTR foo-id_metadata super-test--helix-pages--adobe_head foo-id AkcHu8fRFT7HarTR_metadata z8NGXvKB0X5Fzcnd',
         link: '</scripts/scripts.js>; rel=modulepreload; as=script; crossorigin=use-credentials',
       });
     });
 
     it('handles error while loading mapped metadata', async () => {
-      loader.status('metadata.json', 500);
+      loader.status('generic-product/metadata.json', 500);
       await render(new URL('https://helix-pipeline.com/products'), null, 502);
     });
 

test/steps/fetch-mapped-metadata.test.js

@@ -11,7 +11,7 @@
  */
 /* eslint-env mocha */
 import assert from 'assert';
-import { PipelineStatusError } from '../../src/index.js';
+import { PipelineResponse, PipelineStatusError } from '../../src/index.js';
 import { StaticS3Loader } from '../StaticS3Loader.js';
 import fetchMappedMetadata from '../../src/steps/fetch-mapped-metadata.js';
 import { FileS3Loader } from '../FileS3Loader.js';
@@ -30,7 +30,7 @@ describe('Fetch Mapped Metadata', () => {
       s3Loader: new FileS3Loader()
         .rewrite('foo-id/live/mapped/metadata.json', 'metadata-kv.json'),
     };
-    await fetchMappedMetadata(state);
+    await fetchMappedMetadata(state, new PipelineResponse());
     assert.deepEqual(state.mappedMetadata.getModifiers('/new/foo'), {
       description: 'Lorem ipsum dolor sit amet.',
       keywords: 'ACME, CORP, PR',
@@ -53,7 +53,7 @@ describe('Fetch Mapped Metadata', () => {
           body: 'this is no json!',
           headers: new Map(),
         }),
-    });
+    }, new PipelineResponse());
     await assert.rejects(promise, new PipelineStatusError(500, 'failed parsing of /mapped/metadata.json: Unexpected token \'h\', "this is no json!" is not valid JSON'));
   });
 
@@ -74,7 +74,7 @@ describe('Fetch Mapped Metadata', () => {
         }),
     });
     await assert.rejects(promise, new PipelineStatusError(500, 'failed loading of /mapped/metadata.json: data must be an array'));
-  });
+  }, new PipelineResponse());
 
   it('ignores metadata with no data array', async () => {
     const state = {
@@ -94,7 +94,7 @@ describe('Fetch Mapped Metadata', () => {
     };
     await fetchMappedMetadata(state);
     assert.strictEqual(state.mappedMetadata, Modifiers.EMPTY);
-  });
+  }, new PipelineResponse());
 
   it('throws error on generic error', async () => {
     const promise = fetchMappedMetadata({
@@ -113,5 +113,5 @@ describe('Fetch Mapped Metadata', () => {
         }),
     });
     await assert.rejects(promise, new PipelineStatusError(502, 'failed to load /mapped/metadata.json: 500'));
-  });
+  }, new PipelineResponse());
 });