From 0004d128d10717df8f533ba4c58991d0c550c9c3 Mon Sep 17 00:00:00 2001 From: Tobias Bocanegra Date: Fri, 8 Nov 2024 10:45:32 +0100 Subject: [PATCH 1/2] fix: respect last-modified from mapped metadata fixes #737 --- package-lock.json | 4 +- src/html-pipe.js | 2 +- src/steps/fetch-mapped-metadata.js | 5 +- test/FileS3Loader.js | 9 +- test/fixtures/code/main/helix-config.json | 17 +++ .../code/{ => super-test}/404-test.html | 0 .../helix-config-head-with-script.json | 0 .../helix-config-no-head-html.json | 0 .../helix-config-no-head.json | 0 .../{ => super-test}/helix-config.corrupt | 0 .../code/{ => super-test}/helix-config.json | 0 .../{ => super-test}/my-block.selector.html | 0 .../code/{ => super-test/spa}/index.html | 0 .../code/{ => super-test}/static.html | 0 .../content/{ => .helix}/config-all-ld.json | 0 .../content/{ => .helix}/config-all.json | 0 test/fixtures/content/blog/index.md | 3 + .../content/generic-product/metadata.json | 66 +++++++++++ test/fixtures/content/one-section/index.md | 3 + .../content/{ => special}/default-article.md | 0 test/html-pipe.test.js | 32 ++++- test/rendering.test.js | 109 ++++++++++-------- test/sitemap-pipe.test.js | 62 +++++----- test/steps/fetch-mapped-metadata.test.js | 12 +- 24 files changed, 226 insertions(+), 98 deletions(-) create mode 100644 test/fixtures/code/main/helix-config.json rename test/fixtures/code/{ => super-test}/404-test.html (100%) rename test/fixtures/code/{ => super-test}/helix-config-head-with-script.json (100%) rename test/fixtures/code/{ => super-test}/helix-config-no-head-html.json (100%) rename test/fixtures/code/{ => super-test}/helix-config-no-head.json (100%) rename test/fixtures/code/{ => super-test}/helix-config.corrupt (100%) rename test/fixtures/code/{ => super-test}/helix-config.json (100%) rename test/fixtures/code/{ => super-test}/my-block.selector.html (100%) rename test/fixtures/code/{ => super-test/spa}/index.html (100%) rename test/fixtures/code/{ => super-test}/static.html (100%) rename test/fixtures/content/{ => .helix}/config-all-ld.json (100%) rename test/fixtures/content/{ => .helix}/config-all.json (100%) create mode 100644 test/fixtures/content/blog/index.md create mode 100644 test/fixtures/content/generic-product/metadata.json rename test/fixtures/content/{ => special}/default-article.md (100%) diff --git a/package-lock.json b/package-lock.json index 8681d5f8..dd4b0fa8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@adobe/helix-html-pipeline", - "version": "5.13.5", + "version": "5.13.6", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@adobe/helix-html-pipeline", - "version": "5.13.5", + "version": "5.13.6", "license": "Apache-2.0", "dependencies": { "@adobe/helix-markdown-support": "7.1.6", diff --git a/src/html-pipe.js b/src/html-pipe.js index b5f19e10..55c7807d 100644 --- a/src/html-pipe.js +++ b/src/html-pipe.js @@ -148,7 +148,7 @@ export async function htmlPipe(state, req) { await Promise.all([ fetchConfigAll(state, req, res), contentPromise, - fetchMappedMetadata(state), + fetchMappedMetadata(state, res), ]); await requireProject(state, req, res); diff --git a/src/steps/fetch-mapped-metadata.js b/src/steps/fetch-mapped-metadata.js index fd577e94..28b36e45 100644 --- a/src/steps/fetch-mapped-metadata.js +++ b/src/steps/fetch-mapped-metadata.js @@ -12,6 +12,7 @@ import { PipelineStatusError } from '../PipelineStatusError.js'; import { Modifiers } from '../utils/modifiers.js'; +import { extractLastModified, updateLastModified } from '../utils/last-modified.js'; /** * Loads metadata for a mapped path and puts it into `state.mappedMetadata`. If path is not @@ -20,9 +21,10 @@ import { Modifiers } from '../utils/modifiers.js'; * * @type PipelineStep * @param {PipelineState} state + * @param {PipelineResponse} res * @returns {Promise} */ -export default async function fetchMappedMetadata(state) { +export default async function fetchMappedMetadata(state, res) { state.mappedMetadata = Modifiers.EMPTY; if (!state.mapped) { return; @@ -51,6 +53,7 @@ export default async function fetchMappedMetadata(state) { state.mappedMetadata = Modifiers.fromModifierSheet( data, ); + updateLastModified(state, res, extractLastModified(ret.headers)); return; } if (ret.status !== 404) { diff --git a/test/FileS3Loader.js b/test/FileS3Loader.js index dfd4f389..af6d9c15 100644 --- a/test/FileS3Loader.js +++ b/test/FileS3Loader.js @@ -53,15 +53,14 @@ export class FileS3Loader { if (!dir) { throw Error(`unknown bucketId: ${bucketId}`); } - // eslint-disable-next-line no-console - let fileName = key.split('/').pop(); + let fileName = key.split('/').slice(2).join('/'); fileName = this.rewrites.reduce((result, rewrite) => rewrite(key) || result, null) || fileName; const status = this.statusCodeOverrides[fileName]; const headers = this.headerOverride[fileName] ?? new Map(); if (status) { // eslint-disable-next-line no-console - console.log(`FileS3Loader: loading ${bucketId}/${key} -> ${status}`); + console.log(`FileS3Loader: loading ${bucketId}/${fileName} -> ${status}`); return { status, body: '', @@ -73,7 +72,7 @@ export class FileS3Loader { try { const body = await readFile(file, 'utf-8'); // eslint-disable-next-line no-console - console.log(`FileS3Loader: loading ${bucketId}/${key} -> 200`); + console.log(`FileS3Loader: loading ${bucketId}/${fileName} -> 200`); return { status: 200, body, @@ -86,7 +85,7 @@ export class FileS3Loader { }; } catch (e) { // eslint-disable-next-line no-console - console.log(`FileS3Loader: loading ${bucketId}/${key} -> 404 (${e.message})`); + console.log(`FileS3Loader: loading ${bucketId}/${fileName} -> 404 (${e.message})`); return { status: 404, body: '', diff --git a/test/fixtures/code/main/helix-config.json b/test/fixtures/code/main/helix-config.json new file mode 100644 index 00000000..3f72b450 --- /dev/null +++ b/test/fixtures/code/main/helix-config.json @@ -0,0 +1,17 @@ +{ + "head": { + "html": "\n\n\n\n" + }, + "fstab": { + "mountpoints": { + "/": { + "url": "https://adobe.sharepoint.com/sites/cg-helix/Shared%20Documents" + } + } + }, + "content": { + "/": { + "contentBusId": "foo-id" + } + } +} diff --git a/test/fixtures/code/404-test.html b/test/fixtures/code/super-test/404-test.html similarity index 100% rename from test/fixtures/code/404-test.html rename to test/fixtures/code/super-test/404-test.html diff --git a/test/fixtures/code/helix-config-head-with-script.json b/test/fixtures/code/super-test/helix-config-head-with-script.json similarity index 100% rename from test/fixtures/code/helix-config-head-with-script.json rename to test/fixtures/code/super-test/helix-config-head-with-script.json diff --git a/test/fixtures/code/helix-config-no-head-html.json b/test/fixtures/code/super-test/helix-config-no-head-html.json similarity index 100% rename from test/fixtures/code/helix-config-no-head-html.json rename to test/fixtures/code/super-test/helix-config-no-head-html.json diff --git a/test/fixtures/code/helix-config-no-head.json b/test/fixtures/code/super-test/helix-config-no-head.json similarity index 100% rename from test/fixtures/code/helix-config-no-head.json rename to test/fixtures/code/super-test/helix-config-no-head.json diff --git a/test/fixtures/code/helix-config.corrupt b/test/fixtures/code/super-test/helix-config.corrupt similarity index 100% rename from test/fixtures/code/helix-config.corrupt rename to test/fixtures/code/super-test/helix-config.corrupt diff --git a/test/fixtures/code/helix-config.json b/test/fixtures/code/super-test/helix-config.json similarity index 100% rename from test/fixtures/code/helix-config.json rename to test/fixtures/code/super-test/helix-config.json diff --git a/test/fixtures/code/my-block.selector.html b/test/fixtures/code/super-test/my-block.selector.html similarity index 100% rename from test/fixtures/code/my-block.selector.html rename to test/fixtures/code/super-test/my-block.selector.html diff --git a/test/fixtures/code/index.html b/test/fixtures/code/super-test/spa/index.html similarity index 100% rename from test/fixtures/code/index.html rename to test/fixtures/code/super-test/spa/index.html diff --git a/test/fixtures/code/static.html b/test/fixtures/code/super-test/static.html similarity index 100% rename from test/fixtures/code/static.html rename to test/fixtures/code/super-test/static.html diff --git a/test/fixtures/content/config-all-ld.json b/test/fixtures/content/.helix/config-all-ld.json similarity index 100% rename from test/fixtures/content/config-all-ld.json rename to test/fixtures/content/.helix/config-all-ld.json diff --git a/test/fixtures/content/config-all.json b/test/fixtures/content/.helix/config-all.json similarity index 100% rename from test/fixtures/content/config-all.json rename to test/fixtures/content/.helix/config-all.json diff --git a/test/fixtures/content/blog/index.md b/test/fixtures/content/blog/index.md new file mode 100644 index 00000000..def65cb4 --- /dev/null +++ b/test/fixtures/content/blog/index.md @@ -0,0 +1,3 @@ + +# Hello + diff --git a/test/fixtures/content/generic-product/metadata.json b/test/fixtures/content/generic-product/metadata.json new file mode 100644 index 00000000..8a8bdc4e --- /dev/null +++ b/test/fixtures/content/generic-product/metadata.json @@ -0,0 +1,66 @@ +{ + ":version": 3, + ":type": "multi-sheet", + ":names": [ + "default" + ], + "default": { + "total": 2, + "offset": 0, + "limit": 2, + "data": [ + { + "URL": "/page-*", + "Category": "rendering-test" + }, + { + "url": "**/page-*-blocks", + "Glob Test": "match ** and * combo" + }, + { + "Url": "**/marketing/**", + "Category": "Marketing" + }, + { + "URL": "/page-metadata-json.html", + "Image": "/media_cf867e391c0b433ec3d416c979aafa1f8e4aae9c.png", + "Keywords": "Baz, Bar, Foo", + "og:publisher": "Adobe" + }, + { + "URL": "/page-metadata-json", + "Image": "/media_cf867e391c0b433ec3d416c979aafa1f8e4aae9c.png", + "Keywords": "Baz, Bar, Foo", + "og:publisher": "Adobe" + }, + { + "URL": "/exact-match.html", + "Keywords": "Exactomento", + "og:publisher": "Adobe", + "Short Title": "E" + }, + { + "URL": "/page-metadata-block", + "Short Title": "global-meta" + }, + { + "URL": "/exact-match", + "Keywords": "Exactomento", + "og:publisher": "Adobe", + "Short Title": "E" + }, + { + "URL": "/exact-folder/", + "Keywords": "Exactomento Folder", + "og:publisher": "Adobe", + "Short Title": "E" + }, + { + "URL": "/products**", + "Keywords": "Exactomento Mapped Folder", + "og:publisher": "Adobe", + "Short Title": "E" + } + ] + } +} diff --git a/test/fixtures/content/one-section/index.md b/test/fixtures/content/one-section/index.md index e69de29b..def65cb4 100644 --- a/test/fixtures/content/one-section/index.md +++ b/test/fixtures/content/one-section/index.md @@ -0,0 +1,3 @@ + +# Hello + diff --git a/test/fixtures/content/default-article.md b/test/fixtures/content/special/default-article.md similarity index 100% rename from test/fixtures/content/default-article.md rename to test/fixtures/content/special/default-article.md diff --git a/test/html-pipe.test.js b/test/html-pipe.test.js index e77407e2..d1abd07c 100644 --- a/test/html-pipe.test.js +++ b/test/html-pipe.test.js @@ -34,7 +34,7 @@ describe('HTML Pipe Test', () => { const resp = await htmlPipe( new PipelineState({ log: console, - s3Loader: new FileS3Loader().status('config-all.json', 404), + s3Loader: new FileS3Loader().status('.helix/config-all.json', 404), owner: 'adobe', repo: 'helix-pages', ref: 'super-test', @@ -186,6 +186,36 @@ describe('HTML Pipe Test', () => { }); }); + it('serves 404 for missing resource', async () => { + const s3Loader = new FileS3Loader(); + const state = new PipelineState({ + log: console, + s3Loader, + owner: 'adobe', + repo: 'helix-pages', + ref: 'main', + partition: 'live', + path: '/doesnotexist.md', + timer: { + update: () => { }, + }, + }); + const resp = await htmlPipe( + state, + new PipelineRequest(new URL('https://www.hlx.live/')), + ); + assert.strictEqual(resp.status, 404); + assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), { + 'access-control-allow-origin': '*', + 'content-type': 'text/html; charset=utf-8', + 'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT', + 'x-error': 'failed to load /doesnotexist.md from content-bus: 404', + 'x-surrogate-key': 'HtFBsF6g4PvudhiW foo-id main--helix-pages--adobe_404 main--helix-pages--adobe_code', + // this is coming from the config-all/headers + link: '; rel=modulepreload; as=script; crossorigin=use-credentials', + }); + }); + it('renders /', async () => { const s3Loader = new FileS3Loader(); const state = new PipelineState({ diff --git a/test/rendering.test.js b/test/rendering.test.js index 00cf282f..579dbb0d 100644 --- a/test/rendering.test.js +++ b/test/rendering.test.js @@ -56,8 +56,7 @@ describe('Rendering', () => { // eslint-disable-next-line no-param-reassign url = new URL(`https://helix-pages.com/${url}`); } - const spec = url.pathname.split('/').pop(); - const expFile = path.resolve(__testdir, 'fixtures', 'content', `${spec}.html`); + const expFile = path.resolve(__testdir, 'fixtures', 'content', `${url.pathname.substring(1)}.html`); let expHtml = null; try { expHtml = await readFile(expFile, 'utf-8'); @@ -86,7 +85,7 @@ describe('Rendering', () => { } if (!spec) { // eslint-disable-next-line no-param-reassign - spec = url.pathname.split('/').pop(); + spec = url.pathname.substring(1); } const response = await render(url, '.plain'); const actHtml = response.body; @@ -198,82 +197,82 @@ describe('Rendering', () => { }); it('renders multi value meta tags from metadata block in paragraphs', async () => { - loader.status('config-all.json', 404); + loader.status('.helix/config-all.json', 404); await testRender('page-metadata-block-multi-p', 'head'); }); it('renders multi value meta tags from metadata block in unordered lists', async () => { - loader.status('config-all.json', 404); + loader.status('.helix/config-all.json', 404); await testRender('page-metadata-block-multi-ul', 'head'); }); it('renders multi value meta tags from metadata block in ordered lists', async () => { - loader.status('config-all.json', 404); + loader.status('.helix/config-all.json', 404); await testRender('page-metadata-block-multi-ol', 'head'); }); it('renders multi value meta tags from metadata block in links', async () => { - loader.status('config-all.json', 404); + loader.status('.helix/config-all.json', 404); await testRender('page-metadata-block-multi-a', 'head'); }); it('renders canonical from metadata block', async () => { - loader.status('config-all.json', 404); + loader.status('.helix/config-all.json', 404); await testRender('page-metadata-block-canonical', 'head'); }); it('does not no og:url for empty string in document', async () => { - loader.status('config-all.json', 404); + loader.status('.helix/config-all.json', 404); await testRender('page-metadata-block-empty-url', 'head'); }); it('uses correct title and hero image', async () => { - loader.status('config-all.json', 404); + loader.status('.helix/config-all.json', 404); await testRender(new URL('https://super-test--helix-pages--adobe.hlx3.page/marketing/page-metadata-content-blocks'), 'head'); }); it('uses correct image', async () => { - loader.status('config-all.json', 404); + loader.status('.helix/config-all.json', 404); await testRender('image', 'html'); }); it('uses correct image - no alt text', async () => { - loader.status('config-all.json', 404); + loader.status('.helix/config-all.json', 404); await testRender('image-no-alt', 'html'); }); it('uses correct image - with title attribute', async () => { - loader.status('config-all.json', 404); + loader.status('.helix/config-all.json', 404); await testRender('image-with-title', 'html'); }); it('uses correct image - from metadata', async () => { - loader.status('config-all.json', 404); + loader.status('.helix/config-all.json', 404); await testRender('image-from-meta', 'html'); }); it('uses correct image - from metadata with rewrite', async () => { - loader.status('config-all.json', 404); + loader.status('.helix/config-all.json', 404); await testRender('image-from-meta-rewrite', 'html'); }); it('uses correct image - from metadata with rewrite (link)', async () => { - loader.status('config-all.json', 404); + loader.status('.helix/config-all.json', 404); await testRender('image-from-meta-rewrite-link', 'html'); }); it('uses correct description', async () => { - loader.status('config-all.json', 404); + loader.status('.helix/config-all.json', 404); await testRender('description-long', 'head'); }); it('uses correct description from table', async () => { - loader.status('config-all.json', 404); + loader.status('.helix/config-all.json', 404); await testRender('description', 'head'); }); it('uses correct description with blockquote', async () => { - loader.status('config-all.json', 404); + loader.status('.helix/config-all.json', 404); await testRender('description-blockquote', 'head'); }); @@ -282,32 +281,32 @@ describe('Rendering', () => { }); it('sets proper twitter fallbacks', async () => { - loader.status('config-all.json', 404); + loader.status('.helix/config-all.json', 404); await testRender('page-metadata-twitter-fallback', 'head'); }); it('injects json ld', async () => { - loader.status('config-all.json', 404); + loader.status('.helix/config-all.json', 404); await testRender('page-metadata-jsonld', 'head'); }); it('chooses last json-ld if multiple', async () => { - loader.rewrite('config-all.json', 'config-all-ld.json'); + loader.rewrite('.helix/config-all.json', '.helix/config-all-ld.json'); await testRender('page-metadata-jsonld-multi', 'head'); }); it('injects global json ld', async () => { - loader.rewrite('config-all.json', 'config-all-ld.json'); + loader.rewrite('.helix/config-all.json', '.helix/config-all-ld.json'); await testRender('page-metadata-jsonld-global', 'head'); }); it('detects errors in json ld', async () => { - loader.status('config-all.json', 404); + loader.status('.helix/config-all.json', 404); await testRender('page-metadata-jsonld-error', 'head'); }); it('prevents xss in json ld', async () => { - loader.status('config-all.json', 404); + loader.status('.helix/config-all.json', 404); await testRender('page-metadata-jsonld-xss', 'head'); }); }); @@ -345,17 +344,17 @@ describe('Rendering', () => { }); it('renders header correctly if head is missing', async () => { - loader.rewrite('helix-config.json', 'helix-config-no-head.json'); + loader.rewrite('super-test/helix-config.json', 'super-test/helix-config-no-head.json'); await testRender('no-head-html', 'html'); }); it('renders header correctly if head has linefeed', async () => { - loader.rewrite('helix-config.json', 'helix-config-head-with-script.json'); + loader.rewrite('super-test/helix-config.json', 'super-test/helix-config-head-with-script.json'); await testRender('head-with-script', 'html'); }); it('renders header correctly if head.html is missing', async () => { - loader.rewrite('helix-config.json', 'helix-config-no-head-html.json'); + loader.rewrite('super-test/helix-config.json', 'super-test/helix-config-no-head-html.json'); await testRender('no-head-html', 'html'); }); @@ -369,8 +368,8 @@ describe('Rendering', () => { it('renders 404.html if content not found', async () => { loader - .rewrite('404.html', '404-test.html') - .headers('404-test.html', 'x-amz-meta-x-source-last-modified', 'Wed, 12 Oct 2009 17:50:00 GMT'); + .rewrite('404.html', 'super-test/404-test.html') + .headers('super-test/404-test.html', 'x-amz-meta-x-source-last-modified', 'Wed, 12 Oct 2009 17:50:00 GMT'); const { body, headers } = await testRender('not-found-with-handler', 'html', 404); assert.deepStrictEqual(Object.fromEntries(headers.entries()), { 'content-type': 'text/html; charset=utf-8', @@ -385,8 +384,8 @@ describe('Rendering', () => { it('renders 404.html if content not found for .plain.html', async () => { loader - .rewrite('404.html', '404-test.html') - .headers('404-test.html', 'x-amz-meta-x-source-last-modified', 'Wed, 12 Oct 2009 17:50:00 GMT'); + .rewrite('super-test/404.html', 'super-test/404-test.html') + .headers('super-test/404-test.html', 'x-amz-meta-x-source-last-modified', 'Wed, 12 Oct 2009 17:50:00 GMT'); const { body, headers } = await testRender('not-found-with-handler.plain.html', 'html', 404); assert.deepStrictEqual(Object.fromEntries(headers.entries()), { 'content-type': 'text/html; charset=utf-8', @@ -400,8 +399,8 @@ describe('Rendering', () => { it('renders 404.html if content not found for static html', async () => { loader - .rewrite('404.html', '404-test.html') - .headers('404-test.html', 'x-amz-meta-x-source-last-modified', 'Wed, 12 Oct 2009 17:50:00 GMT'); + .rewrite('super-test/404.html', 'super-test/404-test.html') + .headers('super-test/404-test.html', 'x-amz-meta-x-source-last-modified', 'Wed, 12 Oct 2009 17:50:00 GMT'); const { body, headers } = await testRender('not-found-with-handler.html', 'html', 404); assert.deepStrictEqual(Object.fromEntries(headers.entries()), { 'content-type': 'text/html; charset=utf-8', @@ -415,7 +414,7 @@ describe('Rendering', () => { }); it('renders 404 if helix-config not found', async () => { - loader.status('helix-config.json', 404); + loader.status('super-test/helix-config.json', 404); await testRender('no-head-html', 'html', 404); }); @@ -447,7 +446,7 @@ describe('Rendering', () => { }); it('renders 400 for invalid helix-config', async () => { - loader.rewrite('helix-config.json', 'helix-config.corrupt'); + loader.rewrite('super-test/helix-config.json', 'super-test/helix-config.corrupt'); await testRender('no-head-html', 'html', 400); }); @@ -477,7 +476,7 @@ describe('Rendering', () => { }); it('renders redirect for static html (code)', async () => { - loader.headers('static.html', 'x-amz-meta-redirect-location', '/foo'); + loader.headers('super-test/static.html', 'x-amz-meta-redirect-location', '/foo'); const ret = await render(new URL('https://localhost/static.html'), '', 301); assert.strictEqual(ret.headers.get('location'), '/foo'); }); @@ -532,8 +531,8 @@ describe('Rendering', () => { it('respect folder mapping: render 404 if mapped missing', async () => { loader.status('document1.md', 404); loader.status('articles/document1.md', 404); - loader.status('default-article.md', 404); - loader.rewrite('404.html', '404-test.html'); + loader.status('special/default-article.md', 404); + loader.rewrite('super-test/404.html', 'super-test/404-test.html'); const resp = await render(new URL('https://helix-pipeline.com/articles/document1'), '', 404); assert.strictEqual(resp.body, 'There might be dragons.\n'); @@ -553,7 +552,10 @@ describe('Rendering', () => { }); it('respect metadata with folder mapping: self and descendents', async () => { - loader.status('config-all.json', 404); + loader.status('.helix/config-all.json', 404); + loader + .headers('generic-product/metadata.json', 'last-modified', 'Thu Nov 07 2024 00:00:00 GMT+0000'); + let resp = await render(new URL('https://helix-pipeline.com/products')); assert.strictEqual(resp.status, 200); assert.match(resp.body, //); @@ -567,15 +569,20 @@ describe('Rendering', () => { assert.match(resp.body, //); assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), { 'content-type': 'text/html; charset=utf-8', - 'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT', + 'last-modified': 'Thu Nov 07 2024 00:00:00 GMT+0000', 'x-surrogate-key': 'AkcHu8fRFT7HarTR foo-id_metadata super-test--helix-pages--adobe_head foo-id AkcHu8fRFT7HarTR_metadata z8NGXvKB0X5Fzcnd', }); }); - it('uses last modified from helix-config', async () => { - loader.status('config-all.json', 404); + it('handles error while loading mapped metadata', async () => { + loader.status('generic-product/metadata.json', 500); + await render(new URL('https://helix-pipeline.com/products'), null, 502); + }); + + it('uses last modified from config', async () => { + loader.status('.helix/config-all.json', 404); loader - .headers('helix-config.json', 'x-amz-meta-x-source-last-modified', 'Wed, 12 Jan 2022 11:33:01 GMT') + .headers('super-test/helix-config.json', 'x-amz-meta-x-source-last-modified', 'Wed, 12 Jan 2022 11:33:01 GMT') .headers('index.md', 'x-amz-meta-x-source-last-modified', 'Wed, 12 Jan 2022 10:50:00 GMT') .headers('metadata.json', 'x-amz-meta-x-source-last-modified', 'Wed, 12 Jan 2022 09:50:00 GMT'); const { status, body, headers } = await render(new URL('https://helix-pipeline.com/blog/')); @@ -588,10 +595,10 @@ describe('Rendering', () => { }); }); - it('uses last modified from metadata.json', async () => { - loader.status('config-all.json', 404); + it.skip('uses last modified from metadata.json', async () => { + loader.status('.helix/config-all.json', 404); loader - .headers('helix-config.json', 'x-amz-meta-x-source-last-modified', 'Wed, 12 Oct 2009 11:50:00 GMT') + .headers('super-test/helix-config.json', 'x-amz-meta-x-source-last-modified', 'Wed, 12 Oct 2009 11:50:00 GMT') .headers('index.md', 'x-amz-meta-x-source-last-modified', 'Wed, 12 Oct 2022 12:50:00 GMT') .headers('metadata.json', 'x-amz-meta-x-source-last-modified', 'Wed, 12 Oct 2022 09:33:01 GMT'); const { status, body, headers } = await render(new URL('https://helix-pipeline.com/blog/')); @@ -605,17 +612,17 @@ describe('Rendering', () => { }); it('ignores last modified from metadata.json for plain', async () => { - loader.status('config-all.json', 404); + loader.status('.helix/config-all.json', 404); loader - .headers('helix-config.json', 'x-amz-meta-x-source-last-modified', 'Wed, 12 Oct 2009 11:50:00 GMT') + .headers('super-test/helix-config.json', 'x-amz-meta-x-source-last-modified', 'Wed, 12 Oct 2009 11:50:00 GMT') .headers('one-section.md', 'x-amz-meta-x-source-last-modified', 'Wed, 12 Oct 2022 12:50:00 GMT') .headers('metadata.json', 'x-amz-meta-x-source-last-modified', 'Wed, 12 Oct 2022 15:33:01 GMT'); - const { status, body, headers } = await render(new URL('https://helix-pipeline.com/blog/one-section'), '.plain'); + const { status, body, headers } = await render(new URL('https://helix-pipeline.com/one-section'), '.plain'); assert.strictEqual(status, 200); assert.match(body, /
\s*

Hello<\/h1>\s*

This is the first section.<\/p>\s*<\/div>/); assert.deepStrictEqual(Object.fromEntries(headers.entries()), { 'content-type': 'text/html; charset=utf-8', - 'x-surrogate-key': 'Nep3VelSa1voMXR- foo-id_metadata super-test--helix-pages--adobe_head foo-id', + 'x-surrogate-key': 'oHjg_WDu20CBS4rD foo-id_metadata super-test--helix-pages--adobe_head foo-id', 'last-modified': 'Wed, 12 Oct 2022 12:50:00 GMT', }); }); diff --git a/test/sitemap-pipe.test.js b/test/sitemap-pipe.test.js index 19e469c8..74ea18e4 100644 --- a/test/sitemap-pipe.test.js +++ b/test/sitemap-pipe.test.js @@ -38,7 +38,7 @@ describe('Sitemap Pipe Test', () => { s3Loader: new FileS3Loader().status('sitemap.xml', 500), owner: 'owner', repo: 'repo', - ref: 'ref', + ref: 'super-test', partition: 'live', path: '/sitemap.xml', }), @@ -61,7 +61,7 @@ describe('Sitemap Pipe Test', () => { .status('sitemap.json', 404), owner: 'owner', repo: 'repo', - ref: 'ref', + ref: 'super-test', partition: 'live', path: '/sitemap.xml', }), @@ -73,7 +73,7 @@ describe('Sitemap Pipe Test', () => { 'content-type': 'text/plain; charset=utf-8', 'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT', 'x-error': 'failed to load /sitemap.xml from content-bus: 404', - 'x-surrogate-key': 'lkDPpF5moMrrCXQM foo-id_metadata ref--repo--owner_head foo-id', + 'x-surrogate-key': 'lkDPpF5moMrrCXQM foo-id_metadata super-test--repo--owner_head foo-id', }); }); @@ -86,7 +86,7 @@ describe('Sitemap Pipe Test', () => { .rewrite('sitemap.json', 'sitemap-corrupt.json'), owner: 'owner', repo: 'repo', - ref: 'ref', + ref: 'super-test', partition: 'live', path: '/sitemap.xml', }), @@ -98,7 +98,7 @@ describe('Sitemap Pipe Test', () => { 'content-type': 'text/plain; charset=utf-8', 'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT', 'x-error': 'Failed to parse /sitemap.json: Unexpected token \'h\', "this is not JSON" is not valid JSON', - 'x-surrogate-key': 'lkDPpF5moMrrCXQM foo-id_metadata ref--repo--owner_head foo-id', + 'x-surrogate-key': 'lkDPpF5moMrrCXQM foo-id_metadata super-test--repo--owner_head foo-id', }); }); @@ -111,7 +111,7 @@ describe('Sitemap Pipe Test', () => { .rewrite('sitemap.json', 'sitemap-bad-data.json'), owner: 'owner', repo: 'repo', - ref: 'ref', + ref: 'super-test', partition: 'live', path: '/sitemap.xml', }), @@ -123,7 +123,7 @@ describe('Sitemap Pipe Test', () => { 'content-type': 'text/plain; charset=utf-8', 'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT', 'x-error': 'Expected \'data\' array not found in /sitemap.json', - 'x-surrogate-key': 'lkDPpF5moMrrCXQM foo-id_metadata ref--repo--owner_head foo-id', + 'x-surrogate-key': 'lkDPpF5moMrrCXQM foo-id_metadata super-test--repo--owner_head foo-id', }); }); @@ -134,7 +134,7 @@ describe('Sitemap Pipe Test', () => { s3Loader: new FileS3Loader().status('sitemap.xml', 404), owner: 'owner', repo: 'repo', - ref: 'ref', + ref: 'super-test', partition: 'preview', path: '/sitemap.xml', }), @@ -145,7 +145,7 @@ describe('Sitemap Pipe Test', () => { 'access-control-allow-origin': '*', 'content-type': 'application/xml; charset=utf-8', 'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT', - 'x-surrogate-key': 'lkDPpF5moMrrCXQM foo-id_metadata ref--repo--owner_head foo-id', + 'x-surrogate-key': 'lkDPpF5moMrrCXQM foo-id_metadata super-test--repo--owner_head foo-id', }); }); @@ -156,7 +156,7 @@ describe('Sitemap Pipe Test', () => { s3Loader: new FileS3Loader().status('sitemap.xml', 404), owner: 'owner', repo: 'repo', - ref: 'ref', + ref: 'super-test', partition: 'preview', path: '/sitemap.xml', }), @@ -167,16 +167,16 @@ describe('Sitemap Pipe Test', () => { 'access-control-allow-origin': '*', 'content-type': 'application/xml; charset=utf-8', 'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT', - 'x-surrogate-key': 'lkDPpF5moMrrCXQM foo-id_metadata ref--repo--owner_head foo-id', + 'x-surrogate-key': 'lkDPpF5moMrrCXQM foo-id_metadata super-test--repo--owner_head foo-id', }); assert.strictEqual(resp.body, ` - https://ref--repo--owner.my.page/ + https://super-test--repo--owner.my.page/ 2023-11-30 - https://ref--repo--owner.my.page/test + https://super-test--repo--owner.my.page/test 2023-12-21 `); @@ -189,7 +189,7 @@ describe('Sitemap Pipe Test', () => { s3Loader: new FileS3Loader().status('sitemap.xml', 404), owner: 'owner', repo: 'repo', - ref: 'ref', + ref: 'super-test', partition: 'preview', path: '/sitemap.xml', }), @@ -200,16 +200,16 @@ describe('Sitemap Pipe Test', () => { 'access-control-allow-origin': '*', 'content-type': 'application/xml; charset=utf-8', 'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT', - 'x-surrogate-key': 'lkDPpF5moMrrCXQM foo-id_metadata ref--repo--owner_head foo-id', + 'x-surrogate-key': 'lkDPpF5moMrrCXQM foo-id_metadata super-test--repo--owner_head foo-id', }); assert.strictEqual(resp.body, ` - https://ref--repo--owner.my.page/ + https://super-test--repo--owner.my.page/ 2023-11-30 - https://ref--repo--owner.my.page/test + https://super-test--repo--owner.my.page/test 2023-12-21 `); @@ -221,10 +221,10 @@ describe('Sitemap Pipe Test', () => { log: console, s3Loader: new FileS3Loader() .status('sitemap.xml', 404) - .status('config-all.json', 404), + .status('.helix/config-all.json', 404), owner: 'owner', repo: 'repo', - ref: 'ref', + ref: 'super-test', partition: 'preview', path: '/sitemap.xml', }), @@ -234,16 +234,16 @@ describe('Sitemap Pipe Test', () => { assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), { 'content-type': 'application/xml; charset=utf-8', 'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT', - 'x-surrogate-key': 'lkDPpF5moMrrCXQM foo-id_metadata ref--repo--owner_head foo-id', + 'x-surrogate-key': 'lkDPpF5moMrrCXQM foo-id_metadata super-test--repo--owner_head foo-id', }); assert.strictEqual(resp.body, ` - https://ref--repo--owner.hlx.page/ + https://super-test--repo--owner.hlx.page/ 2023-11-30 - https://ref--repo--owner.hlx.page/test + https://super-test--repo--owner.hlx.page/test 2023-12-21 `); @@ -256,7 +256,7 @@ describe('Sitemap Pipe Test', () => { s3Loader: new FileS3Loader().status('sitemap.xml', 404), owner: 'owner', repo: 'repo', - ref: 'ref', + ref: 'super-test', partition: 'live', path: '/sitemap.xml', }), @@ -267,7 +267,7 @@ describe('Sitemap Pipe Test', () => { 'access-control-allow-origin': '*', 'content-type': 'application/xml; charset=utf-8', 'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT', - 'x-surrogate-key': 'lkDPpF5moMrrCXQM foo-id_metadata ref--repo--owner_head foo-id', + 'x-surrogate-key': 'lkDPpF5moMrrCXQM foo-id_metadata super-test--repo--owner_head foo-id', }); assert.strictEqual(resp.body, ` @@ -288,10 +288,10 @@ describe('Sitemap Pipe Test', () => { log: console, s3Loader: new FileS3Loader() .status('sitemap.xml', 404) - .status('config-all.json', 404), + .status('.helix/config-all.json', 404), owner: 'owner', repo: 'repo', - ref: 'ref', + ref: 'super-test', partition: 'live', path: '/sitemap.xml', }), @@ -301,16 +301,16 @@ describe('Sitemap Pipe Test', () => { assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), { 'content-type': 'application/xml; charset=utf-8', 'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT', - 'x-surrogate-key': 'lkDPpF5moMrrCXQM foo-id_metadata ref--repo--owner_head foo-id', + 'x-surrogate-key': 'lkDPpF5moMrrCXQM foo-id_metadata super-test--repo--owner_head foo-id', }); assert.strictEqual(resp.body, ` - https://ref--repo--owner.hlx.live/ + https://super-test--repo--owner.hlx.live/ 2023-11-30 - https://ref--repo--owner.hlx.live/test + https://super-test--repo--owner.hlx.live/test 2023-12-21 `); @@ -384,7 +384,7 @@ describe('Sitemap Pipe Test', () => { s3Loader, owner: 'owner', repo: 'repo', - ref: 'ref', + ref: 'super-test', partition: 'live', path: '/sitemap.xml', timer: { @@ -401,7 +401,7 @@ describe('Sitemap Pipe Test', () => { 'access-control-allow-origin': '*', 'content-type': 'application/xml; charset=utf-8', 'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT', - 'x-surrogate-key': 'lkDPpF5moMrrCXQM foo-id_metadata ref--repo--owner_head foo-id', + 'x-surrogate-key': 'lkDPpF5moMrrCXQM foo-id_metadata super-test--repo--owner_head foo-id', }); }); }); diff --git a/test/steps/fetch-mapped-metadata.test.js b/test/steps/fetch-mapped-metadata.test.js index 9cd28320..5fb28279 100644 --- a/test/steps/fetch-mapped-metadata.test.js +++ b/test/steps/fetch-mapped-metadata.test.js @@ -11,7 +11,7 @@ */ /* eslint-env mocha */ import assert from 'assert'; -import { PipelineStatusError } from '../../src/index.js'; +import { PipelineResponse, PipelineStatusError } from '../../src/index.js'; import { StaticS3Loader } from '../StaticS3Loader.js'; import fetchMappedMetadata from '../../src/steps/fetch-mapped-metadata.js'; import { FileS3Loader } from '../FileS3Loader.js'; @@ -30,7 +30,7 @@ describe('Fetch Mapped Metadata', () => { s3Loader: new FileS3Loader() .rewrite('foo-id/live/mapped/metadata.json', 'metadata-kv.json'), }; - await fetchMappedMetadata(state); + await fetchMappedMetadata(state, new PipelineResponse()); assert.deepEqual(state.mappedMetadata.getModifiers('/new/foo'), { description: 'Lorem ipsum dolor sit amet.', keywords: 'ACME, CORP, PR', @@ -53,7 +53,7 @@ describe('Fetch Mapped Metadata', () => { body: 'this is no json!', headers: new Map(), }), - }); + }, new PipelineResponse()); await assert.rejects(promise, new PipelineStatusError(500, 'failed parsing of /mapped/metadata.json: Unexpected token \'h\', "this is no json!" is not valid JSON')); }); @@ -74,7 +74,7 @@ describe('Fetch Mapped Metadata', () => { }), }); await assert.rejects(promise, new PipelineStatusError(500, 'failed loading of /mapped/metadata.json: data must be an array')); - }); + }, new PipelineResponse()); it('ignores metadata with no data array', async () => { const state = { @@ -94,7 +94,7 @@ describe('Fetch Mapped Metadata', () => { }; await fetchMappedMetadata(state); assert.strictEqual(state.mappedMetadata, Modifiers.EMPTY); - }); + }, new PipelineResponse()); it('throws error on generic error', async () => { const promise = fetchMappedMetadata({ @@ -113,5 +113,5 @@ describe('Fetch Mapped Metadata', () => { }), }); await assert.rejects(promise, new PipelineStatusError(502, 'failed to load /mapped/metadata.json: 500')); - }); + }, new PipelineResponse()); }); From 3902c0f904263b8855f104365c9d9b4af6cbbde1 Mon Sep 17 00:00:00 2001 From: Tobias Bocanegra Date: Mon, 18 Nov 2024 16:58:18 +0100 Subject: [PATCH 2/2] chore: refactor tests --- src/html-pipe.js | 2 +- src/steps/fetch-mapped-metadata.js | 5 +---- test/rendering.test.js | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/html-pipe.js b/src/html-pipe.js index 55c7807d..b5f19e10 100644 --- a/src/html-pipe.js +++ b/src/html-pipe.js @@ -148,7 +148,7 @@ export async function htmlPipe(state, req) { await Promise.all([ fetchConfigAll(state, req, res), contentPromise, - fetchMappedMetadata(state, res), + fetchMappedMetadata(state), ]); await requireProject(state, req, res); diff --git a/src/steps/fetch-mapped-metadata.js b/src/steps/fetch-mapped-metadata.js index 28b36e45..fd577e94 100644 --- a/src/steps/fetch-mapped-metadata.js +++ b/src/steps/fetch-mapped-metadata.js @@ -12,7 +12,6 @@ import { PipelineStatusError } from '../PipelineStatusError.js'; import { Modifiers } from '../utils/modifiers.js'; -import { extractLastModified, updateLastModified } from '../utils/last-modified.js'; /** * Loads metadata for a mapped path and puts it into `state.mappedMetadata`. If path is not @@ -21,10 +20,9 @@ import { extractLastModified, updateLastModified } from '../utils/last-modified. * * @type PipelineStep * @param {PipelineState} state - * @param {PipelineResponse} res * @returns {Promise} */ -export default async function fetchMappedMetadata(state, res) { +export default async function fetchMappedMetadata(state) { state.mappedMetadata = Modifiers.EMPTY; if (!state.mapped) { return; @@ -53,7 +51,6 @@ export default async function fetchMappedMetadata(state, res) { state.mappedMetadata = Modifiers.fromModifierSheet( data, ); - updateLastModified(state, res, extractLastModified(ret.headers)); return; } if (ret.status !== 404) { diff --git a/test/rendering.test.js b/test/rendering.test.js index 579dbb0d..b27d0ee4 100644 --- a/test/rendering.test.js +++ b/test/rendering.test.js @@ -569,7 +569,7 @@ describe('Rendering', () => { assert.match(resp.body, //); assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), { 'content-type': 'text/html; charset=utf-8', - 'last-modified': 'Thu Nov 07 2024 00:00:00 GMT+0000', + 'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT', 'x-surrogate-key': 'AkcHu8fRFT7HarTR foo-id_metadata super-test--helix-pages--adobe_head foo-id AkcHu8fRFT7HarTR_metadata z8NGXvKB0X5Fzcnd', }); });