From e821162b9afc649fbb53ef550d51fd61b4ccc346 Mon Sep 17 00:00:00 2001 From: Stefan Guggisberg Date: Mon, 20 May 2024 13:03:15 +0200 Subject: [PATCH 1/3] feat: add contentbusid surrogate key for pipeline responses depending on content resources --- src/json-pipe.js | 11 +++++++++-- src/steps/fetch-content.js | 7 ++++++- src/steps/set-x-surrogate-key-header.js | 1 + test/html-pipe.test.js | 10 +++++----- test/json-pipe.test.js | 22 +++++++++++----------- test/rendering.test.js | 18 +++++++++--------- test/sitemap-pipe.test.js | 18 +++++++++--------- 7 files changed, 50 insertions(+), 37 deletions(-) diff --git a/src/json-pipe.js b/src/json-pipe.js index e39f9b30..eb794111 100644 --- a/src/json-pipe.js +++ b/src/json-pipe.js @@ -61,7 +61,12 @@ async function fetchJsonContent(state, req, res) { res.body = ''; res.headers.delete('content-type'); res.headers.set('location', redirectLocation); - res.headers.set('x-surrogate-key', await computeSurrogateKey(`${contentBusId}${info.path}`)); + const keys = []; + keys.push(await computeSurrogateKey(`${contentBusId}${info.path}`)); + if (state.content.sourceBus === 'content') { + keys.push(contentBusId); + } + res.headers.set('x-surrogate-key', keys.join(' ')); res.error = 'moved'; return; } @@ -92,8 +97,10 @@ async function computeSurrogateKeys(state) { if (state.info.path === '/config.json') { keys.push(await computeSurrogateKey(`${state.site}--${state.org}_config.json`)); } - keys.push(pathKey.replace(/\//g, '_')); // TODO: remove keys.push(await computeSurrogateKey(pathKey)); + if (state.content?.sourceBus === 'content') { + keys.push(state.contentBusId); + } return keys; } diff --git a/src/steps/fetch-content.js b/src/steps/fetch-content.js index 2f87cd58..c6d0a4c1 100644 --- a/src/steps/fetch-content.js +++ b/src/steps/fetch-content.js @@ -41,7 +41,12 @@ export default async function fetchContent(state, req, res) { redirectLocation += '.plain.html'; } res.headers.set('location', redirectLocation); - res.headers.set('x-surrogate-key', await computeSurrogateKey(`${contentBusId}${info.path}`)); + const keys = []; + keys.push(await computeSurrogateKey(`${contentBusId}${info.path}`)); + if (!isCode) { + keys.push(contentBusId); + } + res.headers.set('x-surrogate-key', keys.join(' ')); res.error = 'moved'; return; } diff --git a/src/steps/set-x-surrogate-key-header.js b/src/steps/set-x-surrogate-key-header.js index f52d3004..62a5aacd 100644 --- a/src/steps/set-x-surrogate-key-header.js +++ b/src/steps/set-x-surrogate-key-header.js @@ -49,6 +49,7 @@ export default async function setXSurrogateKeyHeader(state, req, res) { hash, `${contentBusId}_metadata`, `${ref}--${repo}--${owner}_head`, + contentBusId, ]; // for folder-mapped resources, we also need to include the surrogate key of the mapped metadata diff --git a/test/html-pipe.test.js b/test/html-pipe.test.js index 8a1d84c0..2c23c8b5 100644 --- a/test/html-pipe.test.js +++ b/test/html-pipe.test.js @@ -119,7 +119,7 @@ describe('HTML Pipe Test', () => { assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), { 'content-type': 'text/markdown; charset=utf-8', 'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT', - 'x-surrogate-key': 'FzT3jXtDSYMYOTq1 foo-id_metadata super-test--helix-pages--adobe_head', + 'x-surrogate-key': 'FzT3jXtDSYMYOTq1 foo-id_metadata super-test--helix-pages--adobe_head foo-id', }); }); @@ -144,7 +144,7 @@ describe('HTML Pipe Test', () => { assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), { 'content-type': 'text/html; charset=utf-8', 'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT', - 'x-surrogate-key': 'FzT3jXtDSYMYOTq1 foo-id_metadata super-test--helix-pages--adobe_head', + 'x-surrogate-key': 'FzT3jXtDSYMYOTq1 foo-id_metadata super-test--helix-pages--adobe_head foo-id', }); }); @@ -169,7 +169,7 @@ describe('HTML Pipe Test', () => { assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), { 'content-type': 'text/markdown; charset=utf-8', 'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT', - 'x-surrogate-key': 'iQzO-EvK0WKNO_o0 foo-id_metadata super-test--helix-pages--adobe_head', + 'x-surrogate-key': 'iQzO-EvK0WKNO_o0 foo-id_metadata super-test--helix-pages--adobe_head foo-id', }); }); @@ -194,7 +194,7 @@ describe('HTML Pipe Test', () => { assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), { 'content-type': 'text/html; charset=utf-8', 'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT', - 'x-surrogate-key': 'iQzO-EvK0WKNO_o0 foo-id_metadata super-test--helix-pages--adobe_head', + 'x-surrogate-key': 'iQzO-EvK0WKNO_o0 foo-id_metadata super-test--helix-pages--adobe_head foo-id', }); }); @@ -221,7 +221,7 @@ describe('HTML Pipe Test', () => { assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), { 'content-type': 'text/html; charset=utf-8', 'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT', - 'x-surrogate-key': 'kvcvppnfHtt5omSX foo-id_metadata super-test--helix-pages--adobe_head', + 'x-surrogate-key': 'kvcvppnfHtt5omSX foo-id_metadata super-test--helix-pages--adobe_head foo-id', }); }); }); diff --git a/test/json-pipe.test.js b/test/json-pipe.test.js index c1b59609..de0a9bb0 100644 --- a/test/json-pipe.test.js +++ b/test/json-pipe.test.js @@ -137,7 +137,7 @@ describe('JSON Pipe Test', () => { }); assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), { 'content-type': 'application/json', - 'x-surrogate-key': 'foobar_en_index.json Atrz_qDg26DmSe9a', + 'x-surrogate-key': 'Atrz_qDg26DmSe9a foobar', 'last-modified': 'Wed, 12 Oct 2009 17:50:00 GMT', }); }); @@ -148,7 +148,7 @@ describe('JSON Pipe Test', () => { assert.strictEqual(resp.status, 404); assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), { 'x-error': 'failed to load /config.json: 404', - 'x-surrogate-key': 'U_NW4adJU7Qazf-I foobar_config.json kz8SoCaNqfp4ohQo', + 'x-surrogate-key': 'U_NW4adJU7Qazf-I kz8SoCaNqfp4ohQo foobar', }); }); @@ -165,7 +165,7 @@ describe('JSON Pipe Test', () => { assert.strictEqual(resp.status, 200); assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), { 'content-type': 'application/json', - 'x-surrogate-key': 'U_NW4adJU7Qazf-I foobar_config.json kz8SoCaNqfp4ohQo', + 'x-surrogate-key': 'U_NW4adJU7Qazf-I kz8SoCaNqfp4ohQo foobar', }); assert.deepStrictEqual(await resp.json(), { public: { @@ -193,7 +193,7 @@ describe('JSON Pipe Test', () => { }); assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), { 'content-type': 'application/json', - 'x-surrogate-key': 'foobar_en_index.json Atrz_qDg26DmSe9a', + 'x-surrogate-key': 'Atrz_qDg26DmSe9a foobar', 'last-modified': 'Wed, 12 Oct 2009 17:50:00 GMT', }); }); @@ -214,7 +214,7 @@ describe('JSON Pipe Test', () => { 'access-control-allow-origin': '*', 'content-security-policy': 'default-src \'self\'', 'last-modified': 'Wed, 12 Oct 2009 17:50:00 GMT', - 'x-surrogate-key': 'foobar_en_index.json Atrz_qDg26DmSe9a', + 'x-surrogate-key': 'Atrz_qDg26DmSe9a foobar', 'content-type': 'application/json', }); }); @@ -229,7 +229,7 @@ describe('JSON Pipe Test', () => { 'access-control-allow-origin': '*', 'content-security-policy': 'default-src \'self\'', 'x-error': 'failed to load /en/index.json: 404', - 'x-surrogate-key': 'foobar_en_index.json Atrz_qDg26DmSe9a', + 'x-surrogate-key': 'Atrz_qDg26DmSe9a foobar', }); }); @@ -249,7 +249,7 @@ describe('JSON Pipe Test', () => { assert.strictEqual(resp.status, 301); assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), { 'location': '/de/index.json', - 'x-surrogate-key': 'Atrz_qDg26DmSe9a', + 'x-surrogate-key': 'Atrz_qDg26DmSe9a foobar', }); }); @@ -291,7 +291,7 @@ describe('JSON Pipe Test', () => { const headers = Object.fromEntries(resp.headers.entries()); assert.deepStrictEqual(headers, { 'last-modified': 'Wed, 12 Oct 2009 17:50:00 GMT', - 'x-surrogate-key': 'foobar_en_index.json Atrz_qDg26DmSe9a', + 'x-surrogate-key': 'Atrz_qDg26DmSe9a foobar', 'content-type': 'application/json', }); }); @@ -323,7 +323,7 @@ describe('JSON Pipe Test', () => { const headers = Object.fromEntries(resp.headers.entries()); assert.deepStrictEqual(headers, { 'last-modified': 'Wed, 12 Oct 2009 15:50:00 GMT', - 'x-surrogate-key': 'foobar_en_index.json Atrz_qDg26DmSe9a', + 'x-surrogate-key': 'Atrz_qDg26DmSe9a foobar', 'content-type': 'application/json', }); }); @@ -359,7 +359,7 @@ describe('JSON Pipe Test', () => { assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), { 'content-type': 'application/json', 'last-modified': 'Wed, 12 Oct 2009 17:50:00 GMT', - 'x-surrogate-key': 'ref--repo--owner_en_index.json SIMSxecp2CJXqGYs', + 'x-surrogate-key': 'SIMSxecp2CJXqGYs', }); }); @@ -394,7 +394,7 @@ describe('JSON Pipe Test', () => { assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), { 'content-type': 'application/json', 'last-modified': 'Wed, 12 Oct 2009 17:50:00 GMT', - 'x-surrogate-key': 'ref--repo--owner_en_index.json SIMSxecp2CJXqGYs', + 'x-surrogate-key': 'SIMSxecp2CJXqGYs', }); }); diff --git a/test/rendering.test.js b/test/rendering.test.js index a51cdb65..1e3fcd25 100644 --- a/test/rendering.test.js +++ b/test/rendering.test.js @@ -406,17 +406,17 @@ describe('Rendering', () => { describe('Miscellaneous', () => { it('sets the surrogate-keys correctly', async () => { const resp = await testRender('page-block-empty-cols'); - assert.strictEqual(resp.headers.get('x-surrogate-key'), 'rDFj9gBeGHx_FI2T foo-id_metadata super-test--helix-pages--adobe_head'); + assert.strictEqual(resp.headers.get('x-surrogate-key'), 'rDFj9gBeGHx_FI2T foo-id_metadata super-test--helix-pages--adobe_head foo-id'); }); it('sets the surrogate-keys correctly for plain', async () => { const resp = await testRenderPlain('one-section'); - assert.strictEqual(resp.headers.get('x-surrogate-key'), 'oHjg_WDu20CBS4rD foo-id_metadata super-test--helix-pages--adobe_head'); + assert.strictEqual(resp.headers.get('x-surrogate-key'), 'oHjg_WDu20CBS4rD foo-id_metadata super-test--helix-pages--adobe_head foo-id'); }); it('sets the surrogate-keys correctly for index.plain.html', async () => { const resp = await testRenderPlain('one-section/index', 'one-section/index'); - assert.strictEqual(resp.headers.get('x-surrogate-key'), 'Vp-I6NB8PSor1sI6 foo-id_metadata super-test--helix-pages--adobe_head'); + assert.strictEqual(resp.headers.get('x-surrogate-key'), 'Vp-I6NB8PSor1sI6 foo-id_metadata super-test--helix-pages--adobe_head foo-id'); }); it('renders the fedpub header correctly', async () => { @@ -569,7 +569,7 @@ describe('Rendering', () => { assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), { 'access-control-allow-origin': '*', 'content-type': 'text/html; charset=utf-8', - 'x-surrogate-key': 'AkcHu8fRFT7HarTR foo-id_metadata super-test--helix-pages--adobe_head AkcHu8fRFT7HarTR_metadata z8NGXvKB0X5Fzcnd', + 'x-surrogate-key': 'AkcHu8fRFT7HarTR foo-id_metadata super-test--helix-pages--adobe_head foo-id AkcHu8fRFT7HarTR_metadata z8NGXvKB0X5Fzcnd', link: '; rel=modulepreload; as=script; crossorigin=use-credentials', }); }); @@ -586,7 +586,7 @@ describe('Rendering', () => { 'access-control-allow-origin': '*', 'content-type': 'text/html; charset=utf-8', 'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT', - 'x-surrogate-key': 'SCKEB3bkK0hFm4aS foo-id_metadata super-test--helix-pages--adobe_head SCKEB3bkK0hFm4aS_metadata JHEAK7b1XZvfOJpY', + 'x-surrogate-key': 'SCKEB3bkK0hFm4aS foo-id_metadata super-test--helix-pages--adobe_head foo-id SCKEB3bkK0hFm4aS_metadata JHEAK7b1XZvfOJpY', link: '; rel=modulepreload; as=script; crossorigin=use-credentials', }); }); @@ -609,7 +609,7 @@ describe('Rendering', () => { 'access-control-allow-origin': '*', 'content-type': 'text/html; charset=utf-8', 'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT', - 'x-surrogate-key': 'SUhNxkR2spoxY489 foo-id_metadata super-test--helix-pages--adobe_head', + 'x-surrogate-key': 'SUhNxkR2spoxY489 foo-id_metadata super-test--helix-pages--adobe_head foo-id', link: '; rel=modulepreload; as=script; crossorigin=use-credentials', }); }); @@ -630,7 +630,7 @@ describe('Rendering', () => { 'access-control-allow-origin': '*', 'content-type': 'text/html; charset=utf-8', 'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT', - 'x-surrogate-key': 'AkcHu8fRFT7HarTR foo-id_metadata super-test--helix-pages--adobe_head AkcHu8fRFT7HarTR_metadata z8NGXvKB0X5Fzcnd', + 'x-surrogate-key': 'AkcHu8fRFT7HarTR foo-id_metadata super-test--helix-pages--adobe_head foo-id AkcHu8fRFT7HarTR_metadata z8NGXvKB0X5Fzcnd', link: '; rel=modulepreload; as=script; crossorigin=use-credentials', }); }); @@ -649,7 +649,7 @@ describe('Rendering', () => { assert.match(body, //); assert.deepStrictEqual(Object.fromEntries(headers.entries()), { 'content-type': 'text/html; charset=utf-8', - 'x-surrogate-key': 'o_fNQBWBLWTIfYqV foo-id_metadata super-test--helix-pages--adobe_head', + 'x-surrogate-key': 'o_fNQBWBLWTIfYqV foo-id_metadata super-test--helix-pages--adobe_head foo-id', 'last-modified': 'Wed, 12 Jan 2022 11:33:01 GMT', }); }); @@ -662,7 +662,7 @@ describe('Rendering', () => { 'access-control-allow-origin': '*', 'content-type': 'text/html; charset=utf-8', 'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT', - 'x-surrogate-key': 'VmeAc3K7QsCRzj5Z foo-id_metadata super-test--helix-pages--adobe_head', + 'x-surrogate-key': 'VmeAc3K7QsCRzj5Z foo-id_metadata super-test--helix-pages--adobe_head foo-id', link: '; rel=modulepreload; as=script; crossorigin=use-credentials', }); }); diff --git a/test/sitemap-pipe.test.js b/test/sitemap-pipe.test.js index 55b5c26f..e81815b9 100644 --- a/test/sitemap-pipe.test.js +++ b/test/sitemap-pipe.test.js @@ -74,7 +74,7 @@ describe('Sitemap Pipe Test', () => { assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), { 'content-type': 'text/plain; charset=utf-8', 'x-error': 'failed to load /sitemap.xml from content-bus: 404', - 'x-surrogate-key': 'RXei-6EcTEMTEIqi foobar_metadata ref--repo--owner_head', + 'x-surrogate-key': 'RXei-6EcTEMTEIqi foobar_metadata ref--repo--owner_head foobar', }); }); @@ -92,7 +92,7 @@ describe('Sitemap Pipe Test', () => { assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), { 'content-type': 'text/plain; charset=utf-8', 'x-error': 'Failed to parse /sitemap.json: Unexpected token \'h\', "this is not JSON" is not valid JSON', - 'x-surrogate-key': 'RXei-6EcTEMTEIqi foobar_metadata ref--repo--owner_head', + 'x-surrogate-key': 'RXei-6EcTEMTEIqi foobar_metadata ref--repo--owner_head foobar', }); }); @@ -110,7 +110,7 @@ describe('Sitemap Pipe Test', () => { assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), { 'content-type': 'text/plain; charset=utf-8', 'x-error': "Expected 'data' array not found in /sitemap.json", - 'x-surrogate-key': 'RXei-6EcTEMTEIqi foobar_metadata ref--repo--owner_head', + 'x-surrogate-key': 'RXei-6EcTEMTEIqi foobar_metadata ref--repo--owner_head foobar', }); }); @@ -128,7 +128,7 @@ describe('Sitemap Pipe Test', () => { assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), { 'content-type': 'application/xml; charset=utf-8', 'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT', - 'x-surrogate-key': 'RXei-6EcTEMTEIqi foobar_metadata ref--repo--owner_head', + 'x-surrogate-key': 'RXei-6EcTEMTEIqi foobar_metadata ref--repo--owner_head foobar', }); assert.strictEqual(resp.body, ` @@ -154,7 +154,7 @@ describe('Sitemap Pipe Test', () => { assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), { 'content-type': 'application/xml; charset=utf-8', 'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT', - 'x-surrogate-key': 'RXei-6EcTEMTEIqi foobar_metadata ref--repo--owner_head', + 'x-surrogate-key': 'RXei-6EcTEMTEIqi foobar_metadata ref--repo--owner_head foobar', }); assert.strictEqual(resp.body, ` @@ -190,7 +190,7 @@ describe('Sitemap Pipe Test', () => { assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), { 'content-type': 'application/xml; charset=utf-8', 'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT', - 'x-surrogate-key': 'RXei-6EcTEMTEIqi foobar_metadata ref--repo--owner_head', + 'x-surrogate-key': 'RXei-6EcTEMTEIqi foobar_metadata ref--repo--owner_head foobar', }); assert.strictEqual(resp.body, ` @@ -227,7 +227,7 @@ describe('Sitemap Pipe Test', () => { assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), { 'content-type': 'application/xml; charset=utf-8', 'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT', - 'x-surrogate-key': 'RXei-6EcTEMTEIqi foobar_metadata ref--repo--owner_head', + 'x-surrogate-key': 'RXei-6EcTEMTEIqi foobar_metadata ref--repo--owner_head foobar', }); assert.strictEqual(resp.body, ` @@ -264,7 +264,7 @@ describe('Sitemap Pipe Test', () => { assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), { 'content-type': 'application/xml; charset=utf-8', 'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT', - 'x-surrogate-key': 'RXei-6EcTEMTEIqi foobar_metadata ref--repo--owner_head', + 'x-surrogate-key': 'RXei-6EcTEMTEIqi foobar_metadata ref--repo--owner_head foobar', }); assert.strictEqual(resp.body, ` @@ -294,7 +294,7 @@ describe('Sitemap Pipe Test', () => { assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), { 'content-type': 'application/xml; charset=utf-8', 'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT', - 'x-surrogate-key': 'RXei-6EcTEMTEIqi foobar_metadata ref--repo--owner_head', + 'x-surrogate-key': 'RXei-6EcTEMTEIqi foobar_metadata ref--repo--owner_head foobar', }); assert.strictEqual(resp.body, ` From edaa518db81259d4c247c6cc8445277f41a3ccfb Mon Sep 17 00:00:00 2001 From: Stefan Guggisberg Date: Tue, 21 May 2024 15:34:08 +0200 Subject: [PATCH 2/3] feat: add ref--repo-owner_code for all code resources --- src/json-pipe.js | 2 ++ src/steps/fetch-404.js | 2 +- src/steps/fetch-content.js | 7 +++++-- test/json-pipe.test.js | 4 ++-- test/rendering.test.js | 16 +++++++++++----- 5 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/json-pipe.js b/src/json-pipe.js index eb794111..d313a118 100644 --- a/src/json-pipe.js +++ b/src/json-pipe.js @@ -100,6 +100,8 @@ async function computeSurrogateKeys(state) { keys.push(await computeSurrogateKey(pathKey)); if (state.content?.sourceBus === 'content') { keys.push(state.contentBusId); + } else { + keys.push(`${state.ref}--${state.repo}--${state.owner}_code`); } return keys; } diff --git a/src/steps/fetch-404.js b/src/steps/fetch-404.js index 1fe837f0..8acfde04 100644 --- a/src/steps/fetch-404.js +++ b/src/steps/fetch-404.js @@ -38,5 +38,5 @@ export default async function fetch404(state, req, res) { // set 404 keys in any case const pathKey = await getPathKey(state); - res.headers.set('x-surrogate-key', `${pathKey} ${ref}--${repo}--${owner}_404`); + res.headers.set('x-surrogate-key', `${pathKey} ${ref}--${repo}--${owner}_404 ${ref}--${repo}--${owner}_code`); } diff --git a/src/steps/fetch-content.js b/src/steps/fetch-content.js index c6d0a4c1..080c7220 100644 --- a/src/steps/fetch-content.js +++ b/src/steps/fetch-content.js @@ -42,8 +42,11 @@ export default async function fetchContent(state, req, res) { } res.headers.set('location', redirectLocation); const keys = []; - keys.push(await computeSurrogateKey(`${contentBusId}${info.path}`)); - if (!isCode) { + if (isCode) { + keys.push(await computeSurrogateKey(`${ref}--${repo}--${owner}${info.path}`)); + keys.push(`${ref}--${repo}--${owner}_code`); + } else { + keys.push(await computeSurrogateKey(`${contentBusId}${info.path}`)); keys.push(contentBusId); } res.headers.set('x-surrogate-key', keys.join(' ')); diff --git a/test/json-pipe.test.js b/test/json-pipe.test.js index de0a9bb0..061dd479 100644 --- a/test/json-pipe.test.js +++ b/test/json-pipe.test.js @@ -359,7 +359,7 @@ describe('JSON Pipe Test', () => { assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), { 'content-type': 'application/json', 'last-modified': 'Wed, 12 Oct 2009 17:50:00 GMT', - 'x-surrogate-key': 'SIMSxecp2CJXqGYs', + 'x-surrogate-key': 'SIMSxecp2CJXqGYs ref--repo--owner_code', }); }); @@ -394,7 +394,7 @@ describe('JSON Pipe Test', () => { assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), { 'content-type': 'application/json', 'last-modified': 'Wed, 12 Oct 2009 17:50:00 GMT', - 'x-surrogate-key': 'SIMSxecp2CJXqGYs', + 'x-surrogate-key': 'SIMSxecp2CJXqGYs ref--repo--owner_code', }); }); diff --git a/test/rendering.test.js b/test/rendering.test.js index 1e3fcd25..4c990022 100644 --- a/test/rendering.test.js +++ b/test/rendering.test.js @@ -469,7 +469,7 @@ describe('Rendering', () => { assert.deepStrictEqual(Object.fromEntries(headers.entries()), { 'content-type': 'text/html; charset=utf-8', 'last-modified': 'Wed, 12 Oct 2009 17:50:00 GMT', - 'x-surrogate-key': 'OYsA_wfqip5EuBu6 super-test--helix-pages--adobe_404', + 'x-surrogate-key': 'OYsA_wfqip5EuBu6 super-test--helix-pages--adobe_404 super-test--helix-pages--adobe_code', 'x-error': 'failed to load /not-found-with-handler.md from content-bus: 404', 'access-control-allow-origin': '*', link: '; rel=modulepreload; as=script; crossorigin=use-credentials', @@ -485,7 +485,7 @@ describe('Rendering', () => { assert.deepStrictEqual(Object.fromEntries(headers.entries()), { 'content-type': 'text/html; charset=utf-8', 'last-modified': 'Wed, 12 Oct 2009 17:50:00 GMT', - 'x-surrogate-key': 'OYsA_wfqip5EuBu6 super-test--helix-pages--adobe_404', + 'x-surrogate-key': 'OYsA_wfqip5EuBu6 super-test--helix-pages--adobe_404 super-test--helix-pages--adobe_code', 'x-error': 'failed to load /not-found-with-handler.md from content-bus: 404', 'access-control-allow-origin': '*', }); @@ -501,7 +501,7 @@ describe('Rendering', () => { 'content-type': 'text/html; charset=utf-8', 'last-modified': 'Wed, 12 Oct 2009 17:50:00 GMT', 'x-error': 'failed to load /not-found-with-handler.html from code-bus: 404', - 'x-surrogate-key': 'ta3V7wR3zlRh1b0E super-test--helix-pages--adobe_404', + 'x-surrogate-key': 'ta3V7wR3zlRh1b0E super-test--helix-pages--adobe_404 super-test--helix-pages--adobe_code', link: '; rel=modulepreload; as=script; crossorigin=use-credentials', 'access-control-allow-origin': '*', }); @@ -517,7 +517,7 @@ describe('Rendering', () => { 'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT', link: '; rel=modulepreload; as=script; crossorigin=use-credentials', 'x-error': 'request to /index.md not allowed (no-index).', - 'x-surrogate-key': 'FzT3jXtDSYMYOTq1 super-test--helix-pages--adobe_404', + 'x-surrogate-key': 'FzT3jXtDSYMYOTq1 super-test--helix-pages--adobe_404 super-test--helix-pages--adobe_code', }); assert.strictEqual(body.trim(), ''); }); @@ -534,12 +534,18 @@ describe('Rendering', () => { assert.strictEqual(ret.headers.get('location'), '/foo.plain.html'); }); - it('renders redirect for static html', async () => { + it('renders redirect for static html (content)', async () => { loader.headers('static-content.html', 'x-amz-meta-redirect-location', '/foo'); const ret = await render(new URL('https://localhost/static-content.html'), '', 301); assert.strictEqual(ret.headers.get('location'), '/foo'); }); + it('renders redirect for static html (code)', async () => { + loader.headers('static.html', 'x-amz-meta-redirect-location', '/foo'); + const ret = await render(new URL('https://localhost/static.html'), '', 301); + assert.strictEqual(ret.headers.get('location'), '/foo'); + }); + it('respect folder mapping: skip existing resources', async () => { loader.status('products.md', 200); let resp = await render(new URL('https://helix-pipeline.com/products'), '', 200); From 797aad2a1b34aa7eb95fc9ef57a65b60afed2c76 Mon Sep 17 00:00:00 2001 From: Stefan Guggisberg Date: Wed, 22 May 2024 13:58:49 +0200 Subject: [PATCH 3/3] feat: add ref--repo--owner_code surrogate key for pipeline responses depending on codebus resources --- src/json-pipe.js | 5 ++++- src/robots-pipe.js | 1 - test/json-pipe.test.js | 36 +++++++++++++++++++++++------------- test/robots-pipe.test.js | 6 +++--- 4 files changed, 30 insertions(+), 18 deletions(-) diff --git a/src/json-pipe.js b/src/json-pipe.js index d313a118..acd58581 100644 --- a/src/json-pipe.js +++ b/src/json-pipe.js @@ -62,9 +62,12 @@ async function fetchJsonContent(state, req, res) { res.headers.delete('content-type'); res.headers.set('location', redirectLocation); const keys = []; - keys.push(await computeSurrogateKey(`${contentBusId}${info.path}`)); if (state.content.sourceBus === 'content') { + keys.push(await computeSurrogateKey(`${contentBusId}${info.path}`)); keys.push(contentBusId); + } else { + keys.push(`${ref}--${repo}--${owner}_code`); + keys.push(await computeSurrogateKey(`${ref}--${repo}--${owner}${info.path}`)); } res.headers.set('x-surrogate-key', keys.join(' ')); res.error = 'moved'; diff --git a/src/robots-pipe.js b/src/robots-pipe.js index 0e1c31cd..b63e5a55 100644 --- a/src/robots-pipe.js +++ b/src/robots-pipe.js @@ -120,7 +120,6 @@ async function computeSurrogateKeys(state) { const pathKey = `${state.ref}--${state.repo}--${state.owner}${state.info.path}`; keys.push(await computeSurrogateKey(`${state.site}--${state.org}_config.json`)); - keys.push(pathKey.replace(/\//g, '_')); // TODO: remove keys.push(await computeSurrogateKey(pathKey)); return keys; } diff --git a/test/json-pipe.test.js b/test/json-pipe.test.js index 061dd479..e418122f 100644 --- a/test/json-pipe.test.js +++ b/test/json-pipe.test.js @@ -253,7 +253,29 @@ describe('JSON Pipe Test', () => { }); }); - it('ignores newer last modified from metadata.json even if newer', async () => { + it('respects redirects (code)', async () => { + const state = createDefaultState(); + state.s3Loader + .reply( + 'helix-code-bus', + 'owner/repo/ref/en/index.json', + new PipelineResponse(TEST_SINGLE_SHEET, { + headers: { + 'content-type': 'application/json', + 'x-amz-meta-redirect-location': '/de/index.json', + }, + }), + ) + .reply('helix-content-bus', 'foobar/preview/en/index.json', null); + const resp = await jsonPipe(state, new PipelineRequest('https://json-filter.com/?limit=10&offset=5')); + assert.strictEqual(resp.status, 301); + assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), { + 'location': '/de/index.json', + 'x-surrogate-key': 'ref--repo--owner_code SIMSxecp2CJXqGYs', + }); + }); + + it('ignores last modified from metadata.json even if newer', async () => { const state = createDefaultState(); state.s3Loader.reply( 'helix-content-bus', @@ -267,18 +289,6 @@ describe('JSON Pipe Test', () => { }, }), ); - state.s3Loader.reply( - 'helix-code-bus', - 'foobar/preview/metadata.json', - new PipelineResponse(JSON.stringify({ - data: [ - ], - }), { - headers: { - 'last-modified': 'Wed, 15 Oct 2009 17:50:00 GMT', - }, - }), - ); const resp = await jsonPipe(state, new PipelineRequest('https://json-filter.com/?limit=10&offset=5')); assert.strictEqual(resp.status, 200); assert.deepStrictEqual(await resp.json(), { diff --git a/test/robots-pipe.test.js b/test/robots-pipe.test.js index 60d7d80a..ea472e53 100644 --- a/test/robots-pipe.test.js +++ b/test/robots-pipe.test.js @@ -69,7 +69,7 @@ describe('Robots Pipe Test', () => { assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), { 'content-type': 'text/plain; charset=utf-8', vary: 'x-forwarded-host', - 'x-surrogate-key': 'U_NW4adJU7Qazf-I ref--repo--owner_robots.txt ZcR1sjWODctSccZh', + 'x-surrogate-key': 'U_NW4adJU7Qazf-I ZcR1sjWODctSccZh', }); assert.strictEqual(resp.body, 'this is my robots.txt'); }); @@ -149,7 +149,7 @@ describe('Robots Pipe Test', () => { assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), { 'content-type': 'text/plain; charset=utf-8', vary: 'x-forwarded-host', - 'x-surrogate-key': 'U_NW4adJU7Qazf-I ref--repo--owner_robots.txt ZcR1sjWODctSccZh', + 'x-surrogate-key': 'U_NW4adJU7Qazf-I ZcR1sjWODctSccZh', }); assert.strictEqual(resp.body, `User-Agent: * Allow: / @@ -184,7 +184,7 @@ Sitemap: https://www.example.com/sitemap.xml`); assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), { 'content-type': 'text/plain; charset=utf-8', vary: 'x-forwarded-host', - 'x-surrogate-key': 'U_NW4adJU7Qazf-I ref--repo--owner_robots.txt ZcR1sjWODctSccZh', + 'x-surrogate-key': 'U_NW4adJU7Qazf-I ZcR1sjWODctSccZh', }); assert.strictEqual(resp.body, `User-Agent: * Allow: /