Skip to content

feat: add contentbusid surrogate key for pipeline responses depending… #603

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
May 22, 2024
Merged

feat: add contentbusid surrogate key for pipeline responses depending… #603

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e821162
feat: add contentbusid surrogate key for pipeline responses depending…
stefan-guggisberg May 20, 2024
edaa518
feat: add ref--repo-owner_code for all code resources
stefan-guggisberg May 21, 2024
797aad2
feat: add ref--repo--owner_code surrogate key for pipeline responses …
stefan-guggisberg May 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions src/json-pipe.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,12 @@ async function fetchJsonContent(state, req, res) {
res.body = '';
res.headers.delete('content-type');
res.headers.set('location', redirectLocation);
res.headers.set('x-surrogate-key', await computeSurrogateKey(`${contentBusId}${info.path}`));
const keys = [];
keys.push(await computeSurrogateKey(`${contentBusId}${info.path}`));
if (state.content.sourceBus === 'content') {
keys.push(contentBusId);
}
res.headers.set('x-surrogate-key', keys.join(' '));
res.error = 'moved';
return;
}
Expand Down Expand Up @@ -92,8 +97,12 @@ async function computeSurrogateKeys(state) {
if (state.info.path === '/config.json') {
keys.push(await computeSurrogateKey(`${state.site}--${state.org}_config.json`));
}
keys.push(pathKey.replace(/\//g, '_')); // TODO: remove
keys.push(await computeSurrogateKey(pathKey));
if (state.content?.sourceBus === 'content') {
keys.push(state.contentBusId);
} else {
keys.push(`${state.ref}--${state.repo}--${state.owner}_code`);
}
return keys;
}

Expand Down
2 changes: 1 addition & 1 deletion src/steps/fetch-404.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,5 +38,5 @@ export default async function fetch404(state, req, res) {

// set 404 keys in any case
const pathKey = await getPathKey(state);
res.headers.set('x-surrogate-key', `${pathKey} ${ref}--${repo}--${owner}_404`);
res.headers.set('x-surrogate-key', `${pathKey} ${ref}--${repo}--${owner}_404 ${ref}--${repo}--${owner}_code`);
}
10 changes: 9 additions & 1 deletion src/steps/fetch-content.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,15 @@ export default async function fetchContent(state, req, res) {
redirectLocation += '.plain.html';
}
res.headers.set('location', redirectLocation);
res.headers.set('x-surrogate-key', await computeSurrogateKey(`${contentBusId}${info.path}`));
const keys = [];
if (isCode) {
keys.push(await computeSurrogateKey(`${ref}--${repo}--${owner}${info.path}`));
keys.push(`${ref}--${repo}--${owner}_code`);
} else {
keys.push(await computeSurrogateKey(`${contentBusId}${info.path}`));
keys.push(contentBusId);
}
res.headers.set('x-surrogate-key', keys.join(' '));
res.error = 'moved';
return;
}
Expand Down
1 change: 1 addition & 0 deletions src/steps/set-x-surrogate-key-header.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ export default async function setXSurrogateKeyHeader(state, req, res) {
hash,
`${contentBusId}_metadata`,
`${ref}--${repo}--${owner}_head`,
contentBusId,
];

// for folder-mapped resources, we also need to include the surrogate key of the mapped metadata
Expand Down
10 changes: 5 additions & 5 deletions test/html-pipe.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ describe('HTML Pipe Test', () => {
assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), {
'content-type': 'text/markdown; charset=utf-8',
'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT',
'x-surrogate-key': 'FzT3jXtDSYMYOTq1 foo-id_metadata super-test--helix-pages--adobe_head',
'x-surrogate-key': 'FzT3jXtDSYMYOTq1 foo-id_metadata super-test--helix-pages--adobe_head foo-id',
});
});

Expand All @@ -144,7 +144,7 @@ describe('HTML Pipe Test', () => {
assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), {
'content-type': 'text/html; charset=utf-8',
'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT',
'x-surrogate-key': 'FzT3jXtDSYMYOTq1 foo-id_metadata super-test--helix-pages--adobe_head',
'x-surrogate-key': 'FzT3jXtDSYMYOTq1 foo-id_metadata super-test--helix-pages--adobe_head foo-id',
});
});

Expand All @@ -169,7 +169,7 @@ describe('HTML Pipe Test', () => {
assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), {
'content-type': 'text/markdown; charset=utf-8',
'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT',
'x-surrogate-key': 'iQzO-EvK0WKNO_o0 foo-id_metadata super-test--helix-pages--adobe_head',
'x-surrogate-key': 'iQzO-EvK0WKNO_o0 foo-id_metadata super-test--helix-pages--adobe_head foo-id',
});
});

Expand All @@ -194,7 +194,7 @@ describe('HTML Pipe Test', () => {
assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), {
'content-type': 'text/html; charset=utf-8',
'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT',
'x-surrogate-key': 'iQzO-EvK0WKNO_o0 foo-id_metadata super-test--helix-pages--adobe_head',
'x-surrogate-key': 'iQzO-EvK0WKNO_o0 foo-id_metadata super-test--helix-pages--adobe_head foo-id',
});
});

Expand All @@ -221,7 +221,7 @@ describe('HTML Pipe Test', () => {
assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), {
'content-type': 'text/html; charset=utf-8',
'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT',
'x-surrogate-key': 'kvcvppnfHtt5omSX foo-id_metadata super-test--helix-pages--adobe_head',
'x-surrogate-key': 'kvcvppnfHtt5omSX foo-id_metadata super-test--helix-pages--adobe_head foo-id',
});
});
});
22 changes: 11 additions & 11 deletions test/json-pipe.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ describe('JSON Pipe Test', () => {
});
assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), {
'content-type': 'application/json',
'x-surrogate-key': 'foobar_en_index.json Atrz_qDg26DmSe9a',
'x-surrogate-key': 'Atrz_qDg26DmSe9a foobar',
'last-modified': 'Wed, 12 Oct 2009 17:50:00 GMT',
});
});
Expand All @@ -148,7 +148,7 @@ describe('JSON Pipe Test', () => {
assert.strictEqual(resp.status, 404);
assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), {
'x-error': 'failed to load /config.json: 404',
'x-surrogate-key': 'U_NW4adJU7Qazf-I foobar_config.json kz8SoCaNqfp4ohQo',
'x-surrogate-key': 'U_NW4adJU7Qazf-I kz8SoCaNqfp4ohQo foobar',
});
});

Expand All @@ -165,7 +165,7 @@ describe('JSON Pipe Test', () => {
assert.strictEqual(resp.status, 200);
assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), {
'content-type': 'application/json',
'x-surrogate-key': 'U_NW4adJU7Qazf-I foobar_config.json kz8SoCaNqfp4ohQo',
'x-surrogate-key': 'U_NW4adJU7Qazf-I kz8SoCaNqfp4ohQo foobar',
});
assert.deepStrictEqual(await resp.json(), {
public: {
Expand Down Expand Up @@ -193,7 +193,7 @@ describe('JSON Pipe Test', () => {
});
assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), {
'content-type': 'application/json',
'x-surrogate-key': 'foobar_en_index.json Atrz_qDg26DmSe9a',
'x-surrogate-key': 'Atrz_qDg26DmSe9a foobar',
'last-modified': 'Wed, 12 Oct 2009 17:50:00 GMT',
});
});
Expand All @@ -214,7 +214,7 @@ describe('JSON Pipe Test', () => {
'access-control-allow-origin': '*',
'content-security-policy': 'default-src \'self\'',
'last-modified': 'Wed, 12 Oct 2009 17:50:00 GMT',
'x-surrogate-key': 'foobar_en_index.json Atrz_qDg26DmSe9a',
'x-surrogate-key': 'Atrz_qDg26DmSe9a foobar',
'content-type': 'application/json',
});
});
Expand All @@ -229,7 +229,7 @@ describe('JSON Pipe Test', () => {
'access-control-allow-origin': '*',
'content-security-policy': 'default-src \'self\'',
'x-error': 'failed to load /en/index.json: 404',
'x-surrogate-key': 'foobar_en_index.json Atrz_qDg26DmSe9a',
'x-surrogate-key': 'Atrz_qDg26DmSe9a foobar',
});
});

Expand All @@ -249,7 +249,7 @@ describe('JSON Pipe Test', () => {
assert.strictEqual(resp.status, 301);
assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), {
'location': '/de/index.json',
'x-surrogate-key': 'Atrz_qDg26DmSe9a',
'x-surrogate-key': 'Atrz_qDg26DmSe9a foobar',
});
});

Expand Down Expand Up @@ -291,7 +291,7 @@ describe('JSON Pipe Test', () => {
const headers = Object.fromEntries(resp.headers.entries());
assert.deepStrictEqual(headers, {
'last-modified': 'Wed, 12 Oct 2009 17:50:00 GMT',
'x-surrogate-key': 'foobar_en_index.json Atrz_qDg26DmSe9a',
'x-surrogate-key': 'Atrz_qDg26DmSe9a foobar',
'content-type': 'application/json',
});
});
Expand Down Expand Up @@ -323,7 +323,7 @@ describe('JSON Pipe Test', () => {
const headers = Object.fromEntries(resp.headers.entries());
assert.deepStrictEqual(headers, {
'last-modified': 'Wed, 12 Oct 2009 15:50:00 GMT',
'x-surrogate-key': 'foobar_en_index.json Atrz_qDg26DmSe9a',
'x-surrogate-key': 'Atrz_qDg26DmSe9a foobar',
'content-type': 'application/json',
});
});
Expand Down Expand Up @@ -359,7 +359,7 @@ describe('JSON Pipe Test', () => {
assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), {
'content-type': 'application/json',
'last-modified': 'Wed, 12 Oct 2009 17:50:00 GMT',
'x-surrogate-key': 'ref--repo--owner_en_index.json SIMSxecp2CJXqGYs',
'x-surrogate-key': 'SIMSxecp2CJXqGYs ref--repo--owner_code',
});
});

Expand Down Expand Up @@ -394,7 +394,7 @@ describe('JSON Pipe Test', () => {
assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), {
'content-type': 'application/json',
'last-modified': 'Wed, 12 Oct 2009 17:50:00 GMT',
'x-surrogate-key': 'ref--repo--owner_en_index.json SIMSxecp2CJXqGYs',
'x-surrogate-key': 'SIMSxecp2CJXqGYs ref--repo--owner_code',
});
});

Expand Down
34 changes: 20 additions & 14 deletions test/rendering.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -406,17 +406,17 @@ describe('Rendering', () => {
describe('Miscellaneous', () => {
it('sets the surrogate-keys correctly', async () => {
const resp = await testRender('page-block-empty-cols');
assert.strictEqual(resp.headers.get('x-surrogate-key'), 'rDFj9gBeGHx_FI2T foo-id_metadata super-test--helix-pages--adobe_head');
assert.strictEqual(resp.headers.get('x-surrogate-key'), 'rDFj9gBeGHx_FI2T foo-id_metadata super-test--helix-pages--adobe_head foo-id');
});

it('sets the surrogate-keys correctly for plain', async () => {
const resp = await testRenderPlain('one-section');
assert.strictEqual(resp.headers.get('x-surrogate-key'), 'oHjg_WDu20CBS4rD foo-id_metadata super-test--helix-pages--adobe_head');
assert.strictEqual(resp.headers.get('x-surrogate-key'), 'oHjg_WDu20CBS4rD foo-id_metadata super-test--helix-pages--adobe_head foo-id');
});

it('sets the surrogate-keys correctly for index.plain.html', async () => {
const resp = await testRenderPlain('one-section/index', 'one-section/index');
assert.strictEqual(resp.headers.get('x-surrogate-key'), 'Vp-I6NB8PSor1sI6 foo-id_metadata super-test--helix-pages--adobe_head');
assert.strictEqual(resp.headers.get('x-surrogate-key'), 'Vp-I6NB8PSor1sI6 foo-id_metadata super-test--helix-pages--adobe_head foo-id');
});

it('renders the fedpub header correctly', async () => {
Expand Down Expand Up @@ -469,7 +469,7 @@ describe('Rendering', () => {
assert.deepStrictEqual(Object.fromEntries(headers.entries()), {
'content-type': 'text/html; charset=utf-8',
'last-modified': 'Wed, 12 Oct 2009 17:50:00 GMT',
'x-surrogate-key': 'OYsA_wfqip5EuBu6 super-test--helix-pages--adobe_404',
'x-surrogate-key': 'OYsA_wfqip5EuBu6 super-test--helix-pages--adobe_404 super-test--helix-pages--adobe_code',
'x-error': 'failed to load /not-found-with-handler.md from content-bus: 404',
'access-control-allow-origin': '*',
link: '</scripts/scripts.js>; rel=modulepreload; as=script; crossorigin=use-credentials',
Expand All @@ -485,7 +485,7 @@ describe('Rendering', () => {
assert.deepStrictEqual(Object.fromEntries(headers.entries()), {
'content-type': 'text/html; charset=utf-8',
'last-modified': 'Wed, 12 Oct 2009 17:50:00 GMT',
'x-surrogate-key': 'OYsA_wfqip5EuBu6 super-test--helix-pages--adobe_404',
'x-surrogate-key': 'OYsA_wfqip5EuBu6 super-test--helix-pages--adobe_404 super-test--helix-pages--adobe_code',
'x-error': 'failed to load /not-found-with-handler.md from content-bus: 404',
'access-control-allow-origin': '*',
});
Expand All @@ -501,7 +501,7 @@ describe('Rendering', () => {
'content-type': 'text/html; charset=utf-8',
'last-modified': 'Wed, 12 Oct 2009 17:50:00 GMT',
'x-error': 'failed to load /not-found-with-handler.html from code-bus: 404',
'x-surrogate-key': 'ta3V7wR3zlRh1b0E super-test--helix-pages--adobe_404',
'x-surrogate-key': 'ta3V7wR3zlRh1b0E super-test--helix-pages--adobe_404 super-test--helix-pages--adobe_code',
link: '</scripts/scripts.js>; rel=modulepreload; as=script; crossorigin=use-credentials',
'access-control-allow-origin': '*',
});
Expand All @@ -517,7 +517,7 @@ describe('Rendering', () => {
'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT',
link: '</scripts/scripts.js>; rel=modulepreload; as=script; crossorigin=use-credentials',
'x-error': 'request to /index.md not allowed (no-index).',
'x-surrogate-key': 'FzT3jXtDSYMYOTq1 super-test--helix-pages--adobe_404',
'x-surrogate-key': 'FzT3jXtDSYMYOTq1 super-test--helix-pages--adobe_404 super-test--helix-pages--adobe_code',
});
assert.strictEqual(body.trim(), '');
});
Expand All @@ -534,12 +534,18 @@ describe('Rendering', () => {
assert.strictEqual(ret.headers.get('location'), '/foo.plain.html');
});

it('renders redirect for static html', async () => {
it('renders redirect for static html (content)', async () => {
loader.headers('static-content.html', 'x-amz-meta-redirect-location', '/foo');
const ret = await render(new URL('https://localhost/static-content.html'), '', 301);
assert.strictEqual(ret.headers.get('location'), '/foo');
});

it('renders redirect for static html (code)', async () => {
loader.headers('static.html', 'x-amz-meta-redirect-location', '/foo');
const ret = await render(new URL('https://localhost/static.html'), '', 301);
assert.strictEqual(ret.headers.get('location'), '/foo');
});

it('respect folder mapping: skip existing resources', async () => {
loader.status('products.md', 200);
let resp = await render(new URL('https://helix-pipeline.com/products'), '', 200);
Expand Down Expand Up @@ -569,7 +575,7 @@ describe('Rendering', () => {
assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), {
'access-control-allow-origin': '*',
'content-type': 'text/html; charset=utf-8',
'x-surrogate-key': 'AkcHu8fRFT7HarTR foo-id_metadata super-test--helix-pages--adobe_head AkcHu8fRFT7HarTR_metadata z8NGXvKB0X5Fzcnd',
'x-surrogate-key': 'AkcHu8fRFT7HarTR foo-id_metadata super-test--helix-pages--adobe_head foo-id AkcHu8fRFT7HarTR_metadata z8NGXvKB0X5Fzcnd',
link: '</scripts/scripts.js>; rel=modulepreload; as=script; crossorigin=use-credentials',
});
});
Expand All @@ -586,7 +592,7 @@ describe('Rendering', () => {
'access-control-allow-origin': '*',
'content-type': 'text/html; charset=utf-8',
'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT',
'x-surrogate-key': 'SCKEB3bkK0hFm4aS foo-id_metadata super-test--helix-pages--adobe_head SCKEB3bkK0hFm4aS_metadata JHEAK7b1XZvfOJpY',
'x-surrogate-key': 'SCKEB3bkK0hFm4aS foo-id_metadata super-test--helix-pages--adobe_head foo-id SCKEB3bkK0hFm4aS_metadata JHEAK7b1XZvfOJpY',
link: '</scripts/scripts.js>; rel=modulepreload; as=script; crossorigin=use-credentials',
});
});
Expand All @@ -609,7 +615,7 @@ describe('Rendering', () => {
'access-control-allow-origin': '*',
'content-type': 'text/html; charset=utf-8',
'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT',
'x-surrogate-key': 'SUhNxkR2spoxY489 foo-id_metadata super-test--helix-pages--adobe_head',
'x-surrogate-key': 'SUhNxkR2spoxY489 foo-id_metadata super-test--helix-pages--adobe_head foo-id',
link: '</scripts/scripts.js>; rel=modulepreload; as=script; crossorigin=use-credentials',
});
});
Expand All @@ -630,7 +636,7 @@ describe('Rendering', () => {
'access-control-allow-origin': '*',
'content-type': 'text/html; charset=utf-8',
'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT',
'x-surrogate-key': 'AkcHu8fRFT7HarTR foo-id_metadata super-test--helix-pages--adobe_head AkcHu8fRFT7HarTR_metadata z8NGXvKB0X5Fzcnd',
'x-surrogate-key': 'AkcHu8fRFT7HarTR foo-id_metadata super-test--helix-pages--adobe_head foo-id AkcHu8fRFT7HarTR_metadata z8NGXvKB0X5Fzcnd',
link: '</scripts/scripts.js>; rel=modulepreload; as=script; crossorigin=use-credentials',
});
});
Expand All @@ -649,7 +655,7 @@ describe('Rendering', () => {
assert.match(body, /<link rel="canonical" href="https:\/\/helix-pipeline\.com\/blog\/">/);
assert.deepStrictEqual(Object.fromEntries(headers.entries()), {
'content-type': 'text/html; charset=utf-8',
'x-surrogate-key': 'o_fNQBWBLWTIfYqV foo-id_metadata super-test--helix-pages--adobe_head',
'x-surrogate-key': 'o_fNQBWBLWTIfYqV foo-id_metadata super-test--helix-pages--adobe_head foo-id',
'last-modified': 'Wed, 12 Jan 2022 11:33:01 GMT',
});
});
Expand All @@ -662,7 +668,7 @@ describe('Rendering', () => {
'access-control-allow-origin': '*',
'content-type': 'text/html; charset=utf-8',
'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT',
'x-surrogate-key': 'VmeAc3K7QsCRzj5Z foo-id_metadata super-test--helix-pages--adobe_head',
'x-surrogate-key': 'VmeAc3K7QsCRzj5Z foo-id_metadata super-test--helix-pages--adobe_head foo-id',
link: '</scripts/scripts.js>; rel=modulepreload; as=script; crossorigin=use-credentials',
});
});
Expand Down
Loading