fix: surrogate key for non-existing .html in code-bus

src/json-pipe.js

@@ -12,6 +12,7 @@
 import { cleanupHeaderValue, computeSurrogateKey } from '@adobe/helix-shared-utils';
 import initConfig from './steps/init-config.js';
 import setCustomResponseHeaders from './steps/set-custom-response-headers.js';
+import { computeContentPathKey, computeCodePathKey } from './steps/set-x-surrogate-key-header.js';
 import { PipelineResponse } from './PipelineResponse.js';
 import jsonFilter from './utils/json-filter.js';
 import { extractLastModified, recordLastModified, setLastModified } from './utils/last-modified.js';
@@ -42,7 +43,7 @@ export default function folderMapping(state) {
 
 async function fetchJsonContent(state, req, res) {
   const {
-    owner, repo, ref, contentBusId, partition, s3Loader, log, info,
+    owner, repo, ref, contentBusId, partition, s3Loader, log,
   } = state;
   const { path } = state.info;
   state.content.sourceBus = 'content';
@@ -65,11 +66,11 @@ async function fetchJsonContent(state, req, res) {
     if (state.content.sourceBus === 'content') {
       // provide either (prefixed) preview or (unprefixed) live content keys
       const contentKeyPrefix = partition === 'preview' ? 'p_' : '';
-      keys.push(`${contentKeyPrefix}${await computeSurrogateKey(`${contentBusId}${info.path}`)}`);
+      keys.push(`${contentKeyPrefix}${await computeContentPathKey(state)}`);
       keys.push(`${contentKeyPrefix}${contentBusId}`);
     } else {
       keys.push(`${ref}--${repo}--${owner}_code`);
-      keys.push(await computeSurrogateKey(`${ref}--${repo}--${owner}${info.path}`));
+      keys.push(await computeCodePathKey(state));
     }
     res.headers.set('x-surrogate-key', keys.join(' '));
     res.error = 'moved';
@@ -100,13 +101,13 @@ async function computeSurrogateKeys(state) {
     keys.push(await computeSurrogateKey(`${state.site}--${state.org}_config.json`));
   }
   if (state.content.sourceBus.includes('code')) {
-    keys.push(await computeSurrogateKey(`${state.ref}--${state.repo}--${state.owner}${state.info.path}`));
+    keys.push(await computeCodePathKey(state));
     keys.push(`${state.ref}--${state.repo}--${state.owner}_code`);
   }
   if (state.content.sourceBus.includes('content')) {
     // provide either (prefixed) preview or (unprefixed) live content keys
     const contentKeyPrefix = state.partition === 'preview' ? 'p_' : '';
-    keys.push(`${contentKeyPrefix}${await computeSurrogateKey(`${state.contentBusId}${state.info.path}`)}`);
+    keys.push(`${contentKeyPrefix}${await computeContentPathKey(state)}`);
     keys.push(`${contentKeyPrefix}${state.contentBusId}`);
   }
 

src/robots-pipe.js

@@ -11,6 +11,7 @@
  */
 import { cleanupHeaderValue, computeSurrogateKey } from '@adobe/helix-shared-utils';
 import renderCode from './steps/render-code.js';
+import { computeCodePathKey } from './steps/set-x-surrogate-key-header.js';
 import setCustomResponseHeaders from './steps/set-custom-response-headers.js';
 import { PipelineResponse } from './PipelineResponse.js';
 import initConfig from './steps/init-config.js';
@@ -119,9 +120,8 @@ function getForwardedHosts(req) {
 async function computeSurrogateKeys(state) {
   const keys = [];
 
-  const pathKey = `${state.ref}--${state.repo}--${state.owner}${state.info.path}`;
   keys.push(await computeSurrogateKey(`${state.site}--${state.org}_config.json`));
-  keys.push(await computeSurrogateKey(pathKey));
+  keys.push(await computeCodePathKey(state));
   return keys;
 }
 

src/steps/fetch-404.js

@@ -10,7 +10,7 @@
  * governing permissions and limitations under the License.
  */
 import { extractLastModified, recordLastModified } from '../utils/last-modified.js';
-import { getPathKey } from './set-x-surrogate-key-header.js';
+import { computeContentPathKey, computeCodePathKey } from './set-x-surrogate-key-header.js';
 
 /**
  * Loads the 404.html from code-bus and stores it in `res.body`
@@ -39,23 +39,24 @@ export default async function fetch404(state, req, res) {
   }
 
   // set 404 keys in any case
-  const pathKey = await getPathKey(state);
+  // always provide code and content keys since a resource could be added later to either bus
+  const keys = [];
+  // content keys
   // provide either (prefixed) preview or (unprefixed) live content keys
   const contentKeyPrefix = partition === 'preview' ? 'p_' : '';
-  const keys = [
-    `${contentKeyPrefix}${pathKey}`,
-    `${contentKeyPrefix}${contentBusId}`,
-    `${ref}--${repo}--${owner}_404`,
-    `${ref}--${repo}--${owner}_code`,
-  ];
-
+  keys.push(`${contentKeyPrefix}${await computeContentPathKey(state)}`);
+  keys.push(`${contentKeyPrefix}${contentBusId}`);
   if (state.info.unmappedPath) {
-    const unmappedPathKey = await getPathKey({
+    const unmappedPathKey = await computeContentPathKey({
       contentBusId,
       info: { path: state.info.unmappedPath },
     });
     keys.push(`${contentKeyPrefix}${unmappedPathKey}`);
   }
+  // code keys
+  keys.push(await computeCodePathKey(state));
+  keys.push(`${ref}--${repo}--${owner}_404`);
+  keys.push(`${ref}--${repo}--${owner}_code`);
 
   res.headers.set('x-surrogate-key', keys.join(' '));
 }

src/steps/fetch-content.js

@@ -9,7 +9,7 @@
  * OF ANY KIND, either express or implied. See the License for the specific language
  * governing permissions and limitations under the License.
  */
-import { computeSurrogateKey } from '@adobe/helix-shared-utils';
+import { computeContentPathKey, computeCodePathKey } from './set-x-surrogate-key-header.js';
 import { extractLastModified, recordLastModified } from '../utils/last-modified.js';
 
 /**
@@ -43,12 +43,12 @@ export default async function fetchContent(state, req, res) {
     res.headers.set('location', redirectLocation);
     const keys = [];
     if (isCode) {
-      keys.push(await computeSurrogateKey(`${ref}--${repo}--${owner}${info.path}`));
+      keys.push(await computeCodePathKey(state));
       keys.push(`${ref}--${repo}--${owner}_code`);
     } else {
       // provide either (prefixed) preview or (unprefixed) live content keys
       const contentKeyPrefix = partition === 'preview' ? 'p_' : '';
-      keys.push(`${contentKeyPrefix}${await computeSurrogateKey(`${contentBusId}${info.path}`)}`);
+      keys.push(`${contentKeyPrefix}${await computeContentPathKey(state)}`);
       keys.push(`${contentKeyPrefix}${contentBusId}`);
     }
     res.headers.set('x-surrogate-key', keys.join(' '));

src/steps/set-x-surrogate-key-header.js

@@ -12,11 +12,12 @@
 import { computeSurrogateKey } from '@adobe/helix-shared-utils';
 
 /**
- * Returns the surrogate key based on the contentBusId and the resource path
+ * Returns the surrogate key for a content-bus resource
+ * based on the contentBusId and the resource path
  * @param state
  * @returns {Promise<string>}
  */
-export async function getPathKey(state) {
+export async function computeContentPathKey(state) {
   const { contentBusId, info } = state;
   let { path } = info;
   // surrogate key for path
@@ -33,6 +34,19 @@ export async function getPathKey(state) {
   return computeSurrogateKey(`${contentBusId}${path}`);
 }
 
+/**
+ * Returns the surrogate key for a code-bus resource
+ * based on the repositry and the resource path
+ * @param state
+ * @returns {Promise<string>}
+ */
+export async function computeCodePathKey(state) {
+  const {
+    owner, repo, ref, info: { path },
+  } = state;
+  return computeSurrogateKey(`${ref}--${repo}--${owner}${path}`);
+}
+
 /**
  * @type PipelineStep
  * @param {PipelineState} state
@@ -50,9 +64,9 @@ export default async function setXSurrogateKeyHeader(state, req, res) {
   // provide either (prefixed) preview or (unprefixed) live content keys
   const contentKeyPrefix = partition === 'preview' ? 'p_' : '';
   const keys = [];
-  const hash = await getPathKey(state);
+  const hash = await computeContentPathKey(state);
   if (isCode) {
-    keys.push(await computeSurrogateKey(`${ref}--${repo}--${owner}${state.info.path}`));
+    keys.push(await computeCodePathKey(state));
     keys.push(`${ref}--${repo}--${owner}_code`);
   } else {
     keys.push(`${contentKeyPrefix}${hash}`);
@@ -64,7 +78,7 @@ export default async function setXSurrogateKeyHeader(state, req, res) {
   if (state.mapped) {
     keys.push(`${contentKeyPrefix}${hash}_metadata`);
     if (state.info.unmappedPath) {
-      keys.push(`${contentKeyPrefix}${await getPathKey({
+      keys.push(`${contentKeyPrefix}${await computeContentPathKey({
         contentBusId,
         info: { path: state.info.unmappedPath },
       })}`);

test/rendering.test.js

@@ -192,7 +192,7 @@
     }
     const response = await render(url, '', expStatus, partition);
     const actHtml = response.body;
     console.log(actHtml);
     if (expStatus === 200) {
       const $actMain = new JSDOM(actHtml).window.document.querySelector(domSelector);
       const $expMain = new JSDOM(expHtml).window.document.querySelector(domSelector);
@@ -531,7 +531,7 @@
       assert.deepStrictEqual(Object.fromEntries(headers.entries()), {
         'content-type': 'text/html; charset=utf-8',
         'last-modified': 'Mon, 12 Oct 2009 17:50:00 GMT',
-        'x-surrogate-key': 'OYsA_wfqip5EuBu6 foo-id super-test--helix-pages--adobe_404 super-test--helix-pages--adobe_code',
+        'x-surrogate-key': 'OYsA_wfqip5EuBu6 foo-id fRCqxCtsDrp5LOYW super-test--helix-pages--adobe_404 super-test--helix-pages--adobe_code',
         'x-error': 'failed to load /not-found-with-handler.md from content-bus: 404',
         'access-control-allow-origin': '*',
         link: '</scripts/scripts.js>; rel=modulepreload; as=script; crossorigin=use-credentials',
@@ -547,7 +547,7 @@
       assert.deepStrictEqual(Object.fromEntries(headers.entries()), {
         'content-type': 'text/html; charset=utf-8',
         'last-modified': 'Mon, 12 Oct 2009 17:50:00 GMT',
-        'x-surrogate-key': 'OYsA_wfqip5EuBu6 foo-id super-test--helix-pages--adobe_404 super-test--helix-pages--adobe_code',
+        'x-surrogate-key': 'OYsA_wfqip5EuBu6 foo-id fpt80Bs0_DS5RDD4 super-test--helix-pages--adobe_404 super-test--helix-pages--adobe_code',
         'x-error': 'failed to load /not-found-with-handler.md from content-bus: 404',
         'access-control-allow-origin': '*',
       });
@@ -563,7 +563,7 @@
         'content-type': 'text/html; charset=utf-8',
         'last-modified': 'Mon, 12 Oct 2009 17:50:00 GMT',
         'x-error': 'failed to load /not-found-with-handler.html from code-bus: 404',
-        'x-surrogate-key': 'ta3V7wR3zlRh1b0E foo-id super-test--helix-pages--adobe_404 super-test--helix-pages--adobe_code',
+        'x-surrogate-key': 'ta3V7wR3zlRh1b0E foo-id 9q9qs7DEYGc4lYTJ super-test--helix-pages--adobe_404 super-test--helix-pages--adobe_code',
         link: '</scripts/scripts.js>; rel=modulepreload; as=script; crossorigin=use-credentials',
         'access-control-allow-origin': '*',
       });
@@ -579,7 +579,7 @@
         'last-modified': 'Fri, 30 Apr 2021 03:47:18 GMT',
         link: '</scripts/scripts.js>; rel=modulepreload; as=script; crossorigin=use-credentials',
         'x-error': 'request to /index.md not allowed (no-index).',
-        'x-surrogate-key': 'FzT3jXtDSYMYOTq1 foo-id super-test--helix-pages--adobe_404 super-test--helix-pages--adobe_code',
+        'x-surrogate-key': 'FzT3jXtDSYMYOTq1 foo-id 03oC8MtxiF9EGbkp super-test--helix-pages--adobe_404 super-test--helix-pages--adobe_code',
       });
       assert.strictEqual(body.trim(), '');
     });
@@ -591,7 +591,7 @@
         'content-type': 'text/html; charset=utf-8',
         link: '</scripts/scripts.js>; rel=modulepreload; as=script; crossorigin=use-credentials',
         'x-error': 'failed to load /not-a-page.md from content-bus: 404',
-        'x-surrogate-key': 'gPHXKWdMY_R8KV2Z foo-id super-test--helix-pages--adobe_404 super-test--helix-pages--adobe_code QJqsV4atnOA47sHc',
+        'x-surrogate-key': 'gPHXKWdMY_R8KV2Z foo-id QJqsV4atnOA47sHc nzU8FWOQ2xQBWJo_ super-test--helix-pages--adobe_404 super-test--helix-pages--adobe_code',
       });
       assert.strictEqual(body.trim(), '');
       // preview (code coverage)
@@ -601,7 +601,7 @@
         'content-type': 'text/html; charset=utf-8',
         link: '</scripts/scripts.js>; rel=modulepreload; as=script; crossorigin=use-credentials',
         'x-error': 'failed to load /not-a-page.md from content-bus: 404',
-        'x-surrogate-key': 'p_gPHXKWdMY_R8KV2Z p_foo-id super-test--helix-pages--adobe_404 super-test--helix-pages--adobe_code p_QJqsV4atnOA47sHc',
+        'x-surrogate-key': 'p_gPHXKWdMY_R8KV2Z p_foo-id p_QJqsV4atnOA47sHc nzU8FWOQ2xQBWJo_ super-test--helix-pages--adobe_404 super-test--helix-pages--adobe_code',
       });
       assert.strictEqual(resp.body.trim(), '');
     });
@@ -623,6 +623,18 @@
       assert.strictEqual(ret.headers.get('location'), '/foo.plain.html');
     });
 
+    it('returns identical surrogate key for redirected /one-section and /one-section.plain.html', async () => {
+      loader.headers('one-section.md', 'x-amz-meta-redirect-location', '/foo');
+      let resp = await render(new URL('https://localhost/one-section'), '', 301);
+      assert.strictEqual(resp.headers.get('location'), '/foo');
+      const key1 = resp.headers.get('x-surrogate-key');
+
+      resp = await render(new URL('https://localhost/one-section'), '.plain', 301);
+      assert.strictEqual(resp.headers.get('location'), '/foo.plain.html');
+      const key2 = resp.headers.get('x-surrogate-key');
+      assert.strictEqual(key1, key2);
+    });
+
     it('renders redirect for static html (content)', async () => {
       loader.headers('static-content.html', 'x-amz-meta-redirect-location', '/foo');
       const ret = await render(new URL('https://localhost/static-content.html'), '', 301);