chore: page meta last modified 5.x

src/PipelineResponse.js

@@ -30,7 +30,7 @@ export class PipelineResponse {
       document: undefined,
       headers,
       error: undefined,
-      lastModifiedTime: 0,
+      lastModifiedSources: {},
     });
   }
 

src/html-pipe.js

@@ -38,6 +38,7 @@ import { PipelineResponse } from './PipelineResponse.js';
 import { validatePathInfo } from './utils/path.js';
 import { initAuthRoute } from './utils/auth.js';
 import fetchMappedMetadata from './steps/fetch-mapped-metadata.js';
+import { applyMetaLastModified, setLastModified } from './utils/last-modified.js';
 
 /**
  * Fetches the content and if not found, fetches the 404.html
@@ -162,6 +163,7 @@ export async function htmlPipe(state, req) {
       log[level](`pipeline status: ${res.status} ${res.error}`);
       res.headers.set('x-error', cleanupHeaderValue(res.error));
       if (res.status < 500) {
+        setLastModified(state, res);
         await setCustomResponseHeaders(state, req, res);
       }
       return res;
@@ -188,8 +190,10 @@ export async function htmlPipe(state, req) {
       await render(state, req, res);
       state.timer?.update('serialize');
       await tohtml(state, req, res);
+      await applyMetaLastModified(state, res);
     }
 
+    setLastModified(state, res);
     await setCustomResponseHeaders(state, req, res);
     await setXSurrogateKeyHeader(state, req, res);
   } catch (e) {

src/json-pipe.js

@@ -14,7 +14,7 @@ import fetchConfigAll from './steps/fetch-config-all.js';
 import setCustomResponseHeaders from './steps/set-custom-response-headers.js';
 import { PipelineResponse } from './PipelineResponse.js';
 import jsonFilter from './utils/json-filter.js';
-import { extractLastModified, updateLastModified } from './utils/last-modified.js';
+import { extractLastModified, recordLastModified, setLastModified } from './utils/last-modified.js';
 import { authenticate } from './steps/authenticate.js';
 import fetchConfig from './steps/fetch-config.js';
 import { getPathInfo } from './utils/path.js';
@@ -85,7 +85,7 @@ async function fetchJsonContent(state, req, res) {
     state.content.sourceLocation = ret.headers.get('x-amz-meta-x-source-location');
     log.info(`source-location: ${state.content.sourceLocation}`);
 
-    updateLastModified(state, res, extractLastModified(ret.headers));
+    recordLastModified(state, res, 'content', extractLastModified(ret.headers));
   } else {
     // also add code surrogate key in case json is later added to code bus (#688)
     state.content.sourceBus = 'code|content';
@@ -195,6 +195,7 @@ export async function jsonPipe(state, req) {
     const keys = await computeSurrogateKeys(state);
     res.headers.set('x-surrogate-key', keys.join(' '));
 
+    setLastModified(state, res);
     await setCustomResponseHeaders(state, req, res);
     return res;
   } catch (e) {

src/sitemap-pipe.js

@@ -20,6 +20,7 @@ import setXSurrogateKeyHeader from './steps/set-x-surrogate-key-header.js';
 import setCustomResponseHeaders from './steps/set-custom-response-headers.js';
 import { PipelineStatusError } from './PipelineStatusError.js';
 import { PipelineResponse } from './PipelineResponse.js';
+import { extractLastModified, recordLastModified, setLastModified } from './utils/last-modified.js';
 
 async function generateSitemap(state) {
   const {
@@ -118,6 +119,7 @@ export async function sitemapPipe(state, req) {
       const ret = await generateSitemap(state);
       if (ret.status === 200) {
         res.status = 200;
+        recordLastModified(state, res, 'content', extractLastModified(ret.headers));
         delete res.error;
         state.content.data = ret.body;
       }
@@ -129,6 +131,7 @@ export async function sitemapPipe(state, req) {
 
     state.timer?.update('serialize');
     await renderCode(state, req, res);
+    setLastModified(state, res);
     await setCustomResponseHeaders(state, req, res);
     await setXSurrogateKeyHeader(state, req, res);
   } catch (e) {

src/steps/fetch-404.js

@@ -9,7 +9,7 @@
  * OF ANY KIND, either express or implied. See the License for the specific language
  * governing permissions and limitations under the License.
  */
-import { extractLastModified } from '../utils/last-modified.js';
+import { extractLastModified, recordLastModified } from '../utils/last-modified.js';
 import { getPathKey } from './set-x-surrogate-key-header.js';
 
 /**
@@ -29,7 +29,7 @@ export default async function fetch404(state, req, res) {
     // override last-modified if source-last-modified is set
     const lastModified = extractLastModified(ret.headers);
     if (lastModified) {
-      ret.headers.set('last-modified', lastModified);
+      recordLastModified(state, res, 'content', lastModified);
     }
 
     // keep 404 response status

src/steps/fetch-config-all.js

@@ -11,7 +11,7 @@
  */
 
 import { PipelineStatusError } from '../PipelineStatusError.js';
-import { extractLastModified, updateLastModified } from '../utils/last-modified.js';
+import { extractLastModified, recordLastModified } from '../utils/last-modified.js';
 import { globToRegExp, Modifiers } from '../utils/modifiers.js';
 import { getOriginalHost } from './utils.js';
 
@@ -74,7 +74,7 @@ export default async function fetchConfigAll(state, req, res) {
 
     if (state.type === 'html' && state.info.selector !== 'plain') {
       // also update last-modified (only for extensionless html pipeline)
-      updateLastModified(state, res, extractLastModified(ret.headers));
+      recordLastModified(state, res, 'configAll', extractLastModified(ret.headers));
     }
     // set custom preview and live hosts
     state.previewHost = replaceParams(state.config.cdn?.preview?.host, state);

src/steps/fetch-config.js

@@ -9,7 +9,7 @@
  * OF ANY KIND, either express or implied. See the License for the specific language
  * governing permissions and limitations under the License.
  */
-import { extractLastModified, updateLastModified } from '../utils/last-modified.js';
+import { extractLastModified, recordLastModified } from '../utils/last-modified.js';
 import { PipelineStatusError } from '../PipelineStatusError.js';
 
 /**
@@ -67,11 +67,11 @@ export default async function fetchConfig(state, req, res) {
     const configLastModified = extractLastModified(ret.headers);
 
     // update last modified of fstab
-    updateLastModified(state, res, config.fstab?.lastModified || configLastModified);
+    recordLastModified(state, res, 'config', config.fstab?.lastModified || configLastModified);
 
     // for html requests, also consider the HEAD config
     if (state.type === 'html' && state.info.selector !== 'plain' && config.head?.lastModified) {
-      updateLastModified(state, res, config.head.lastModified);
+      recordLastModified(state, res, 'head', config.head.lastModified);
     }
   }
 

src/steps/fetch-content.js

@@ -10,7 +10,7 @@
  * governing permissions and limitations under the License.
  */
 import { computeSurrogateKey } from '@adobe/helix-shared-utils';
-import { extractLastModified, updateLastModified } from '../utils/last-modified.js';
+import { extractLastModified, recordLastModified } from '../utils/last-modified.js';
 
 /**
  * Loads the content from either the content-bus or code-bus and stores it in `state.content`
@@ -67,7 +67,7 @@ export default async function fetchContent(state, req, res) {
     state.content.sourceLocation = ret.headers.get('x-amz-meta-x-source-location');
     log.info(`source-location: ${state.content.sourceLocation}`);
 
-    updateLastModified(state, res, extractLastModified(ret.headers));
+    recordLastModified(state, res, 'content', extractLastModified(ret.headers));
 
     // reject requests to /index *after* checking for redirects
     // (https://github.com/adobe/helix-pipeline-service/issues/290)

src/utils/last-modified.js

@@ -11,30 +11,43 @@
  */
 
 /**
- * Updates the context.content.lastModified if the time in `timeString` is newer than the existing
- * one if none exists yet. please note that it generates helper property `lastModifiedTime` in
- * unix epoch format.
- *
- * the date string will be a "http-date": https://httpwg.org/specs/rfc7231.html#http.date
+ * Records the last modified for the given source.
  *
  * @param {PipelineState} state
- * @param {PipelineResponse} res the pipeline context
+ * @param {PipelineResponse} res the pipeline response
+ * @param {string} source the source providing a last-modified date
  * @param {string} httpDate http-date string
  */
-export function updateLastModified(state, res, httpDate) {
+export function recordLastModified(state, res, source, httpDate) {
   if (!httpDate) {
     return;
   }
   const { log } = state;
-  const time = new Date(httpDate).getTime();
-  if (Number.isNaN(time)) {
-    log.warn(`updateLastModified date is invalid: ${httpDate}`);
+  const date = new Date(httpDate);
+  if (Number.isNaN(date.valueOf())) {
+    log.warn(`last-modified date is invalid: ${httpDate} for ${source}`);
     return;
   }
+  res.lastModifiedSources[source] = {
+    time: date.valueOf(),
+    date: date.toUTCString(),
+  };
+}
 
-  if (time > (res.lastModifiedTime ?? 0)) {
-    res.lastModifiedTime = time;
-    res.headers.set('last-modified', httpDate);
+/**
+ * Calculates the last modified by using the latest date from all the recorded sources
+ * and sets it on the `last-modified` header.
+ *
+ * @param {PipelineState} state
+ * @param {PipelineResponse} res the pipeline response
+ */
+export function setLastModified(state, res) {
+  let latestTime = 0;
+  for (const { time, date } of Object.values(res.lastModifiedSources)) {
+    if (time > latestTime) {
+      latestTime = time;
+      res.headers.set('last-modified', date);
+    }
   }
 }
 
@@ -55,3 +68,13 @@ export function extractLastModified(headers) {
   }
   return headers.get('last-modified');
 }
+
+/**
+ * Sets the metadata last modified entry to the one define in the page specific metadata if
+ * it exists. this allows to control the last-modified per metadata record.
+ * @param {PipelineState} state
+ * @param {PipelineResponse} res the pipeline response
+ */
+export function applyMetaLastModified(state, res) {
+  recordLastModified(state, res, 'metadata', state.content.meta.page['last-modified']);
+}

test/fixtures/content/generic-product/metadata.json

@@ -60,6 +60,10 @@
         "Keywords": "Exactomento Mapped Folder",
         "og:publisher": "Adobe",
         "Short Title": "E"
+      },
+      {
+        "URL": "/products/product2",
+        "last-modified": "2024-12-25T03:33:33Z"
       }
     ]
   }

test/json-pipe.test.js

@@ -108,7 +108,7 @@ describe('JSON Pipe Test', () => {
             headers: {
               'content-type': 'application/json',
               'x-amz-meta-x-source-location': 'foo-bar',
-              'last-modified': 'Wed, 12 Oct 2009 17:50:00 GMT',
+              'last-modified': 'Mon, 12 Oct 2009 17:50:00 GMT',
             },
           }),
         )
@@ -144,7 +144,7 @@ describe('JSON Pipe Test', () => {
     assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), {
       'content-type': 'application/json',
       'x-surrogate-key': 'Atrz_qDg26DmSe9a foobar',
-      'last-modified': 'Wed, 12 Oct 2009 17:50:00 GMT',
+      'last-modified': 'Mon, 12 Oct 2009 17:50:00 GMT',
     });
   });
 
@@ -168,7 +168,7 @@ describe('JSON Pipe Test', () => {
     assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), {
       'content-type': 'application/json',
       'x-surrogate-key': 'Atrz_qDg26DmSe9a foobar',
-      'last-modified': 'Wed, 12 Oct 2009 17:50:00 GMT',
+      'last-modified': 'Mon, 12 Oct 2009 17:50:00 GMT',
     });
   });
 
@@ -206,7 +206,7 @@ describe('JSON Pipe Test', () => {
     assert.deepStrictEqual(headers, {
       'access-control-allow-origin': '*',
       'content-security-policy': 'default-src \'self\'',
-      'last-modified': 'Wed, 12 Oct 2009 17:50:00 GMT',
+      'last-modified': 'Mon, 12 Oct 2009 17:50:00 GMT',
       'x-surrogate-key': 'Atrz_qDg26DmSe9a foobar',
       'content-type': 'application/json',
     });
@@ -311,7 +311,7 @@ describe('JSON Pipe Test', () => {
     });
     const headers = Object.fromEntries(resp.headers.entries());
     assert.deepStrictEqual(headers, {
-      'last-modified': 'Wed, 12 Oct 2009 17:50:00 GMT',
+      'last-modified': 'Mon, 12 Oct 2009 17:50:00 GMT',
       'x-surrogate-key': 'Atrz_qDg26DmSe9a foobar',
       'content-type': 'application/json',
     });
@@ -326,7 +326,7 @@ describe('JSON Pipe Test', () => {
         headers: {
           'content-type': 'application/json',
           'x-amz-meta-x-source-location': 'foo-bar',
-          'last-modified': 'Wed, 12 Oct 2009 17:50:00 GMT',
+          'last-modified': 'Mon, 12 Oct 2009 17:50:00 GMT',
           'x-amz-meta-x-source-last-modified': 'Wed, 12 Oct 2009 15:50:00 GMT',
         },
       }),
@@ -343,7 +343,7 @@ describe('JSON Pipe Test', () => {
     });
     const headers = Object.fromEntries(resp.headers.entries());
     assert.deepStrictEqual(headers, {
-      'last-modified': 'Wed, 12 Oct 2009 15:50:00 GMT',
+      'last-modified': 'Mon, 12 Oct 2009 15:50:00 GMT',
       'x-surrogate-key': 'Atrz_qDg26DmSe9a foobar',
       'content-type': 'application/json',
     });
@@ -364,7 +364,7 @@ describe('JSON Pipe Test', () => {
             headers: {
               'content-type': 'application/json',
               'x-amz-meta-x-source-location': 'foo-bar',
-              'last-modified': 'Wed, 12 Oct 2009 17:50:00 GMT',
+              'last-modified': 'Mon, 12 Oct 2009 17:50:00 GMT',
             },
           }),
         )
@@ -385,7 +385,7 @@ describe('JSON Pipe Test', () => {
     });
     assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), {
       'content-type': 'application/json',
-      'last-modified': 'Wed, 12 Oct 2009 17:50:00 GMT',
+      'last-modified': 'Mon, 12 Oct 2009 17:50:00 GMT',
       'x-surrogate-key': 'SIMSxecp2CJXqGYs ref--repo--owner_code',
     });
   });
@@ -408,7 +408,7 @@ describe('JSON Pipe Test', () => {
             headers: {
               'content-type': 'application/json',
               'x-amz-meta-x-source-location': 'foo-bar',
-              'last-modified': 'Wed, 12 Oct 2009 17:50:00 GMT',
+              'last-modified': 'Mon, 12 Oct 2009 17:50:00 GMT',
             },
           }),
         )
@@ -426,7 +426,7 @@ describe('JSON Pipe Test', () => {
     });
     assert.deepStrictEqual(Object.fromEntries(resp.headers.entries()), {
       'content-type': 'application/json',
-      'last-modified': 'Wed, 12 Oct 2009 17:50:00 GMT',
+      'last-modified': 'Mon, 12 Oct 2009 17:50:00 GMT',
       'x-surrogate-key': 'SIMSxecp2CJXqGYs ref--repo--owner_code',
     });
   });
@@ -529,7 +529,7 @@ describe('JSON Pipe Test', () => {
             headers: {
               'content-type': 'application/json',
               'x-amz-meta-x-source-location': 'foo-bar',
-              'last-modified': 'Wed, 12 Oct 2009 17:50:00 GMT',
+              'last-modified': 'Mon, 12 Oct 2009 17:50:00 GMT',
             },
           }),
         )
@@ -624,7 +624,7 @@ describe('JSON Pipe Test', () => {
             headers: {
               'content-type': 'application/json',
               'x-amz-meta-x-source-location': 'foo-bar',
-              'last-modified': 'Wed, 12 Oct 2009 17:50:00 GMT',
+              'last-modified': 'Mon, 12 Oct 2009 17:50:00 GMT',
             },
           }),
         )
@@ -648,7 +648,7 @@ describe('JSON Pipe Test', () => {
         headers: {
           'content-type': 'application/json',
           'x-amz-meta-x-source-location': 'foo-bar',
-          'last-modified': 'Wed, 12 Oct 2009 17:50:00 GMT',
+          'last-modified': 'Mon, 12 Oct 2009 17:50:00 GMT',
         },
       }),
     );
@@ -683,7 +683,7 @@ describe('JSON Pipe Test', () => {
         headers: {
           'content-type': 'application/json',
           'x-amz-meta-x-source-location': 'foo-bar',
-          'last-modified': 'Wed, 12 Oct 2009 17:50:00 GMT',
+          'last-modified': 'Mon, 12 Oct 2009 17:50:00 GMT',
         },
       }),
     );