Cache static assets in service worker

Author: hkarani

next.config.js

@@ -1,6 +1,6 @@
 const { withPlausibleProxy } = require('next-plausible')
 const { InjectManifest } = require('workbox-webpack-plugin')
-const { generatePrecacheManifest } = require('./sw/build.js')
+const { generateDummyPrecacheManifest, addStaticAssetsInServiceWorker } = require('./sw/build.js')
 const webpack = require('webpack')
 
 let isProd = process.env.NODE_ENV === 'production'
@@ -214,7 +214,7 @@ module.exports = withPlausibleProxy()({
   },
   webpack: (config, { isServer, dev, defaultLoaders }) => {
     if (isServer) {
-      generatePrecacheManifest()
+      generateDummyPrecacheManifest()
       const workboxPlugin = new InjectManifest({
         // ignore the precached manifest which includes the webpack assets
         // since they are not useful to us
@@ -258,6 +258,25 @@ module.exports = withPlausibleProxy()({
 
       config.plugins.push(workboxPlugin)
     }
+    // Static assets cannot be added in server build they are emitted in the client build phase https://nextjs.org/docs/14/app/api-reference/next-config-js/webpack
+    // and are not needed in dev
+    // The service worker ONLY includes the assets in precache-manifest.json injected by Workbox plugin when the build starts
+    // but our static assets only exist later in the build.
+    // However the precache-manifest.json assets are hardcoded in an array in the generated service worker by Worbox in public/sw.js
+    // So we patch it to include our own custom assets
+
+    if (!dev) {
+      config.plugins.push({
+        apply: (compiler) => {
+          compiler.hooks.afterEmit.tapPromise(
+            'AddEmittedStaticAssetUrlsToServiceWorker',
+            async () => {
+              await addStaticAssetsInServiceWorker()
+            }
+          )
+        }
+      })
+    }
 
     config.module.rules.push(
       {

sw/build.js

@@ -1,5 +1,5 @@
 const { createHash } = require('crypto')
-const { readdirSync, readFileSync, statSync, writeFileSync } = require('fs')
+const { readdirSync, readFileSync, statSync, writeFileSync, existsSync } = require('fs')
 const { join } = require('path')
 
 const getRevision = filePath => createHash('md5').update(readFileSync(filePath)).digest('hex')
@@ -20,10 +20,50 @@ function formatBytes (bytes, decimals = 2) {
   return `${formattedSize} ${sizes[i]}`
 }
 
-function generatePrecacheManifest () {
+function escapeForSingleQuotedJsString (str) {
+  return str
+    .replace(/\\/g, '\\\\')
+    .replace(/'/g, "\\'")
+    .replace(/\r/g, '\\r')
+    .replace(/\n/g, '\\n')
+    .replace(/\$/g, '\\$')
+}
+
+function generateDummyPrecacheManifest () {
+  // A dummy manifest,to be easily referenced in the public/sw.js when patched
+  // This will be pathced with custom assets urls after Next builds
+  const manifest = [{
+    url: '/dummy/path/test1.js',
+    revision: 'rev-123'
+  }]
+
+  const output = join(__dirname, 'precache-manifest.json')
+  writeFileSync(output, JSON.stringify(manifest, null, 2))
+
+  console.log(`Created precache manifest at ${output}.`)
+}
+
+function patchSwAssetsURL (assetUrlsArray) {
+  const fullPath = join(__dirname, '../public/sw.js')
+  let content = readFileSync(fullPath, 'utf-8')
+  const patchedArray = JSON.stringify(assetUrlsArray)
+  const escapedPatchedArrayJson = escapeForSingleQuotedJsString(patchedArray)
+  const regex = /JSON\.parse\(\s*'\[\{"url":"\/dummy\/path\/test1\.js","revision":"rev-123"\}\]'\s*\)/
+  if (!regex.test(content)) {
+    console.warn('⚠️  No match found for precache manifest in sw.js')
+    return
+  }
+
+  content = content.replace(regex, () => {
+    return `JSON.parse('${escapedPatchedArrayJson}')`
+  })
+  writeFileSync(fullPath, content, 'utf-8')
+  console.log('✅ Patched service worker cached assets')
+}
+
+async function addStaticAssetsInServiceWorker () {
   const manifest = []
   let size = 0
-
   const addToManifest = (filePath, url, s) => {
     const revision = getRevision(filePath)
     manifest.push({ url, revision })
@@ -35,7 +75,9 @@ function generatePrecacheManifest () {
   const staticMatch = f => [/\.(gif|jpe?g|ico|png|ttf|woff|woff2)$/].some(m => m.test(f))
   staticFiles.filter(staticMatch).forEach(file => {
     const stats = statSync(file)
-    addToManifest(file, file.slice(staticDir.length), stats.size)
+    // Normalize path separators for URLs
+    const url = file.slice(staticDir.length).replace(/\\/g, '/')
+    addToManifest(file, url, stats.size)
   })
 
   const pagesDir = join(__dirname, '../pages')
@@ -45,17 +87,66 @@ function generatePrecacheManifest () {
   const pageMatch = f => precacheURLs.some(url => fileToUrl(f) === url)
   pagesFiles.filter(pageMatch).forEach(file => {
     const stats = statSync(file)
-    // This is not ideal since dependencies of the pages may have changed
-    // but we would still generate the same revision ...
-    // The ideal solution would be to create a revision from the file generated by webpack
-    // in .next/server/pages but the file may not exist yet when we run this script
     addToManifest(file, fileToUrl(file), stats.size)
   })
 
-  const output = 'sw/precache-manifest.json'
-  writeFileSync(output, JSON.stringify(manifest, null, 2))
+  const nextStaticDir = join(__dirname, '../.next/static')
+  // Wait until folder is emitted
+  console.log('⏳ Waiting for .next/static to be emitted...')
+  let folderRetries = 0
+  while (!existsSync(nextStaticDir) && folderRetries < 10) {
+    // eslint-disable-next-line no-await-in-loop
+    await new Promise(resolve => setTimeout(resolve, 500))
+    folderRetries++
+  }
+
+  if (!existsSync(nextStaticDir)) {
+    // Still write the manifest with whatever was collected from public/ and pages/
+    const output = join(__dirname, 'precache-manifest.json')
+    writeFileSync(output, JSON.stringify(manifest, null, 2))
+    console.warn(
+      `⚠️ .next/static not found. Created precache manifest at ${output} with only public/ and pages/ assets.`
+
+    )
+    return manifest
+  }
 
-  console.log(`Created precache manifest at ${output}. Cache will include ${manifest.length} URLs with a size of ${formatBytes(size)}.`)
+  function snapshot (files) {
+    return files.map(f => `${f}:${statSync(f).size}`).join(',')
+  }
+  // Now watch for stabilization (files are emitted asynchronously)
+  let lastSnapshot = ''
+  let stableCount = 0
+  const maxWaitMs = 60000
+  const startTime = Date.now()
+  while (stableCount < 3 && (Date.now() - startTime) < maxWaitMs) {
+    const files = walkSync(nextStaticDir)
+    const currentSnapshot = snapshot(files)
+    if (currentSnapshot === lastSnapshot) {
+      stableCount++
+    } else {
+      lastSnapshot = currentSnapshot
+      stableCount = 0
+    }
+    await new Promise(resolve => setTimeout(resolve, 500))
+  }
+  // finally generate manifest
+  const nextStaticFiles = walkSync(nextStaticDir)
+  nextStaticFiles.forEach(file => {
+    const stats = statSync(file)
+    // Normalize path separators for URLs
+    const url = `/_next/static${file.slice(nextStaticDir.length).replace(/\\/g, '/')}`
+    addToManifest(file, url, stats.size)
+  })
+  // write manifest
+  const output = join(__dirname, 'precache-manifest.json')
+  writeFileSync(output, JSON.stringify(manifest, null, 2))
+  console.log(
+    `✅ Created precache manifest at ${output}. Cache will include ${manifest.length} URLs with a size of ${formatBytes(size)}.`
+  )
+  const data = readFileSync(output, 'utf-8')
+  const manifestArray = JSON.parse(data)
+  patchSwAssetsURL(manifestArray)
 }
 
-module.exports = { generatePrecacheManifest }
+module.exports = { generateDummyPrecacheManifest, addStaticAssetsInServiceWorker }