Implement LRU strategy #6

Author: julianrubisch

index.js

@@ -4,6 +4,8 @@ const sprintf = require("sprintf-js").sprintf;
 const fs = require("fs");
 const path = require("path");
 
+const getFolderSize = require("get-folder-size");
+
 function makeDirIfNotExists(dir) {
   if (!fs.existsSync(dir)) {
     fs.mkdirSync(dir);
@@ -63,13 +65,67 @@ function touch(path) {
   }
 }
 
+function evictLeastRecentlyUsed(cacheDir, maxSize, logger) {
+  getFolderSize(cacheDir, (err, size) => {
+    if (err) {
+      throw err;
+    }
+
+    if (size >= maxSize) {
+      try {
+        // find least recently used file
+        const leastRecentlyUsed = findLeastRecentlyUsed(cacheDir);
+
+        // and delete it
+        const { dir } = path.parse(leastRecentlyUsed.path);
+        fs.unlinkSync(leastRecentlyUsed.path);
+        fs.rmdirSync(dir);
+
+        if (logger) {
+          logger.info(`Evicted ${leastRecentlyUsed.path} from cache`);
+        }
+
+        evictLeastRecentlyUsed(cacheDir, maxSize);
+      } catch (e) {
+        logger.error(e);
+      }
+    }
+  });
+}
+
+function findLeastRecentlyUsed(dir, result) {
+  let files = fs.readdirSync(dir);
+  result = result || { atime: Date.now(), path: "" };
+
+  files.forEach(file => {
+    const newBase = path.join(dir, file);
+
+    if (fs.statSync(newBase).isDirectory()) {
+      result = findLeastRecentlyUsed(newBase, result);
+    } else {
+      const { atime } = fs.statSync(newBase);
+
+      if (atime < result.atime) {
+        result = {
+          atime,
+          path: newBase
+        };
+      }
+    }
+  });
+
+  return result;
+}
+
 const middleWare = (module.exports = function(options) {
   return async function(req, res, next) {
     options = options || {};
     options.cacheDir =
       options && options.cacheDir
         ? options.cacheDir
         : path.join(process.cwd(), "/tmp");
+    options.maxSize =
+      options && options.maxSize ? options.maxSize : 1024 * 1024 * 1024;
 
     const {
       dir1,
@@ -88,7 +144,7 @@ const middleWare = (module.exports = function(options) {
         const firstFile = fs.readdirSync(assetCachePath)[0];
 
         // touch file for LRU eviction
-        touch(`${assetCachePath}/${firstFile}`);
+        middleWare.touch(`${assetCachePath}/${firstFile}`);
 
         const [contentType, contentLength] = middleWare.decodeAssetCacheName(
           firstFile
@@ -124,6 +180,12 @@ const middleWare = (module.exports = function(options) {
         res.locals.contentType = blob.type;
         res.locals.contentLength = blob.size;
 
+        middleWare.evictLeastRecentlyUsed(
+          options.cacheDir,
+          options.maxSize,
+          options.logger
+        );
+
         fs.writeFileSync(`${assetCachePath}/${fileName}`, res.locals.buffer);
 
         const [seconds, nanoSeconds] = process.hrtime(startTime);
@@ -137,7 +199,6 @@ const middleWare = (module.exports = function(options) {
 
       next();
     } catch (e) {
-      console.log(e);
       // in case fs.writeFileSync writes partial data and fails
       if (fs.existsSync(assetCachePath)) {
         fs.unlinkSync(assetCachePath);
@@ -157,3 +218,6 @@ middleWare.makeAssetCachePath = makeAssetCachePath;
 middleWare.makeDirIfNotExists = makeDirIfNotExists;
 middleWare.encodeAssetCacheName = encodeAssetCacheName;
 middleWare.decodeAssetCacheName = decodeAssetCacheName;
+middleWare.findLeastRecentlyUsed = findLeastRecentlyUsed;
+middleWare.evictLeastRecentlyUsed = evictLeastRecentlyUsed;
+middleWare.touch = touch;

package-lock.json

@@ -22,6 +22,7 @@
   },
   "homepage": "https://github.com/julianrubisch/express-asset-file-cache-middleware#readme",
   "dependencies": {
+    "get-folder-size": "^2.0.1",
     "node-fetch": "^2.6.0",
     "sprintf-js": "^1.1.2"
   },

package.json

@@ -30,6 +30,7 @@ describe("Middleware", function() {
           arrayBuffer: sinon.stub().resolves([])
         })
       });
+      sinon.stub(middleware, "evictLeastRecentlyUsed");
       this.makePathSpy = sinon
         .stub(middleware, "makeAssetCachePath")
         .returns({ dir1: "a1", dir2: "b2", path: "./a1/b2/0123456789abcdef" });
@@ -69,6 +70,8 @@ describe("Middleware", function() {
         .stub(fs, "readdirSync")
         .withArgs("./a1/b2/0123456789abcdef")
         .returns(["dW5kZWZpbmVkOnVuZGVmaW5lZA=="]);
+      sinon.stub(middleware, "touch");
+
       const mw = middleware({ cacheDir: "." });
 
       await mw(
@@ -83,6 +86,8 @@ describe("Middleware", function() {
           "./a1/b2/0123456789abcdef/dW5kZWZpbmVkOnVuZGVmaW5lZA=="
         )
         .and.returned(Buffer.from([]));
+
+      fs.readdirSync.restore();
     });
 
     // it falls back to a default cache key
@@ -132,6 +137,11 @@ describe("Middleware", function() {
       this.nextSpy.resetHistory();
       this.makePathSpy.resetHistory();
     });
+
+    after(function() {
+      path.join.restore();
+      middleware.evictLeastRecentlyUsed.restore();
+    });
   });
 
   describe("asset cache name en/decoding", function() {
@@ -147,4 +157,80 @@ describe("Middleware", function() {
       ).to.deep.equal(["image/png", "4096"]);
     });
   });
+
+  describe("evicting of least recently used files", function() {
+    before(function() {
+      sinon
+        .stub(fs, "readdirSync")
+        .withArgs("/tmp")
+        .returns(["test1", "test2"])
+        .withArgs("/tmp/test1")
+        .returns(["foo"])
+        .withArgs("/tmp/test2")
+        .returns(["bar"]);
+      sinon
+        .stub(fs, "statSync")
+        .withArgs("/tmp/test1")
+        .returns({
+          isDirectory() {
+            return true;
+          }
+        })
+        .withArgs("/tmp/test2")
+        .returns({
+          isDirectory() {
+            return true;
+          }
+        })
+        .withArgs("/tmp/test1")
+        .returns({
+          isDirectory() {
+            return true;
+          }
+        })
+        .withArgs("/tmp/test2")
+        .returns({
+          isDirectory() {
+            return true;
+          }
+        })
+        .withArgs("/tmp/test1/foo")
+        .returns({
+          isDirectory() {
+            return false;
+          },
+          atime: 1000
+        })
+        .withArgs("/tmp/test2/bar")
+        .returns({
+          isDirectory() {
+            return false;
+          },
+          atime: 2000
+        });
+      sinon
+        .stub(path, "join")
+        .withArgs("/tmp", "test1")
+        .returns("/tmp/test1")
+        .withArgs("/tmp/test1", "foo")
+        .returns("/tmp/test1/foo")
+        .withArgs("/tmp", "test2")
+        .returns("/tmp/test2")
+        .withArgs("/tmp/test2", "bar")
+        .returns("/tmp/test2/bar");
+    });
+
+    it("returns the least recently used file", function() {
+      expect(middleware.findLeastRecentlyUsed("/tmp")).to.deep.equal({
+        atime: 1000,
+        path: "/tmp/test1/foo"
+      });
+    });
+
+    after(function() {
+      fs.readdirSync.restore();
+      fs.statSync.restore();
+      path.join.restore();
+    });
+  });
 });