Skip to content

Commit b0915d5

Browse files
memallocmemalloc
authored
Merge pull request #8 from julianrubisch/6-lru-eviction-strategy
6 lru eviction strategy
2 parents bf875ad + 4a0394f commit b0915d5

File tree

5 files changed

+309
-118
lines changed

5 files changed

+309
-118
lines changed

‎README.md

Lines changed: 125 additions & 116 deletions
Original file line numberDiff line numberDiff line change
@@ -1,123 +1,132 @@
1-
# express-asset-file-cache-middleware
1+
# express-asset-file-cache-middleware
22
<!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
33
[![All Contributors](https://img.shields.io/badge/all_contributors-2-orange.svg?style=flat-square)](#contributors-)
44
<!-- ALL-CONTRIBUTORS-BADGE:END -->
5-
6-
![Build Status](https://github.com/julianrubisch/express-asset-file-cache-middleware/workflows/Node%20CI/badge.svg)
7-
8-
A modest express.js middleware to locally cache assets (images, videos, audio, etc.) for faster access and proxying, for use in e.g. Electron apps.
9-
10-
## TL;DR
11-
12-
For offline use of dynamic assets, e.g. in your Electron app or local express server.
13-
14-
## Usage
15-
16-
```javascript
17-
const express = require("express");
18-
const fileCacheMiddleware = require("express-asset-file-cache-middleware");
19-
20-
const app = express();
21-
22-
app.get(
23-
"/assets/:asset_id",
24-
async (req, res, next) => {
25-
res.locals.fetchUrl = `https://cdn.example.org/path/to/actual/asset/${req.params.asset_id}`;
26-
27-
res.locals.cacheKey = `${someExpirableUniqueKey}`;
28-
next();
29-
},
30-
fileCacheMiddleware({ cacheDir: "/tmp" }),
31-
(req, res) => {
32-
res.set({
33-
"Content-Type": res.locals.contentType,
34-
"Content-Length": res.locals.contentLength
35-
});
36-
res.end(res.locals.buffer, "binary");
37-
}
38-
);
39-
40-
app.listen(3000);
41-
```
42-
43-
It works by fetching your asset in between two callbacks on e.g. a route, by attaching a `fetchUrl` onto `res.locals`. When the asset isn't cached on disk already, it will write it into a directory specified by the option `cacheDir`. If it finds a file that's alread there, it will use that.
44-
5+
6+
![Build Status](https://github.com/julianrubisch/express-asset-file-cache-middleware/workflows/Node%20CI/badge.svg)
7+
8+
A modest express.js middleware to locally cache assets (images, videos, audio, etc.) for faster access and proxying, for use in e.g. Electron apps.
9+
10+
## TL;DR
11+
12+
For offline use of dynamic assets, e.g. in your Electron app or local express server.
13+
14+
## Usage
15+
16+
```javascript
17+
const express = require("express");
18+
const fileCacheMiddleware = require("express-asset-file-cache-middleware");
19+
20+
const app = express();
21+
22+
app.get(
23+
"/assets/:asset_id",
24+
async (req, res, next) => {
25+
res.locals.fetchUrl = `https://cdn.example.org/path/to/actual/asset/${req.params.asset_id}`;
26+
27+
res.locals.cacheKey = `${someExpirableUniqueKey}`;
28+
next();
29+
},
30+
fileCacheMiddleware({ cacheDir: "/tmp", maxSize: 10 * 1024 * 1024 * 1024 }),
31+
(req, res) => {
32+
res.set({
33+
"Content-Type": res.locals.contentType,
34+
"Content-Length": res.locals.contentLength
35+
});
36+
res.end(res.locals.buffer, "binary");
37+
}
38+
);
39+
40+
app.listen(3000);
41+
```
42+
43+
It works by fetching your asset in between two callbacks on e.g. a route, by attaching a `fetchUrl` onto `res.locals`. When the asset isn't cached on disk already, it will write it into a directory specified by the option `cacheDir`. If it finds a file that's alread there, it will use that.
44+
4545
The asset's `contentType` and `contentLength` are stored base64 encoded in the filename, thus no offline database is necessary
46-
47-
Note that setting `cacheKey` and `cacheDir` isn't strictly necessary, it will fall back to `res.local.fetchUrl` and `path.join(process.cwd(), "/tmp")`, respectively.
48-
49-
## Install
50-
51-
$ npm install express-asset-file-cache-middleware
52-
53-
or
54-
55-
$ yarn add express-asset-file-cache-middleware
56-
57-
## API
58-
59-
### Input
60-
61-
#### `res.locals.fetchUrl` (required)
62-
63-
The URL of the asset to cache.
64-
65-
#### `res.locals.cacheKey` (optional)
66-
67-
A unique, expireable cache key. If your asset contains a checksum/digest, you're already done, because it falls back to `res.locals.fetchUrl`.
68-
69-
### Output
70-
71-
To further process the response, the following entries of `res.locals` are set:
72-
73-
#### `res.locals.buffer`
74-
75-
The cached asset as a binary buffer. Most likely, you will end the request chain with
76-
77-
```javascript
78-
res.end(res.locals.buffer, "binary");
79-
```
80-
81-
#### `res.locals.contentType` and `res.locals.contentLength`
82-
83-
If you're serving your assets in the response, you'll need to set
84-
85-
```javascript
86-
res.set({
87-
"Content-Type": res.locals.contentType,
88-
"Content-Length": res.locals.contentLength
89-
});
90-
```
91-
92-
## Options
93-
94-
You can pass the following options to the middleware:
95-
96-
### `cacheDir` (optional)
97-
98-
The root directory where the file cache will be located. Falls back to `path.join(process.cwd(), "/tmp")`.
99-
100-
### `logger` (optional)
101-
102-
A logger to use for debugging, e.g. Winston, console, etc.
103-
104-
## Tests
105-
106-
Run the test suite:
107-
108-
```bash
109-
# install dependencies
110-
$ npm install
111-
112-
# unit tests
113-
$ npm test
114-
```
115-
116-
## License
117-
118-
The MIT License (MIT)
119-
120-
Copyright (c) 2019 Julian Rubisch
46+
47+
Note that setting `cacheKey` and `cacheDir` isn't strictly necessary, it will fall back to `res.local.fetchUrl` and `path.join(process.cwd(), "/tmp")`, respectively.
48+
49+
## LRU Eviction
50+
51+
To avoid cluttering your device, an LRU (least recently used) cache eviction strategy is in place. Per default, when your cache dir grows over 1 GB of size, the least recently used (accessed) files will be evicted (deleted), until enough disk space is available again. You can change the cache dir size by specifying `options.maxSize` (in bytes) when creating the middleware.
52+
53+
54+
## Install
55+
56+
$ npm install express-asset-file-cache-middleware
57+
58+
or
59+
60+
$ yarn add express-asset-file-cache-middleware
61+
62+
## API
63+
64+
### Input
65+
66+
#### `res.locals.fetchUrl` (required)
67+
68+
The URL of the asset to cache.
69+
70+
#### `res.locals.cacheKey` (optional)
71+
72+
A unique, expireable cache key. If your asset contains a checksum/digest, you're already done, because it falls back to `res.locals.fetchUrl`.
73+
74+
### Output
75+
76+
To further process the response, the following entries of `res.locals` are set:
77+
78+
#### `res.locals.buffer`
79+
80+
The cached asset as a binary buffer. Most likely, you will end the request chain with
81+
82+
```javascript
83+
res.end(res.locals.buffer, "binary");
84+
```
85+
86+
#### `res.locals.contentType` and `res.locals.contentLength`
87+
88+
If you're serving your assets in the response, you'll need to set
89+
90+
```javascript
91+
res.set({
92+
"Content-Type": res.locals.contentType,
93+
"Content-Length": res.locals.contentLength
94+
});
95+
```
96+
97+
## Options
98+
99+
You can pass the following options to the middleware:
100+
101+
### `cacheDir` (optional)
102+
103+
The root directory where the file cache will be located. Falls back to `path.join(process.cwd(), "/tmp")`.
104+
105+
### `logger` (optional)
106+
107+
A logger to use for debugging, e.g. Winston, console, etc.
108+
109+
### `maxSize` (optional)
110+
The maximum size of the cache directory, from which LRU eviction is applied. Defaults to 1 GB (1024 * 1024 * 1024).
111+
112+
113+
## Tests
114+
115+
Run the test suite:
116+
117+
```bash
118+
# install dependencies
119+
$ npm install
120+
121+
# unit tests
122+
$ npm test
123+
```
124+
125+
## License
126+
127+
The MIT License (MIT)
128+
129+
Copyright (c) 2019 Julian Rubisch
121130

122131
## Contributors ✨
123132

‎index.js

Lines changed: 77 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ const sprintf = require("sprintf-js").sprintf;
44
const fs = require("fs");
55
const path = require("path");
66

7+
const getFolderSize = require("get-folder-size");
8+
79
function makeDirIfNotExists(dir) {
810
if (!fs.existsSync(dir)) {
911
fs.mkdirSync(dir);
@@ -54,13 +56,76 @@ function decodeAssetCacheName(encodedString) {
5456
return decodedFileName.split(":");
5557
}
5658

59+
function touch(path) {
60+
const time = new Date();
61+
try {
62+
fs.utimesSync(path, time, time);
63+
} catch (err) {
64+
fs.closeSync(fs.openSync(path, "w"));
65+
}
66+
}
67+
68+
function evictLeastRecentlyUsed(cacheDir, maxSize, logger) {
69+
getFolderSize(cacheDir, (err, size) => {
70+
if (err) {
71+
throw err;
72+
}
73+
74+
if (size >= maxSize) {
75+
try {
76+
// find least recently used file
77+
const leastRecentlyUsed = findLeastRecentlyUsed(cacheDir);
78+
79+
// and delete it
80+
const { dir } = path.parse(leastRecentlyUsed.path);
81+
fs.unlinkSync(leastRecentlyUsed.path);
82+
fs.rmdirSync(dir);
83+
84+
if (logger) {
85+
logger.info(`Evicted ${leastRecentlyUsed.path} from cache`);
86+
}
87+
88+
evictLeastRecentlyUsed(cacheDir, maxSize, logger);
89+
} catch (e) {
90+
logger.error(e);
91+
}
92+
}
93+
});
94+
}
95+
96+
function findLeastRecentlyUsed(dir, result) {
97+
let files = fs.readdirSync(dir);
98+
result = result || { atime: Date.now(), path: "" };
99+
100+
files.forEach(file => {
101+
const newBase = path.join(dir, file);
102+
103+
if (fs.statSync(newBase).isDirectory()) {
104+
result = findLeastRecentlyUsed(newBase, result);
105+
} else {
106+
const { atime } = fs.statSync(newBase);
107+
108+
if (atime < result.atime) {
109+
result = {
110+
atime,
111+
path: newBase
112+
};
113+
}
114+
}
115+
});
116+
117+
return result;
118+
}
119+
57120
const middleWare = (module.exports = function(options) {
58121
return async function(req, res, next) {
59122
options = options || {};
60123
options.cacheDir =
61124
options && options.cacheDir
62125
? options.cacheDir
63126
: path.join(process.cwd(), "/tmp");
127+
options.maxSize =
128+
options && options.maxSize ? options.maxSize : 1024 * 1024 * 1024;
64129

65130
const {
66131
dir1,
@@ -78,6 +143,9 @@ const middleWare = (module.exports = function(options) {
78143
if (fs.existsSync(assetCachePath)) {
79144
const firstFile = fs.readdirSync(assetCachePath)[0];
80145

146+
// touch file for LRU eviction
147+
middleWare.touch(`${assetCachePath}/${firstFile}`);
148+
81149
const [contentType, contentLength] = middleWare.decodeAssetCacheName(
82150
firstFile
83151
);
@@ -112,6 +180,12 @@ const middleWare = (module.exports = function(options) {
112180
res.locals.contentType = blob.type;
113181
res.locals.contentLength = blob.size;
114182

183+
middleWare.evictLeastRecentlyUsed(
184+
options.cacheDir,
185+
options.maxSize,
186+
options.logger
187+
);
188+
115189
fs.writeFileSync(`${assetCachePath}/${fileName}`, res.locals.buffer);
116190

117191
const [seconds, nanoSeconds] = process.hrtime(startTime);
@@ -125,7 +199,6 @@ const middleWare = (module.exports = function(options) {
125199

126200
next();
127201
} catch (e) {
128-
console.log(e);
129202
// in case fs.writeFileSync writes partial data and fails
130203
if (fs.existsSync(assetCachePath)) {
131204
fs.unlinkSync(assetCachePath);
@@ -145,3 +218,6 @@ middleWare.makeAssetCachePath = makeAssetCachePath;
145218
middleWare.makeDirIfNotExists = makeDirIfNotExists;
146219
middleWare.encodeAssetCacheName = encodeAssetCacheName;
147220
middleWare.decodeAssetCacheName = decodeAssetCacheName;
221+
middleWare.findLeastRecentlyUsed = findLeastRecentlyUsed;
222+
middleWare.evictLeastRecentlyUsed = evictLeastRecentlyUsed;
223+
middleWare.touch = touch;

0 commit comments

Comments
 (0)