fix: Fix read file fn which uses double the memory (#886)

* fix: Fix read file fn which uses double the memory

* Development snapshot

* Development snapshot

Author: runk

src/fs.ts

@@ -5,5 +5,6 @@ export default {
   existsSync: fs.existsSync,
   readFile: util.promisify(fs.readFile),
   watchFile: fs.watchFile,
-  createReadStream: fs.createReadStream, 
+  createReadStream: fs.createReadStream,
+  stat: util.promisify(fs.stat),
 };

src/index.test.ts

@@ -19,6 +19,7 @@ describe('index', () => {
       watchHandler = cb;
     });
     sandbox.spy(fs, 'createReadStream');
+    sandbox.spy(fs, 'readFile');
   });
   afterEach(() => {
     sandbox.restore();
@@ -51,17 +52,17 @@ describe('index', () => {
       const lookup = await maxmind.open(dbPath, options);
       assert(lookup.get('2001:230::'));
       assert((fs.watchFile as SinonSpy).calledOnce);
-      assert((fs.createReadStream as SinonSpy).calledOnce);
+      assert((fs.readFile as SinonSpy).calledOnce);
     });
 
     it('should work with auto updates', async () => {
       const options = { watchForUpdates: true };
       const lookup = await maxmind.open(dbPath, options);
       assert(lookup.get('2001:230::'));
       assert((fs.watchFile as SinonSpy).calledOnce);
-      assert((fs.createReadStream as SinonSpy).calledOnce);
+      assert((fs.readFile as SinonSpy).calledOnce);
       await watchHandler();
-      assert((fs.createReadStream as SinonSpy).calledTwice);
+      assert((fs.readFile as SinonSpy).calledTwice);
     });
 
     it('should work with auto updates and call specified hook', async () => {

src/index.ts

@@ -6,6 +6,9 @@ import ip from './ip';
 import isGzip from './is-gzip';
 import utils from './utils';
 
+const LARGE_FILE_THRESHOLD = 512 * 1024 * 1024;
+const STREAM_WATERMARK = 8 * 1024 * 1024;
+
 type Callback = () => void;
 
 export interface OpenOpts {
@@ -17,27 +20,48 @@ export interface OpenOpts {
   watchForUpdatesHook?: Callback;
 }
 
-const readFile = async (filepath: string): Promise<Buffer> => {
-  return new Promise((resolve, reject) => {
-    const chunks: Buffer[] = [];
+/**
+ * Read large file in chunks.
+ *
+ * Reason it's not used for all file sizes is that it's slower than fs.readFile and uses
+ * a bit more memory due to the buffer operations.
+ *
+ * Node seems to have a limit of 2GB for fs.readFileSync, so we need to use streams for
+ * larger files.
+ *
+ * @param filepath
+ * @param size
+ * @returns
+ */
+const readLargeFile = async (filepath: string, size: number): Promise<Buffer> =>
+  new Promise((resolve, reject) => {
+    let buffer = Buffer.allocUnsafe(size);
+    let offset = 0;
     const stream = fs.createReadStream(filepath, {
-      highWaterMark: 64 * 1024 * 1024, // 64 MB chunks
+      highWaterMark: STREAM_WATERMARK,
     });
 
     stream.on('data', (chunk: Buffer) => {
-      chunks.push(chunk);
+      chunk.copy(buffer, offset);
+      offset += chunk.length;
     });
 
     stream.on('end', () => {
-      resolve(Buffer.concat(chunks));
+      stream.close();
+      resolve(buffer);
     });
 
     stream.on('error', (err) => {
       reject(err);
     });
   });
-};
 
+const readFile = async (filepath: string): Promise<Buffer> => {
+  const fstat = await fs.stat(filepath);
+  return fstat.size < LARGE_FILE_THRESHOLD
+    ? fs.readFile(filepath)
+    : readLargeFile(filepath, fstat.size);
+};
 
 export const open = async <T extends Response>(
   filepath: string,