Use a cache to avoid excessive reloading of tsdoc.json

Author: octogonz

eslint-plugin/src/index.ts

@@ -10,10 +10,11 @@ import {
 } from '@microsoft/tsdoc-config';
 import * as eslint from "eslint";
 import * as ESTree from "estree";
+import * as path from 'path';
 
 const tsdocMessageIds: {[x: string]: string} = {};
 
-const defaultTSDocConfiguration: TSDocConfiguration = new TSDocConfiguration()
+const defaultTSDocConfiguration: TSDocConfiguration = new TSDocConfiguration();
 defaultTSDocConfiguration.allTsdocMessageIds.forEach((messageId: string) => {
   tsdocMessageIds[messageId] = `${messageId}: {{unformattedText}}`;
 });
@@ -22,6 +23,33 @@ interface IPlugin {
   rules: {[x: string]: eslint.Rule.RuleModule};
 }
 
+// To debug the plugin, temporarily uncomment the body of this function
+function debug(message: string): void {
+  message = require("process").pid + ": " + message;
+  console.log(message);
+  require('fs').writeFileSync('C:\\Git\\log.txt', message + '\r\n', { flag: 'as' });
+}
+
+interface ICachedConfig {
+  loadTimeMs: number;
+  lastCheckTimeMs: number;
+  configFile: TSDocConfigFile;
+}
+
+// How often to check for modified input files.  If a file's modification timestamp has changed, then we will
+// evict the cache entry immediately.
+const CACHE_CHECK_INTERVAL_MS: number = 15*1000;
+
+// Evict old entries from the cache after this much time, regardless of whether the file was detected as being
+// modified or not.
+const CACHE_EXPIRE_MS: number = 30*1000;
+
+// If this many objects accumulate in the cache, then it is cleared to avoid a memory leak.
+const CACHE_MAX_SIZE: number = 100;
+
+// findConfigPathForFolder() result --> loaded tsdoc.json configuration
+const cachedConfigs: Map<string, ICachedConfig> = new Map<string, ICachedConfig>();
+
 const plugin: IPlugin = {
   rules: {
     // NOTE: The actual ESLint rule name will be "tsdoc/syntax".  It is calculated by deleting "eslint-plugin-"
@@ -43,10 +71,67 @@ const plugin: IPlugin = {
         }
       },
       create: (context: eslint.Rule.RuleContext) => {
-        const tsdocConfiguration: TSDocConfiguration = new TSDocConfiguration();
-
         const sourceFilePath: string = context.getFilename();
-        const tsdocConfigFile: TSDocConfigFile = TSDocConfigFile.loadForFolder(sourceFilePath);
+
+        debug(`Linting: "${sourceFilePath}"`);
+
+        // First, determine the file to be loaded. If not found, the configFilePath will be an empty string.
+        const configFilePath: string ='';// TSDocConfigFile.findConfigPathForFolder(sourceFilePath);
+
+        // If configFilePath is an empty string, then we'll use the folder of sourceFilePath as our cache key
+        // (instead of an empty string)
+        const cacheKey: string = configFilePath || path.dirname(sourceFilePath);
+        debug(`Cache key: "${cacheKey}"`);
+
+        const nowMs: number = (new Date()).getTime();
+
+        let cachedConfig: ICachedConfig | undefined = undefined;
+
+        // Do we have a cached object?
+        cachedConfig = cachedConfigs.get(cacheKey);
+
+        if (cachedConfig) {
+          debug('Cache hit');
+
+          // Is the cached object still valid?
+          const loadAgeMs: number = nowMs - cachedConfig.loadTimeMs;
+          const lastCheckAgeMs: number = nowMs - cachedConfig.lastCheckTimeMs;
+
+          if (loadAgeMs > CACHE_EXPIRE_MS || loadAgeMs < 0) {
+            debug('Evicting because item is expired');
+            cachedConfig = undefined;
+            cachedConfigs.delete(cacheKey);
+          } else if (lastCheckAgeMs > CACHE_CHECK_INTERVAL_MS || lastCheckAgeMs < 0) {
+            debug('Checking for modifications');
+            cachedConfig.lastCheckTimeMs = nowMs;
+            if (cachedConfig.configFile.checkForModifiedFiles()) {
+              // Invalidate the cache because it failed to load completely
+              debug('Evicting because item was modified');
+              cachedConfig = undefined;
+              cachedConfigs.delete(cacheKey);
+            }
+          }
+        }
+
+        // Load the object
+        if (!cachedConfig) {
+          if (cachedConfigs.size > CACHE_MAX_SIZE) {
+            debug('Clearing cache');
+            cachedConfigs.clear(); // avoid a memory leak
+          }
+
+          debug(`LOADING CONFIG: ${configFilePath}`);
+          cachedConfig = {
+            configFile: TSDocConfigFile.loadFile(configFilePath),
+            lastCheckTimeMs: nowMs,
+            loadTimeMs: nowMs
+          };
+
+          cachedConfigs.set(cacheKey, cachedConfig);
+        }
+
+        const tsdocConfigFile: TSDocConfigFile = cachedConfig.configFile;
+        const tsdocConfiguration: TSDocConfiguration = new TSDocConfiguration()
 
         if (!tsdocConfigFile.fileNotFound) {
           if (tsdocConfigFile.hasErrors) {

tsdoc-config/src/TSDocConfigFile.ts

@@ -59,6 +59,7 @@ export class TSDocConfigFile {
   private readonly _extendsFiles: TSDocConfigFile[];
   private _filePath: string;
   private _fileNotFound: boolean;
+  private _fileMTime: number;
   private _hasErrors: boolean;
   private _tsdocSchema: string;
   private readonly _extendsPaths: string[];
@@ -71,6 +72,7 @@ export class TSDocConfigFile {
     this._filePath = '';
     this._fileNotFound = true;
     this._hasErrors = false;
+    this._fileMTime = 0;
     this._tsdocSchema = '';
     this._extendsPaths = [];
     this._tagDefinitions= [];
@@ -130,14 +132,57 @@ export class TSDocConfigFile {
     return this._tagDefinitions;
   }
 
+  /**
+   * This can be used for cache eviction.  It returns true if the modification timestamp has changed for
+   * any of the files that were read when loading this `TSDocConfigFile`, which indicates that the file should be
+   * reloaded.  It does not consider cases where `TSDocConfigFile.fileNotFound` was `true`.
+   *
+   * @remarks
+   * This can be used for cache eviction.  An example eviction strategy might be like this:
+   *
+   * - call `checkForModifiedFiles()` once per second, and reload the configuration if it returns true
+   *
+   * - otherwise, reload the configuration when it is more than 10 seconds old (to handle less common cases such
+   *   as creation of a missing file, or creation of a file at an earlier location in the search path).
+   */
+  public checkForModifiedFiles(): boolean {
+    if (this._checkForModifiedFile()) {
+      return true;
+    }
+    for (const extendsFile of this.extendsFiles) {
+      if (extendsFile.checkForModifiedFiles()) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Checks the last modification time for `TSDocConfigFile.filePath` and returns `true` if it has changed
+   * since the file was loaded.  If the file is missing, this returns `false`.  If the timestamp cannot be read,
+   * then this returns `true`.
+   */
+  private _checkForModifiedFile(): boolean {
+    if (this._fileNotFound || !this._filePath) {
+      return false;
+    }
+
+    try {
+      const mtimeMs: number = fs.statSync(this._filePath).mtimeMs;
+      return mtimeMs !== this._fileMTime;
+    } catch (error) {
+      return true;
+    }
+  }
+
   private _reportError(parserMessageParameters: IParserMessageParameters): void {
     this.log.addMessage(new ParserMessage(parserMessageParameters));
     this._hasErrors = true;
   }
 
   private _loadJsonFile(): void {
     const configJsonContent: string = fs.readFileSync(this._filePath).toString();
-
+    this._fileMTime = fs.statSync(this._filePath).mtimeMs;
     this._fileNotFound = false;
 
     const configJson: IConfigJson = jju.parse(configJsonContent, { mode: 'cjson' });
@@ -248,7 +293,13 @@ export class TSDocConfigFile {
     }
   }
 
-  private static _findConfigPathForFolder(folderPath: string): string {
+  /**
+   * For the given folder, look for the relevant tsdoc.json file (if any), and return its path.
+   *
+   * @param folderPath - the path to a folder where the search should start
+   * @returns the (possibly relative) path to tsdoc.json, or an empty string if not found
+   */
+  public static findConfigPathForFolder(folderPath: string): string {
     if (folderPath) {
       let foundFolder: string = folderPath;
       for (;;) {
@@ -276,16 +327,23 @@ export class TSDocConfigFile {
   }
 
   /**
-   * For the given folder, discover the relevant tsdoc.json files (if any), and load them.
+   * Calls `TSDocConfigFile.findConfigPathForFolder()` to find the relevant tsdoc.json config file, if one exists.
+   * Then calls `TSDocConfigFile.findConfigPathForFolder()` to return the loaded result.
    * @param folderPath - the path to a folder where the search should start
    */
   public static loadForFolder(folderPath: string): TSDocConfigFile {
-    const configFile: TSDocConfigFile = new TSDocConfigFile();
-    const rootConfigPath: string = TSDocConfigFile._findConfigPathForFolder(folderPath);
+    const rootConfigPath: string = TSDocConfigFile.findConfigPathForFolder(folderPath);
+    return TSDocConfigFile.loadForFolder(rootConfigPath);
+  }
 
+  /**
+   * Loads the specified tsdoc.json and any base files that it refers to using the "extends" option.
+   * @param tsdocJsonFilePath - the path to the tsdoc.json config file
+   */
+  public static loadFile(tsdocJsonFilePath: string): TSDocConfigFile {
+    const configFile: TSDocConfigFile = new TSDocConfigFile();
     const alreadyVisitedPaths: Set<string> = new Set<string>();
-    configFile._loadWithExtends(rootConfigPath, undefined, alreadyVisitedPaths);
-
+    configFile._loadWithExtends(tsdocJsonFilePath, undefined, alreadyVisitedPaths);
     return configFile;
   }