Merge pull request #39 from sourcegraph/progress-bar

Fix #37, add progress bar

Author: olafurpg

.github/workflows/sourcegraph.yml

@@ -12,8 +12,6 @@ jobs:
       - run: yarn install
       - run: yarn global add ts-node
       - run: ts-node src/main.ts index
-      - uses: actions/setup-go@v2
-        with: { go-version: '1.17' }
       - name: Convert LSIF Typed into LSIF Graph
         run: |
           curl -Lo lsif-typed https://github.com/sourcegraph/lsif-typescript/releases/download/v0.1.8/lsif-typed && chmod +x lsif-typed

package.json

@@ -1,5 +1,4 @@
 {
-  "$schema": "https://raw.githubusercontent.com/sourcegraph/sourcegraph/master/shared/src/schema/extension.schema.json",
   "name": "@sourcegraph/lsif-typescript",
   "version": "0.1.10",
   "description": "LSIF indexer for TypeScript and JavaScript",
@@ -35,6 +34,8 @@
   "homepage": "https://github.com/sourcegraph/lsif-typescript#readme",
   "dependencies": {
     "google-protobuf": "^3.19.3",
+    "pretty-ms": "^7.0.1",
+    "progress": "^2.0.3",
     "typescript": "^4.5.4",
     "yargs": "^17.3.1"
   },
@@ -45,6 +46,8 @@
     "@types/diff": "5.0.2",
     "@types/google-protobuf": "3.15.5",
     "@types/node": "17.0.14",
+    "@types/pretty-ms": "^5.0.1",
+    "@types/progress": "^2.0.5",
     "@types/yargs": "17.0.9",
     "@typescript-eslint/eslint-plugin": "^5.14.0",
     "@typescript-eslint/parser": "^5.14.0",

src/ProjectIndexer.ts

@@ -1,6 +1,9 @@
+import { Writable as WritableStream } from 'node:stream'
 import * as path from 'path'
 import * as url from 'url'
 
+import prettyMilliseconds from 'pretty-ms'
+import ProgressBar from 'progress'
 import * as ts from 'typescript'
 
 import packageJson from '../package.json'
@@ -25,6 +28,7 @@ export class ProjectIndexer {
     this.packages = new Packages(options.projectRoot)
   }
   public index(): void {
+    const startTimestamp = Date.now()
     this.options.writeIndex(
       new lsiftyped.Index({
         metadata: new lsiftyped.Metadata({
@@ -46,12 +50,46 @@ export class ProjectIndexer {
         `no source files in project root '${this.options.workspaceRoot}'`
       )
     }
+
+    const filesToIndex: ts.SourceFile[] = []
     // Visit every sourceFile in the program
-    for (const sourceFile of this.program.getSourceFiles()) {
+    for (const sourceFile of sourceFiles) {
       const includes = this.config.fileNames.includes(sourceFile.fileName)
       if (!includes) {
         continue
       }
+      filesToIndex.push(sourceFile)
+    }
+
+    const jobs = new ProgressBar(
+      `  ${this.options.projectDisplayName} [:bar] :current/:total :title`,
+      {
+        total: filesToIndex.length,
+        renderThrottle: 100,
+        incomplete: '_',
+        complete: '#',
+        width: 20,
+        clear: true,
+        stream: this.options.noProgressBar
+          ? writableNoopStream()
+          : process.stderr,
+      }
+    )
+    let lastWrite = Date.now()
+    for (const sourceFile of filesToIndex) {
+      const title = path.relative(
+        this.options.workspaceRoot,
+        sourceFile.fileName
+      )
+      jobs.tick({ title })
+      if (this.options.noProgressBar) {
+        const now = Date.now()
+        const elapsed = now - lastWrite
+        if (elapsed > 1000) {
+          lastWrite = now
+          process.stdout.write('.')
+        }
+      }
       const document = new lsif.lib.codeintel.lsiftyped.Document({
         relative_path: path.relative(
           this.options.workspaceRoot,
@@ -84,5 +122,21 @@ export class ProjectIndexer {
         )
       }
     }
+    jobs.terminate()
+    const elapsed = Date.now() - startTimestamp
+    if (this.options.noProgressBar) {
+      process.stdout.write('\n')
+    }
+    console.log(
+      `+ ${this.options.projectDisplayName} (${prettyMilliseconds(elapsed)})`
+    )
   }
 }
+
+function writableNoopStream(): WritableStream {
+  return new WritableStream({
+    write(_unused1, _unused2, callback) {
+      setImmediate(callback)
+    },
+  })
+}

src/main.test.ts

@@ -42,6 +42,7 @@ for (const snapshotDirectory of snapshotDirectories) {
     lsifIndex({
       workspaceRoot: inputRoot,
       projectRoot: inputRoot,
+      projectDisplayName: inputRoot,
       inferTSConfig: false,
       writeIndex: partialIndex => {
         if (partialIndex.metadata) {

src/main.ts

@@ -2,6 +2,7 @@
 import * as child_process from 'child_process'
 import * as fs from 'fs'
 import * as path from 'path'
+import { cwd } from 'process'
 
 import * as ts from 'typescript'
 import * as yargs from 'yargs'
@@ -24,9 +25,15 @@ export interface Options {
   /** The directory containing a tsconfig.json file. */
   projectRoot: string
 
+  /** The display name of the project. */
+  projectDisplayName: string
+
   /** Whether to infer the tsconfig.json file, if it's missing. */
   inferTSConfig: boolean
 
+  /** If truthy, disables the progress bar. */
+  noProgressBar?: boolean
+
   /**
    * Callback to consume the generated LSIF Typed payload in a streaming fashion.
    */
@@ -64,6 +71,11 @@ export function main(): void {
           default: 'dump.lsif-typed',
           describe: 'path to the output file',
         })
+        yargs.option('noProgressBar', {
+          type: 'boolean',
+          default: `${!process.stderr.isTTY}`,
+          describe: 'disable the progress bar',
+        })
       },
       argv => {
         const workspaceRoot = argv.project as string
@@ -74,23 +86,31 @@ export function main(): void {
         const projects: string[] = yarnWorkspaces
           ? listYarnWorkspaces(workspaceRoot)
           : [workspaceRoot]
-        const output = fs.openSync(argv.output as string, 'w')
+        let outputPath = argv.output as string
+        if (!path.isAbsolute(outputPath)) {
+          outputPath = path.resolve(cwd(), outputPath)
+        }
+        const output = fs.openSync(outputPath, 'w')
         try {
           // NOTE: we may want index these projects in parallel in the future.
           // We need to be careful about which order we index the projects because
           // they can have dependencies.
           for (const projectRoot of projects) {
+            const projectDisplayName = projectRoot === '.' ? cwd() : projectRoot
             index({
               workspaceRoot,
               projectRoot,
+              projectDisplayName,
               inferTSConfig,
+              noProgressBar: argv.noProgressBar === true,
               writeIndex: (index): void => {
                 fs.writeSync(output, index.serializeBinary())
               },
             })
           }
         } finally {
           fs.close(output)
+          console.log(`done ${outputPath}`)
         }
       }
     )
@@ -142,9 +162,7 @@ export function index(options: Options): void {
           '{"compilerOptions":{"allowJs":true}}'
         )
       } else {
-        console.error(
-          `skipping project '${options.projectRoot}' because it's missing tsconfig.json (expected at ${tsconfigFileName})`
-        )
+        console.error(`- ${options.projectDisplayName} (missing tsconfig.json)`)
         return
       }
     }

yarn.lock

@@ -440,6 +440,20 @@
   resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.1.tgz#d3357479a0fdfdd5907fe67e17e0a85c906e1301"
   integrity sha512-Gj7cI7z+98M282Tqmp2K5EIsoouUEzbBJhQQzDE3jSIRk6r9gsz0oUokqIUR4u1R3dMHo0pDHM7sNOHyhulypw==
 
+"@types/pretty-ms@^5.0.1":
+  version "5.0.1"
+  resolved "https://registry.yarnpkg.com/@types/pretty-ms/-/pretty-ms-5.0.1.tgz#f2f0d7be58caf8613d149053d446e0282ae11ff3"
+  integrity sha512-FFR4uj0p47Yq6JCrOt7DCaiUJIw7t9Vh7wwt3bF6qq99QRqjSH/doEGZsIIgZqhDmwjBObVBkrn0ICm1pY+mPg==
+  dependencies:
+    pretty-ms "*"
+
+"@types/progress@^2.0.5":
+  version "2.0.5"
+  resolved "https://registry.yarnpkg.com/@types/progress/-/progress-2.0.5.tgz#6e0febf3a82cc0ffdc1cebb4e56d6949fd108775"
+  integrity sha512-ZYYVc/kSMkhH9W/4dNK/sLNra3cnkfT2nJyOAIDY+C2u6w72wa0s1aXAezVtbTsnN8HID1uhXCrLwDE2ZXpplg==
+  dependencies:
+    "@types/node" "*"
+
 "@types/yargs-parser@*":
   version "21.0.0"
   resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-21.0.0.tgz#0c60e537fa790f5f9472ed2776c2b71ec117351b"
@@ -2153,6 +2167,11 @@ parse-json@^5.0.0:
     json-parse-even-better-errors "^2.3.0"
     lines-and-columns "^1.1.6"
 
+parse-ms@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/parse-ms/-/parse-ms-2.1.0.tgz#348565a753d4391fa524029956b172cb7753097d"
+  integrity sha512-kHt7kzLoS9VBZfUsiKjv43mr91ea+U05EyKkEtqp7vNbHxmaVuEqN7XxeEVnGrMtYOAxGrDElSi96K7EgO1zCA==
+
 path-exists@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-3.0.0.tgz#ce0ebeaa5f78cb18925ea7d810d7b59b010fd515"
@@ -2218,7 +2237,14 @@ pretty-format@^26.6.2:
     ansi-styles "^4.0.0"
     react-is "^17.0.1"
 
-progress@^2.0.0:
+pretty-ms@*, pretty-ms@^7.0.1:
+  version "7.0.1"
+  resolved "https://registry.yarnpkg.com/pretty-ms/-/pretty-ms-7.0.1.tgz#7d903eaab281f7d8e03c66f867e239dc32fb73e8"
+  integrity sha512-973driJZvxiGOQ5ONsFhOF/DtzPMOMtgC11kCpUrPGMTgqp2q/1gwzCquocrN33is0VZ5GFHXZYMM9l6h67v2Q==
+  dependencies:
+    parse-ms "^2.1.0"
+
+progress@^2.0.0, progress@^2.0.3:
   version "2.0.3"
   resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8"
   integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==