👍 Add function code generator and gen command

Author: lambdalisue

Makefile

@@ -26,6 +26,10 @@ type-check: FORCE	## Type check
 test: FORCE	## Test
 	@deno test --unstable -A
 
+gen: FORCE	## Generate codes
+	@deno run -A ./scripts/gen-function/gen-function.ts
+	@make fmt
+
 dlink: FORCE	## Update dlink
 	(cd denops_std; ${TOOLS}/bin/dlink)
 	@make fmt

scripts/gen-function/format.ts

@@ -0,0 +1,53 @@
+import { Definition, Variant } from "./types.ts";
+
+const translate: Record<string, string> = {
+  "default": "defaultValue",
+  "delete": "delete_",
+  "eval": "eval_",
+  "function": "function_",
+};
+
+function formatDocs(docs: string): string[] {
+  const lines = docs.replaceAll(/\*\//g, "* /").split("\n").map((v) =>
+    ` * ${v}`
+  );
+  return ["/**", ...lines, " */"];
+}
+
+function formatVariants(fn: string, vars: Variant[]): string[] {
+  const lines = vars.map((variant) => {
+    const args = variant.map(({ name, optional }) => {
+      name = translate[name] ?? name;
+      return `${name}${optional ? "?" : ""}: unknown`;
+    });
+    return `export function ${fn}(denops: Denops, ${
+      args.join(", ")
+    }): Promise<unknown>;`;
+  });
+  return lines;
+}
+
+function formatDefinition({ fn, docs, vars }: Definition): string[] {
+  fn = translate[fn] ?? fn;
+  const lines = [
+    ...formatDocs(docs),
+    ...formatVariants(fn, vars),
+    `export function ${fn}(denops: Denops, ...args: unknown[]): Promise<unknown> {`,
+    `  return denops.call('${fn}', ...args);`,
+    "}",
+    "",
+  ];
+  return lines;
+}
+
+export function format(definitions: Definition[], root: string): string[] {
+  const denops = `${root}/vendor/https/deno.land/x/denops_core/mod.ts`;
+  const lines = [
+    "// NOTE: This file is generated. Do NOT modify it manually.",
+    "// deno-lint-ignore-file camelcase",
+    `import { Denops } from "${denops}";`,
+    "",
+    ...definitions.map(formatDefinition),
+  ];
+  return lines.flat();
+}

scripts/gen-function/gen-function.ts

@@ -0,0 +1,71 @@
+import {
+  difference,
+  intersection,
+} from "https://deno.land/x/set_operations@v1.0.0/mod.ts";
+import * as path from "https://deno.land/std@0.100.0/path/mod.ts";
+import * as commonManual from "../../denops_std/function/_manual.ts";
+import * as vimManual from "../../denops_std/function/vim/_manual.ts";
+import * as nvimManual from "../../denops_std/function/nvim/_manual.ts";
+import { parse } from "./parse.ts";
+import { format } from "./format.ts";
+import { downloadString } from "./utils.ts";
+
+const VIM_VERSION = "8.1.2424";
+const NVIM_VERSION = "0.4.4";
+
+const manualFnSet = new Set([
+  ...Object.keys(commonManual),
+  ...Object.keys(vimManual),
+  ...Object.keys(nvimManual),
+]);
+
+const vimHelp = await downloadString(
+  `https://raw.githubusercontent.com/vim/vim/v${VIM_VERSION}/runtime/doc/eval.txt`,
+);
+const vimDefs = parse(vimHelp);
+const vimFnSet = difference(new Set(vimDefs.map((def) => def.fn)), manualFnSet);
+
+const nvimHelp = await downloadString(
+  `https://raw.githubusercontent.com/neovim/neovim/v${NVIM_VERSION}/runtime/doc/eval.txt`,
+);
+const nvimDefs = parse(nvimHelp);
+const nvimFnSet = difference(
+  new Set(nvimDefs.map((def) => def.fn)),
+  manualFnSet,
+);
+
+const commonFnSet = intersection(vimFnSet, nvimFnSet);
+const vimOnlyFnSet = difference(vimFnSet, nvimFnSet);
+const nvimOnlyFnSet = difference(nvimFnSet, vimFnSet);
+
+const commonCode = format(
+  vimDefs.filter((def) => commonFnSet.has(def.fn)),
+  "..",
+);
+const vimOnlyCode = format(
+  vimDefs.filter((def) => vimOnlyFnSet.has(def.fn)),
+  "../..",
+);
+const nvimOnlyCode = format(
+  nvimDefs.filter((def) => nvimOnlyFnSet.has(def.fn)),
+  "../..",
+);
+
+await Deno.writeTextFile(
+  path.fromFileUrl(
+    new URL("../../denops_std/function/_generated.ts", import.meta.url),
+  ),
+  commonCode.join("\n"),
+);
+await Deno.writeTextFile(
+  path.fromFileUrl(
+    new URL("../../denops_std/function/vim/_generated.ts", import.meta.url),
+  ),
+  vimOnlyCode.join("\n"),
+);
+await Deno.writeTextFile(
+  path.fromFileUrl(
+    new URL("../../denops_std/function/nvim/_generated.ts", import.meta.url),
+  ),
+  nvimOnlyCode.join("\n"),
+);

scripts/gen-function/parse.ts

@@ -0,0 +1,131 @@
+import { Definition, Variant } from "./types.ts";
+import { Counter, regexIndexOf } from "./utils.ts";
+
+/**
+ * Parse Vim/Neovim help.
+ *
+ * It extract a function definition block like below and return
+ * a list of `Definition`.
+ *
+ * ```
+ * cursor({lnum}, {col} [, {off}])                    *cursor()*
+ * cursor({list})
+ * 		Positions the cursor at the column ...
+ * 		line {lnum}.  The first column is one...
+ * ```
+ */
+export function parse(content: string): Definition[] {
+  const definitions: Definition[] = [];
+  for (const match of content.matchAll(/\*(\w+?)\(\)\*/g)) {
+    const fn = match[1];
+    const i = match.index || 0;
+    const s = content.lastIndexOf("\n", i);
+    const ms = regexIndexOf(content, /\n[<>\s]/, i);
+    const me = regexIndexOf(content, /\n[^<>\s]/, ms);
+    const e = content.lastIndexOf("\n", me);
+    const block = content
+      .substring(s, e)
+      .replaceAll(/\*.+?\*/g, "") // Remove tags
+      .replaceAll(/\s+\n/g, "\n") // Remove trailing '\s'
+      .trim();
+    definitions.push(parseBlock(fn, block));
+  }
+  return definitions;
+}
+
+/**
+ * Parse function definition block.
+ *
+ * A function definition block is constrcuted with following parts
+ *
+ * ```
+ * cursor({lnum}, {col} [, {off}])                        <- variant
+ * cursor({list})                                         <- variant
+ * 		Positions the cursor at the column ...    <- document
+ * 		line {lnum}.  The first column is one...  <- document
+ * ```
+ *
+ * This function parse content like above and return `Definition`.
+ *
+ */
+function parseBlock(fn: string, body: string): Definition {
+  // Remove '\n' in {variant} to make {variant} single line (ex. `searchpairpos`)
+  body = body.replaceAll(new RegExp(`^\(${fn}\\([^\)]*?\)\n\t*`, "g"), "$1");
+  // Append ')' for an invalid {variant}. (ex. `win_id2tabwin` in Neovim)
+  body = body.replaceAll(new RegExp(`^\(${fn}\\([^\)]*?\)\t+`, "g"), "$1)\t");
+  // Insert '\n' between {variant} and {document} (ex. `argidx`)
+  body = body.replaceAll(new RegExp(`^\(${fn}\\(.*?\\)\)\t`, "g"), "$1\n\t\t");
+
+  // Remove leading '>' or trailing '<' which is used to define code block in help
+  body = body.replaceAll(/\n<|>\n/g, "\n");
+
+  // Split body into vars/docs
+  const vars: Variant[] = [];
+  const docs: string[] = [];
+  body.split("\n").forEach((line) => {
+    if (/^\s/.test(line)) {
+      docs.push(line.replace(/^\t\t/, ""));
+    } else {
+      const variant = parseVariant(line);
+      if (variant) {
+        vars.push(variant);
+      }
+    }
+  });
+  return {
+    fn,
+    docs: docs.join("\n"),
+    vars,
+  };
+}
+
+/**
+ * Parse variant.
+ *
+ * A variant is constructed with following parts
+ *
+ * ```
+ *   fn            args
+ * ~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~
+ * cursor({lnum}, {col} [, {off}])
+ *              |       |
+ *              |       +- optional marker
+ *              +- separator
+ * ```
+ *
+ * This function parse content like above and return `Variant`.
+ *
+ */
+function parseVariant(variant: string): Variant | undefined {
+  // Extract {args} part from {variant}
+  const m = variant.match(new RegExp(`^\\w+\\(\(.+?\)\\)`));
+  if (!m) {
+    // The {variant} does not have {args}, probabliy it's not variant (ex. `strstr`)
+    return undefined;
+  }
+  let optional = false;
+  const counter = new Counter();
+  const args = m[1].split(",").map((t) => {
+    const name = t.replaceAll(/[{}\[\]\s]/g, "");
+    const spread = name.endsWith("...");
+    const arg = {
+      name: name.replace("...", "").replace(/^(\d+)$/, "v$1"),
+      spread,
+      optional,
+    };
+    if (t.endsWith("[")) {
+      optional = true;
+    }
+    counter.count(name);
+    return arg;
+  }).filter((arg) => arg.name);
+  const indexer = new Counter();
+  const uniqueArgs = args.map(({ name, spread, optional }) => {
+    if (counter.get(name) > 1) {
+      const index = indexer.count(name);
+      name = `${name}${index}`;
+    }
+    return { name, spread, optional };
+  });
+  return uniqueArgs;
+}

scripts/gen-function/types.ts

@@ -0,0 +1,13 @@
+export type Arg = {
+  name: string;
+  spread: boolean;
+  optional: boolean;
+};
+
+export type Variant = Arg[];
+
+export type Definition = {
+  fn: string;
+  docs: string;
+  vars: Variant[];
+};

scripts/gen-function/utils.ts

@@ -0,0 +1,34 @@
+import * as io from "https://deno.land/std@0.100.0/io/mod.ts";
+
+export async function downloadString(url: string): Promise<string> {
+  const textDecoder = new TextDecoder();
+  const response = await fetch(url);
+  if (!response.body) {
+    throw new Error(`Failed to read ${url}`);
+  }
+  const reader = io.readerFromStreamReader(response.body.getReader());
+  return textDecoder.decode(await io.readAll(reader));
+}
+
+export function regexIndexOf(s: string, pattern: RegExp, offset = 0): number {
+  const index = s.slice(offset).search(pattern);
+  return index < 0 ? index : index + offset;
+}
+
+export class Counter {
+  #map: Map<string, number>;
+
+  constructor() {
+    this.#map = new Map();
+  }
+
+  count(name: string): number {
+    const value = this.get(name) + 1;
+    this.#map.set(name, value);
+    return value;
+  }
+
+  get(name: string): number {
+    return this.#map.get(name) ?? 0;
+  }
+}