Refactor gen_struct_info to emit JSON directly. NFC (#24489)

I wanted to add something to gen_struct_info functionality, but found it
difficult to comprehend the custom intermediate format of the C
program's output.

I changed it to emit the desired result directly as JSON instead, which
results in the same output but makes code easier to reason about and
debug, as well as possibly tiny bit faster as well.

Author: RReverser

src/lib/libbootstrap.js

@@ -33,28 +33,19 @@ addToLibrary({
     quit_(1, e);
   },
 
-  // printf/puts implementations for when musl is not pulled in - very
-  // partial, but enough for bootstrapping structInfo
-  printf__deps: ['$formatString', '$intArrayToString'],
-  printf__sig: 'ipp',
-  printf: (format, varargs) => {
-    // int printf(const char *restrict format, ...);
-    // http://pubs.opengroup.org/onlinepubs/000095399/functions/printf.html
-    // extra effort to support printf, even without a filesystem. very partial, very hackish
-    var result = formatString(format, varargs);
-    var string = intArrayToString(result);
-    if (string.endsWith('\n')) string = string.slice(0, -1); // remove a final \n, as Module.print will do that
-    out(string);
-    return result.length;
-  },
-
-  puts__sig: 'ip',
-  puts: (s) => {
-    // extra effort to support puts, even without a filesystem. very partial, very hackish
-    var result = UTF8ToString(s);
-    var string = result;
-    if (string.endsWith('\n')) string = string.slice(0, -1); // remove a final \n, as Module.print will do that
-    out(string);
-    return result.length;
+  fd_write__sig: 'iippp',
+  fd_write: (fd, iov, iovcnt, pnum) => {
+    // implementation almost copied from libwasi.js one for SYSCALLS_REQUIRE_FILESYSTEM=0
+    // (the only difference is that we can't use C_STRUCTS here)
+    var num = 0;
+    for (var i = 0; i < iovcnt; i++) {
+      var ptr = {{{ makeGetValue('iov', 0, '*') }}};
+      var len = {{{ makeGetValue('iov', POINTER_SIZE, '*') }}};
+      iov += {{{ POINTER_SIZE }}} * 2;
+      process.stdout.write(HEAPU8.subarray(ptr, ptr + len));
+      num += len;
+    }
+    {{{ makeSetValue('pnum', 0, 'num', '*') }}};
+    return 0;
   },
 });

tools/gen_struct_info.py

@@ -62,6 +62,7 @@
 import tempfile
 import shlex
 import subprocess
+import typing
 
 __scriptdir__ = os.path.dirname(os.path.abspath(__file__))
 __rootdir__ = os.path.dirname(__scriptdir__)
@@ -107,100 +108,69 @@ def show(msg):
     sys.stderr.write('gen_struct_info: %s\n' % msg)
 
 
-# The following three functions generate C code. The output of the compiled code will be
-# parsed later on and then put back together into a dict structure by parse_c_output().
+# The Scope class generates C code which, in turn, outputs JSON.
 #
 # Example:
-#   c_descent('test1', code)
-#   c_set('item', 'i%i', '111', code)
-#   c_set('item2', 'i%i', '9', code)
-#   c_set('item3', 's%s', '"Hello"', code)
-#   c_ascent(code)
-#   c_set('outer', 'f%f', '0.999', code)
-#
-# Will result in:
-#   {
-#     'test1': {
-#       'item': 111,
-#       'item2': 9,
-#       'item3': 'Hello',
-#     },
-#     'outer': 0.999
-#   }
-def c_set(name, type_, value, code):
-  code.append('printf("K' + name + '\\n");')
-  code.append('printf("V' + type_ + '\\n", ' + value + ');')
-
-
-def c_descent(name, code):
-  code.append('printf("D' + name + '\\n");')
-
-
-def c_ascent(code):
-  code.append('printf("A\\n");')
-
-
-def parse_c_output(lines):
-  result = {}
-  cur_level = result
-  parent = []
-  key = None
-
-  for line in lines:
-    arg = line[1:].strip()
-    if '::' in arg:
-      arg = arg.split('::', 1)[1]
-    if line[0] == 'K':
-      # This is a key
-      key = arg
-    elif line[0] == 'V':
-      # A value
-      if arg[0] == 'i':
-        arg = int(arg[1:])
-      elif arg[0] == 'f':
-        arg = float(arg[1:])
-      elif arg[0] == 's':
-        arg = arg[1:]
-
-      cur_level[key] = arg
-    elif line[0] == 'D':
-      # Remember the current level as the last parent.
-      parent.append(cur_level)
-
-      # We descend one level.
-      cur_level[arg] = {}
-      cur_level = cur_level[arg]
-    elif line[0] == 'A':
-      # We return to the parent dict. (One level up.)
-      cur_level = parent.pop()
-
-  return result
-
-
-def gen_inspect_code(path, struct, code):
-  if path[0][-1] == '#':
-    path[0] = path[0].rstrip('#')
-    prefix = ''
-  else:
-    prefix = 'struct '
+#   with Scope(code) as scope: # generates code that outputs beginning of a JSON object '{\n'
+#     scope.set('item', '%i', '111') # generates code that outputs '"item": 111'
+#     scope.set('item2', '%f', '4.2') # generates code that outputs ',\n"item2": 4.2'
+#   # once the scope is exited, it generates code that outputs the end of the JSON object '\n}'
+class Scope:
+  def __init__(self, code: typing.List[str]):
+    self.code = code
+    self.has_data = False
+
+  def __enter__(self):
+    self.code.append('puts("{");')
+    return self
+
+  def __exit__(self, exc_type, exc_val, exc_tb):
+    if self.has_data:
+      self.code.append('puts("");')
+    self.code.append('printf("}");')
+
+  def _start_child(self, name: str):
+    if self.has_data:
+      self.code.append('puts(",");')
+    else:
+      self.has_data = True
+    if '::' in name:
+      name = name.split('::', 1)[1]
+    self.code.append(fr'printf("\"{name}\": ");')
 
-  c_descent(path[-1], code)
+  def child(self, name: str):
+    self._start_child(name)
+    return Scope(self.code)
 
-  if len(path) == 1:
-    c_set('__size__', 'i%zu', 'sizeof (' + prefix + path[0] + ')', code)
-  else:
-    c_set('__size__', 'i%zu', 'sizeof ((' + prefix + path[0] + ' *)0)->' + '.'.join(path[1:]), code)
-    # c_set('__offset__', 'i%zu', 'offsetof(' + prefix + path[0] + ', ' + '.'.join(path[1:]) + ')', code)
-
-  for field in struct:
-    if isinstance(field, dict):
-      # We have to recurse to inspect the nested dict.
-      fname = list(field.keys())[0]
-      gen_inspect_code(path + [fname], field[fname], code)
+  def set(self, name: str, type_: str, value: str):
+    self._start_child(name)
+
+    assert type_.startswith('%')
+    # We only support numeric defines as they are directly compatible with JSON.
+    # Extend to string escaping if we ever need that in the future.
+    assert type_[-1] in {'d', 'i', 'u', 'f', 'F', 'e', 'E'}
+
+    self.code.append(f'printf("{type_}", {value});')
+
+  def gen_inspect_code(self, path: typing.List[str], struct: typing.List[typing.Union[str, dict]]):
+    if path[0][-1] == '#':
+      path[0] = path[0].rstrip('#')
+      prefix = ''
     else:
-      c_set(field, 'i%zu', 'offsetof(' + prefix + path[0] + ', ' + '.'.join(path[1:] + [field]) + ')', code)
+      prefix = 'struct '
+    prefix += path[0]
+
+    with self.child(path[-1]) as scope:
+      path_for_sizeof = [f'({prefix}){{}}'] + path[1:]
+      scope.set('__size__', '%zu', f'sizeof ({".".join(path_for_sizeof)})')
 
-  c_ascent(code)
+      for field in struct:
+        if isinstance(field, dict):
+          # We have to recurse to inspect the nested dict.
+          fname = list(field.keys())[0]
+          self.gen_inspect_code(path + [fname], field[fname])
+        else:
+          scope.set(field, '%zu', f'offsetof({prefix}, {".".join(path[1:] + [field])})')
 
 
 def generate_c_code(headers):
@@ -209,28 +179,23 @@ def generate_c_code(headers):
   code.extend(f'''#include "{header['name']}"''' for header in headers)
 
   code.append('int main() {')
-  c_descent('structs', code)
-  for header in headers:
-    for name, struct in header['structs'].items():
-      gen_inspect_code([name], struct, code)
-
-  c_ascent(code)
-  c_descent('defines', code)
-  for header in headers:
-    for name, type_ in header['defines'].items():
-      # Add the necessary python type, if missing.
-      if '%' not in type_:
-        if type_[-1] in {'d', 'i', 'u'}:
-          # integer
-          type_ = 'i%' + type_
-        elif type_[-1] in {'f', 'F', 'e', 'E', 'g', 'G'}:
-          # float
-          type_ = 'f%' + type_
-        elif type_[-1] in {'x', 'X', 'a', 'A', 'c', 's'}:
-          # hexadecimal or string
-          type_ = 's%' + type_
-
-      c_set(name, type_, name, code)
+
+  with Scope(code) as root:
+    with root.child('structs') as structs:
+      for header in headers:
+        for name, struct in header['structs'].items():
+          structs.gen_inspect_code([name], struct)
+
+    with root.child('defines') as defines:
+      for header in headers:
+        for name, type_ in header['defines'].items():
+          # Add the necessary python type, if missing.
+          if '%' not in type_:
+            type_ = f'%{type_}'
+
+          defines.set(name, type_, name)
+
+  code.append('puts("");')  # Add a newline after the JSON output to flush it.
 
   code.append('return 0;')
   code.append('}')
@@ -254,7 +219,6 @@ def generate_cmd(js_file_path, src_file_path, cflags):
                                '-O0',
                                '-Werror',
                                '-Wno-format',
-                               '-nolibc',
                                '-sBOOTSTRAPPING_STRUCT_INFO',
                                '-sINCOMING_MODULE_JS_API=',
                                '-sSTRICT',
@@ -295,7 +259,7 @@ def inspect_headers(headers, cflags):
   # Run the compiled program.
   show('Calling generated program... ' + js_file_path)
   node_args = shared.node_bigint_flags(config.NODE_JS)
-  info = shared.run_js_tool(js_file_path, node_args=node_args, stdout=shared.PIPE).splitlines()
+  info = shared.run_js_tool(js_file_path, node_args=node_args, stdout=shared.PIPE)
 
   if not DEBUG:
     # Remove all temporary files.
@@ -307,7 +271,7 @@ def inspect_headers(headers, cflags):
       os.unlink(wasm_file_path)
 
   # Parse the output of the program into a dict.
-  return parse_c_output(info)
+  return json.loads(info)
 
 
 def merge_info(target, src):