Refactor Starlark API docgen to not refer to Java annotations directly

Prerequisite for supporting .bzl-defined entities in Starlark API docgen.

PiperOrigin-RevId: 808670253
Change-Id: I73d7fbed1803a5ce37953aabe0e3f3254f830696

Author: tetromino

src/main/java/com/google/devtools/build/docgen/ApiExporter.java

@@ -22,9 +22,8 @@
 import com.google.devtools.build.docgen.builtin.BuiltinProtos.Param;
 import com.google.devtools.build.docgen.builtin.BuiltinProtos.Type;
 import com.google.devtools.build.docgen.builtin.BuiltinProtos.Value;
-import com.google.devtools.build.docgen.starlark.AnnotParamDoc;
-import com.google.devtools.build.docgen.starlark.AnnotStarlarkConstructorMethodDoc;
-import com.google.devtools.build.docgen.starlark.AnnotStarlarkMethodDoc;
+import com.google.devtools.build.docgen.starlark.MemberDoc;
+import com.google.devtools.build.docgen.starlark.ParamDoc;
 import com.google.devtools.build.docgen.starlark.StarlarkDocExpander;
 import com.google.devtools.build.docgen.starlark.StarlarkDocPage;
 import com.google.devtools.common.options.OptionsParser;
@@ -34,6 +33,7 @@
 import java.lang.reflect.Method;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -59,10 +59,13 @@ private static void appendTypes(
     Type.Builder type = Type.newBuilder();
     type.setName(docPage.getName());
     type.setDoc(docPage.getDocumentation());
-    for (AnnotStarlarkMethodDoc meth : docPage.getJavaMethods()) {
+    // Sort members in case-sensitive name order.
+    for (MemberDoc member :
+        ImmutableList.sortedCopyOf(
+            Comparator.comparing(MemberDoc::getName), docPage.getMembers())) {
       // Constructors are exported as global symbols.
-      if (!(meth instanceof AnnotStarlarkConstructorMethodDoc)) {
-        Value.Builder value = collectMethodInfo(meth);
+      if (!member.isConstructor()) {
+        Value.Builder value = collectMethodInfo(member);
         if (type.getName().equals("native")) {
           // Methods from the native package are available as top level functions in BUILD files.
           value.setApiContext(ApiContext.BUILD);
@@ -89,8 +92,8 @@ private static void appendTypes(
   private static void appendGlobals(
       Builtins.Builder builtins,
       Map<String, Object> globals,
-      Map<String, AnnotStarlarkMethodDoc> globalToDoc,
-      Map<String, AnnotStarlarkConstructorMethodDoc> typeNameToConstructor,
+      Map<String, MemberDoc> globalToDoc,
+      Map<String, MemberDoc> typeNameToConstructor,
       ApiContext context) {
     for (Entry<String, Object> entry : globals.entrySet()) {
       String name = entry.getKey();
@@ -101,7 +104,7 @@ private static void appendGlobals(
 
       Value.Builder value = Value.newBuilder();
       if (obj instanceof StarlarkCallable) {
-        AnnotStarlarkMethodDoc meth = globalToDoc.get(name);
+        MemberDoc meth = globalToDoc.get(name);
         if (meth != null) {
           value = collectMethodInfo(meth);
         } else {
@@ -117,8 +120,7 @@ private static void appendGlobals(
             StarlarkMethod annotation = StarlarkAnnotations.getStarlarkMethod(selfCallMethod);
             value = valueFromAnnotation(annotation);
             // For constructors, we can also set the return type.
-            AnnotStarlarkConstructorMethodDoc constructor =
-                typeNameToConstructor.get(entry.getKey());
+            MemberDoc constructor = typeNameToConstructor.get(entry.getKey());
             if (constructor != null && value.hasCallable()) {
               value.getCallableBuilder().setReturnType(constructor.getReturnType());
             }
@@ -242,13 +244,13 @@ private static Value.Builder signatureToValue(Signature sig) {
     return value;
   }
 
-  private static Value.Builder collectMethodInfo(AnnotStarlarkMethodDoc meth) {
+  private static Value.Builder collectMethodInfo(MemberDoc meth) {
     Value.Builder field = Value.newBuilder();
     field.setName(meth.getShortName());
     field.setDoc(meth.getDocumentation());
     if (meth.isCallable()) {
       Callable.Builder callable = Callable.newBuilder();
-      for (AnnotParamDoc par : meth.getParams()) {
+      for (ParamDoc par : meth.getParams()) {
         Param.Builder param = newParam(par.getName(), par.getDefaultValue().isEmpty());
         param.setType(par.getType());
         param.setDoc(par.getDocumentation());
@@ -352,9 +354,9 @@ public static void main(String[] args) {
       Builtins.Builder builtins = Builtins.newBuilder();
 
       ImmutableList<StarlarkDocPage> globalPages = allDocPages.get(Category.GLOBAL_FUNCTION);
-      Map<String, AnnotStarlarkMethodDoc> globalToDoc = new HashMap<>();
+      Map<String, MemberDoc> globalToDoc = new HashMap<>();
       for (StarlarkDocPage globalPage : globalPages) {
-        for (AnnotStarlarkMethodDoc meth : globalPage.getJavaMethods()) {
+        for (MemberDoc meth : globalPage.getMembers()) {
           globalToDoc.put(meth.getShortName(), meth);
         }
       }
@@ -364,7 +366,7 @@ public static void main(String[] args) {
               .filter(e -> !e.getKey().equals(Category.GLOBAL_FUNCTION))
               .flatMap(e -> e.getValue().stream())
               .iterator();
-      Map<String, AnnotStarlarkConstructorMethodDoc> typeNameToConstructor = new HashMap<>();
+      Map<String, MemberDoc> typeNameToConstructor = new HashMap<>();
       while (typesIterator.hasNext()) {
         StarlarkDocPage typeDocPage = typesIterator.next();
         appendTypes(builtins, typeDocPage, symbols.getNativeRules());

src/main/java/com/google/devtools/build/docgen/StarlarkDocumentationCollector.java

@@ -213,7 +213,7 @@ private static void collectBuiltinMethods(
           javaMethod = selfCall;
         }
       }
-      builtinDoc.addMethod(
+      builtinDoc.addMember(
           new AnnotStarlarkOrdinaryMethodDoc(
               builtinDoc.getName(), javaMethod, starlarkMethod, expander));
     }
@@ -246,7 +246,7 @@ private static void collectGlobalMethods(
         // Only add non-constructor global library methods. Constructors are added later.
         // TODO(wyv): add a redirect instead
         if (!entry.getKey().isAnnotationPresent(StarlarkConstructor.class)) {
-          page.addMethod(
+          page.addMember(
               new AnnotStarlarkOrdinaryMethodDoc("", entry.getKey(), entry.getValue(), expander));
         }
       }

src/main/java/com/google/devtools/build/docgen/starlark/AnnotParamDoc.java

@@ -20,20 +20,9 @@
  * A class containing the documentation for a parameter of a {@link
  * net.starlark.java.annot.StarlarkMethod}-annotated Java method callable from Starlark.
  */
-public final class AnnotParamDoc extends StarlarkDoc {
-  /** Repesents the param kind, whether it's a normal param or *arg or **kwargs. */
-  public static enum Kind {
-    NORMAL,
-    // TODO: https://github.com/bazelbuild/stardoc/issues/225 - NORMAL needs to be split into
-    //   NORMAL and KEYWORD_ONLY, since EXTRA_KEYWORDS (or a `*` separator) go before keyword-only
-    //   params, not necessarily immediately before kwargs.
-    EXTRA_POSITIONALS,
-    EXTRA_KEYWORDS,
-  }
-
+public final class AnnotParamDoc extends ParamDoc {
   private final AnnotStarlarkMethodDoc method;
   private final Param param;
-  private final Kind kind;
   private final int paramIndex;
 
   public AnnotParamDoc(
@@ -42,22 +31,13 @@ public AnnotParamDoc(
       StarlarkDocExpander expander,
       Kind kind,
       int paramIndex) {
-    super(expander);
+    super(expander, kind);
     this.method = method;
     this.param = param;
-    this.kind = kind;
     this.paramIndex = paramIndex;
   }
 
-  /**
-   * Returns the string representing the type of this parameter with the link to the documentation
-   * for the type if available.
-   *
-   * <p>If the parameter type is Object, then returns the empty string. If the parameter type is not
-   * a generic, then this method returns a string representing the type name with a link to the
-   * documentation for the type if available. If the parameter type is a generic, then this method
-   * returns a string "CONTAINER of TYPE" (with HTML link markup).
-   */
+  @Override
   public String getType() {
     StringBuilder sb = new StringBuilder();
     if (param.allowedTypes().length == 0) {
@@ -86,10 +66,6 @@ public String getType() {
     return sb.toString();
   }
 
-  public Kind getKind() {
-    return kind;
-  }
-
   public AnnotStarlarkMethodDoc getMethod() {
     return method;
   }
@@ -99,6 +75,7 @@ public String getName() {
     return param.name();
   }
 
+  @Override
   public String getDefaultValue() {
     return param.defaultValue();
   }

src/main/java/com/google/devtools/build/docgen/starlark/AnnotStarlarkConstructorMethodDoc.java

@@ -34,6 +34,11 @@ public AnnotStarlarkConstructorMethodDoc(
     this.fullyQualifiedName = fullyQualifiedName;
   }
 
+  @Override
+  public boolean isConstructor() {
+    return true;
+  }
+
   @Override
   public String getName() {
     return fullyQualifiedName;

src/main/java/com/google/devtools/build/docgen/starlark/AnnotStarlarkMethodDoc.java

@@ -26,7 +26,7 @@
  * An abstract class containing documentation for a {@link StarlarkMethod}-annotated Java method
  * callable from Starlark.
  */
-public abstract class AnnotStarlarkMethodDoc extends StarlarkDoc {
+public abstract class AnnotStarlarkMethodDoc extends MemberDoc {
   protected final Method javaMethod;
   protected final StarlarkMethod annotation;
   protected final ImmutableList<AnnotParamDoc> params;
@@ -39,16 +39,12 @@ public AnnotStarlarkMethodDoc(
     this.params = determineParams();
   }
 
-  /** Returns whether the Starlark method is documented. */
+  @Override
   public final boolean documented() {
     return annotation.documented();
   }
 
-  /**
-   * Returns a string containing additional documentation about the method's return value.
-   *
-   * <p>Returns an empty string by default.
-   */
+  @Override
   public final String getReturnTypeExtraMessage() {
     if (annotation.allowReturnNones()) {
       return " May return <code>None</code>.\n";
@@ -61,35 +57,19 @@ public final Method getMethod() {
     return javaMethod;
   }
 
-  /** Returns a string containing a name for the method's return type. */
-  public abstract String getReturnType();
-
-  /**
-   * Returns whether a method can be called as a function.
-   *
-   * <p>E.g. ctx.label is not callable.
-   */
+  @Override
   public final boolean isCallable() {
     return !annotation.structField();
   }
 
-  /**
-   * Returns a string containing the method's name. GetName() returns the complete signature in case
-   * of overloaded methods. This is used to extract only the name of the method.
-   *
-   * <p>E.g. ctx.new_file is overloaded. In this case getName() returns "new_file(filename)", while
-   * getShortName() returns only "new_file".
-   */
-  public String getShortName() {
-    return getName();
-  }
-
   /** Returns a list containing the documentation for each of the method's parameters. */
+  @Override
   public final ImmutableList<AnnotParamDoc> getParams() {
     return params;
   }
 
-  private String getParameterString() {
+  @Override
+  protected String getParameterString() {
     List<String> argList = new ArrayList<>();
 
     boolean named = false;
@@ -120,7 +100,7 @@ private ImmutableList<AnnotParamDoc> determineParams() {
     for (int i = getStartIndexForParams(); i < annotation.parameters().length; i++) {
       Param param = annotation.parameters()[i];
       if (param.documented()) {
-        paramsBuilder.add(new AnnotParamDoc(this, param, expander, AnnotParamDoc.Kind.NORMAL, i));
+        paramsBuilder.add(new AnnotParamDoc(this, param, expander, ParamDoc.Kind.NORMAL, i));
       }
     }
     if (!annotation.extraPositionals().name().isEmpty()) {
@@ -129,7 +109,7 @@ private ImmutableList<AnnotParamDoc> determineParams() {
               this,
               annotation.extraPositionals(),
               expander,
-              AnnotParamDoc.Kind.EXTRA_POSITIONALS,
+              ParamDoc.Kind.EXTRA_POSITIONALS,
               /* paramIndex= */ -1));
     }
     if (!annotation.extraKeywords().name().isEmpty()) {
@@ -138,18 +118,12 @@ private ImmutableList<AnnotParamDoc> determineParams() {
               this,
               annotation.extraKeywords(),
               expander,
-              AnnotParamDoc.Kind.EXTRA_KEYWORDS,
+              ParamDoc.Kind.EXTRA_KEYWORDS,
               /* paramIndex= */ -1));
     }
     return paramsBuilder.build();
   }
 
-  /**
-   * Returns a string representing the method signature of the Starlark method, which contains HTML
-   * links to the documentation of parameter types if available.
-   */
-  public abstract String getSignature();
-
   protected String getSignature(String fullyQualifiedMethodName) {
     String args = isCallable() ? "(" + getParameterString() + ")" : "";
 

src/main/java/com/google/devtools/build/docgen/starlark/MemberDoc.java

@@ -0,0 +1,91 @@
+// Copyright 2025 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.docgen.starlark;
+
+import com.google.common.collect.ImmutableList;
+
+/**
+ * Documentation for a method or struct field of a Java class annotated with {@link
+ * net.starlark.java.annot.StarlarkBuiltin}, or for a field of a Starlark-defined struct.
+ */
+public abstract class MemberDoc extends StarlarkDoc {
+
+  protected MemberDoc(StarlarkDocExpander expander) {
+    super(expander);
+  }
+
+  /** Returns whether the value is documented. */
+  public abstract boolean documented();
+
+  /**
+   * Returns whether the value can be called as a function.
+   *
+   * <p>For example, {@code ctx.label} is not callable.
+   */
+  public abstract boolean isCallable();
+
+  /**
+   * For a callable value, returns the name for the return type; or the name of the value's own type
+   * otherwise.
+   */
+  public abstract String getReturnType();
+
+  /**
+   * For a callable value, returns a string containing additional documentation about the return
+   * value.
+   *
+   * <p>Returns an empty string by default.
+   */
+  public String getReturnTypeExtraMessage() {
+    return "";
+  }
+
+  /** Returns true if the value is callable and is a constructor of its type. */
+  public boolean isConstructor() {
+    return false;
+  }
+
+  /**
+   * Returns the value's name within its module.
+   *
+   * <p>In most cases, this is the same as {@link #getName}. The exception is for overloaded methods
+   * in a {@link net.starlark.java.annot.StarlarkBuiltin}-annotated Java class. In that case, this
+   * method would return the name of the method, while {@link #getName} would return the method
+   * signature with parameters, e.g. {@code method_name(arg1, arg2)}.
+   */
+  public String getShortName() {
+    return getName();
+  }
+
+  /**
+   * For a callable value, returns a list containing the documentation for each of the method's
+   * parameters; or an empty list otherwise.
+   */
+  public abstract ImmutableList<? extends ParamDoc> getParams();
+
+  /**
+   * For a callable value, returns the string representation of the parameters, for example {@code
+   * "arg1, arg2=None, **kwargs"}; or an empty string otherwise.
+   */
+  protected abstract String getParameterString();
+
+  /**
+   * For a callable value, returns the string representing the method signature of the Starlark
+   * method, which contains HTML links to the documentation of parameter types if available. For a
+   * non-callable value, returns the string representation of the value's type (with HTML links to
+   * the type's documentation, if available) and name.
+   */
+  public abstract String getSignature();
+}