First step to pipelining support - enable reading Java symbols from TASTy (#19074)

This PR contains the minimal forward incompatible changes of pipelining
- reading Java and outline symbols from TASTy.
The other changes to implement pipelining have no impact on TASTy or the
standard library - so can come in a patch release.

To test reading TASTy produced from Java defined classes, we add two
private flags `-Yjava-tasty` and `-Yjava-tasty-output`, which are not
expected to be used by any build tool. The tests allow us to write just
the java signatures to TASTy, package them in a jar, and then read the
Java TASTy from the classpath.

- Keep Java compilation units up to Pickler phase if `-Yjava-tasty` is
set. Skip phases for Java when not needed.
- Add `JAVAattr` and `OUTLINEattr` TASTy attributes, `ELIDED` tree tag.
`ELIDED` trees are pickled as rhs of java term definitions. `ELIDED`
trees can only be unpickled if `OUTLINEattr` is present.
- Java units will set the `JAVAattr` TASTy attribute. As currently we
outline parse Java files we also set the `OUTLINEattr`.
- In the future we might expand `OUTLINEattr` to include outline Scala
typing.
- `OUTLINEattr` and `JAVAattr` do not need any special flags to _read_
from the classpath, however to read outline tasty in the `-from-tasty`
mode (so `tasty-inspector` and `scaladoc` are included) we do require an
explicit `-Yallow-outline-from-tasty`, as method bodies are required for
full functionality.
- write java tasty files to a special jar, set with
`-Yjava-tasty-output`
  this option is for testing purposes only.

Fix sealedDescendants method for Java Enums.
- Rename `JavaEnumTrait` flags to `JavaEnum` (reflecting the actual
flags set)
- test java enum in `SealedDescendantsTest`

fixes #15908

Author: bishabosha

compiler/src/dotty/tools/backend/jvm/BCodeHelpers.scala

@@ -292,7 +292,7 @@ trait BCodeHelpers extends BCodeIdiomatic {
           }
         case Ident(nme.WILDCARD) =>
           // An underscore argument indicates that we want to use the default value for this parameter, so do not emit anything
-        case t: tpd.RefTree if t.symbol.owner.linkedClass.isAllOf(JavaEnumTrait) =>
+        case t: tpd.RefTree if t.symbol.owner.linkedClass.isAllOf(JavaEnum) =>
           val edesc = innerClasesStore.typeDescriptor(t.tpe) // the class descriptor of the enumeration class.
           val evalue = t.symbol.javaSimpleName // value the actual enumeration value.
           av.visitEnum(name, edesc, evalue)

compiler/src/dotty/tools/backend/jvm/BTypesFromSymbols.scala

@@ -304,7 +304,7 @@ class BTypesFromSymbols[I <: DottyBackendInterface](val int: I, val frontendAcce
       .addFlagIf(sym.is(Bridge), ACC_BRIDGE | ACC_SYNTHETIC)
       .addFlagIf(sym.is(Artifact), ACC_SYNTHETIC)
       .addFlagIf(sym.isClass && !sym.isInterface, ACC_SUPER)
-      .addFlagIf(sym.isAllOf(JavaEnumTrait), ACC_ENUM)
+      .addFlagIf(sym.isAllOf(JavaEnum), ACC_ENUM)
       .addFlagIf(sym.is(JavaVarargs), ACC_VARARGS)
       .addFlagIf(sym.is(Synchronized), ACC_SYNCHRONIZED)
       .addFlagIf(sym.isDeprecated, ACC_DEPRECATED)

compiler/src/dotty/tools/dotc/CompilationUnit.scala

@@ -19,7 +19,7 @@ import scala.annotation.internal.sharable
 import scala.util.control.NoStackTrace
 import transform.MacroAnnotations
 
-class CompilationUnit protected (val source: SourceFile) {
+class CompilationUnit protected (val source: SourceFile, val info: CompilationUnitInfo | Null) {
 
   override def toString: String = source.toString
 
@@ -30,6 +30,13 @@ class CompilationUnit protected (val source: SourceFile) {
   /** Is this the compilation unit of a Java file */
   def isJava: Boolean = source.file.name.endsWith(".java")
 
+  /** Is this the compilation unit of a Java file, or TASTy derived from a Java file */
+  def typedAsJava = isJava || {
+    val infoNN = info
+    infoNN != null && infoNN.tastyInfo.exists(_.attributes.isJava)
+  }
+
+
   /** The source version for this unit, as determined by a language import */
   var sourceVersion: Option[SourceVersion] = None
 
@@ -106,7 +113,7 @@ class CompilationUnit protected (val source: SourceFile) {
     myAssignmentSpans.nn
 }
 
-@sharable object NoCompilationUnit extends CompilationUnit(NoSource) {
+@sharable object NoCompilationUnit extends CompilationUnit(NoSource, info = null) {
 
   override def isJava: Boolean = false
 
@@ -122,13 +129,14 @@ object CompilationUnit {
 
   /** Make a compilation unit for top class `clsd` with the contents of the `unpickled` tree */
   def apply(clsd: ClassDenotation, unpickled: Tree, forceTrees: Boolean)(using Context): CompilationUnit =
-    val file = clsd.symbol.associatedFile.nn
-    apply(SourceFile(file, Array.empty[Char]), unpickled, forceTrees)
+    val compilationUnitInfo = clsd.symbol.compilationUnitInfo.nn
+    val file = compilationUnitInfo.associatedFile
+    apply(SourceFile(file, Array.empty[Char]), unpickled, forceTrees, compilationUnitInfo)
 
   /** Make a compilation unit, given picked bytes and unpickled tree */
-  def apply(source: SourceFile, unpickled: Tree, forceTrees: Boolean)(using Context): CompilationUnit = {
+  def apply(source: SourceFile, unpickled: Tree, forceTrees: Boolean, info: CompilationUnitInfo)(using Context): CompilationUnit = {
     assert(!unpickled.isEmpty, unpickled)
-    val unit1 = new CompilationUnit(source)
+    val unit1 = new CompilationUnit(source, info)
     unit1.tpdTree = unpickled
     if (forceTrees) {
       val force = new Force
@@ -156,7 +164,8 @@ object CompilationUnit {
         NoSource
       }
       else source
-    new CompilationUnit(src)
+    val info = if src.exists then CompilationUnitInfo(src.file) else null
+    new CompilationUnit(src, info)
   }
 
   /** Force the tree to be loaded */

compiler/src/dotty/tools/dotc/ast/Desugar.scala

@@ -17,6 +17,7 @@ import reporting.*
 import annotation.constructorOnly
 import printing.Formatting.hl
 import config.Printers
+import parsing.Parsers
 
 import scala.annotation.internal.sharable
 import scala.annotation.threadUnsafe
@@ -143,8 +144,13 @@ object desugar {
 
   /** A value definition copied from `vdef` with a tpt typetree derived from it */
   def derivedTermParam(vdef: ValDef)(using Context): ValDef =
+    derivedTermParam(vdef, vdef.unforcedRhs)
+
+  def derivedTermParam(vdef: ValDef, rhs: LazyTree)(using Context): ValDef =
     cpy.ValDef(vdef)(
-      tpt = DerivedFromParamTree().withSpan(vdef.tpt.span).watching(vdef))
+      tpt = DerivedFromParamTree().withSpan(vdef.tpt.span).watching(vdef),
+      rhs = rhs
+    )
 
 // ----- Desugar methods -------------------------------------------------
 
@@ -544,8 +550,11 @@ object desugar {
       constrTparams.zipWithConserve(impliedTparams)((tparam, impliedParam) =>
         derivedTypeParam(tparam).withAnnotations(impliedParam.mods.annotations))
     val derivedVparamss =
-      constrVparamss.nestedMap(vparam =>
-        derivedTermParam(vparam).withAnnotations(Nil))
+      constrVparamss.nestedMap: vparam =>
+        val derived =
+          if ctx.compilationUnit.isJava then derivedTermParam(vparam, Parsers.unimplementedExpr)
+          else derivedTermParam(vparam)
+        derived.withAnnotations(Nil)
 
     val constr = cpy.DefDef(constr1)(paramss = joinParams(constrTparams, constrVparamss))
 

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -8,7 +8,7 @@ import dotty.tools.dotc.config.Settings.{Setting, SettingGroup}
 import dotty.tools.dotc.config.SourceVersion
 import dotty.tools.dotc.core.Contexts.*
 import dotty.tools.dotc.rewrites.Rewrites
-import dotty.tools.io.{AbstractFile, Directory, JDK9Reflectors, PlainDirectory}
+import dotty.tools.io.{AbstractFile, Directory, JDK9Reflectors, PlainDirectory, NoAbstractFile}
 import Setting.ChoiceWithHelp
 
 import scala.util.chaining.*
@@ -433,4 +433,9 @@ private sealed trait YSettings:
   val YforceInlineWhileTyping: Setting[Boolean] = BooleanSetting("-Yforce-inline-while-typing", "Make non-transparent inline methods inline when typing. Emulates the old inlining behavior of 3.0.0-M3.")
 
   val YdebugMacros: Setting[Boolean] = BooleanSetting("-Ydebug-macros", "Show debug info when quote pattern match fails")
+
+  // Pipeline compilation options
+  val YjavaTasty: Setting[Boolean] = BooleanSetting("-Yjava-tasty", "Pickler phase should compute pickles for .java defined symbols for use by build tools")
+  val YjavaTastyOutput: Setting[AbstractFile] = OutputSetting("-Yjava-tasty-output", "directory|jar", "(Internal use only!) destination for generated .tasty files containing Java type signatures.", NoAbstractFile)
+  val YallowOutlineFromTasty: Setting[Boolean] = BooleanSetting("-Yallow-outline-from-tasty", "Allow outline TASTy to be loaded with the -from-tasty option.")
 end YSettings

compiler/src/dotty/tools/dotc/core/Definitions.scala

@@ -996,6 +996,7 @@ class Definitions {
   @tu lazy val AnnotationClass: ClassSymbol = requiredClass("scala.annotation.Annotation")
   @tu lazy val StaticAnnotationClass: ClassSymbol = requiredClass("scala.annotation.StaticAnnotation")
   @tu lazy val RefiningAnnotationClass: ClassSymbol = requiredClass("scala.annotation.RefiningAnnotation")
+  @tu lazy val JavaAnnotationClass: ClassSymbol = requiredClass("java.lang.annotation.Annotation")
 
   // Annotation classes
   @tu lazy val AllowConversionsAnnot: ClassSymbol = requiredClass("scala.annotation.allowConversions")

compiler/src/dotty/tools/dotc/core/Flags.scala

@@ -576,7 +576,7 @@ object Flags {
   val InlineMethod: FlagSet                  = Inline | Method
   val InlineParam: FlagSet                   = Inline | Param
   val InlineByNameProxy: FlagSet             = InlineProxy | Method
-  val JavaEnumTrait: FlagSet                 = JavaDefined | Enum                             // A Java enum trait
+  val JavaEnum: FlagSet                      = JavaDefined | Enum                             // A Java enum trait
   val JavaEnumValue: FlagSet                 = JavaDefined | EnumValue                        // A Java enum value
   val StaticProtected: FlagSet               = JavaDefined | JavaStatic | Protected           // Java symbol which is `protected` and `static`
   val JavaModule: FlagSet                    = JavaDefined | Module                           // A Java companion object

compiler/src/dotty/tools/dotc/core/Phases.scala

@@ -333,16 +333,25 @@ object Phases {
     def subPhases: List[Run.SubPhase] = Nil
     final def traversals: Int = if subPhases.isEmpty then 1 else subPhases.length
 
+    /** skip the phase for a Java compilation unit, may depend on -Yjava-tasty */
+    def skipIfJava(using Context): Boolean = true
+
     /** @pre `isRunnable` returns true */
     def run(using Context): Unit
 
     /** @pre `isRunnable` returns true */
     def runOn(units: List[CompilationUnit])(using runCtx: Context): List[CompilationUnit] =
       val buf = List.newBuilder[CompilationUnit]
+      // factor out typedAsJava check when not needed
+      val doSkipJava = ctx.settings.YjavaTasty.value && this <= picklerPhase && skipIfJava
       for unit <- units do
         given unitCtx: Context = runCtx.fresh.setPhase(this.start).setCompilationUnit(unit).withRootImports
         if ctx.run.enterUnit(unit) then
-          try run
+          try
+            if doSkipJava && unit.typedAsJava then
+              ()
+            else
+              run
           catch case ex: Throwable if !ctx.run.enrichedErrorMessage =>
             println(ctx.run.enrichErrorMessage(s"unhandled exception while running $phaseName on $unit"))
             throw ex

compiler/src/dotty/tools/dotc/core/SymDenotations.scala

@@ -1695,7 +1695,7 @@ object SymDenotations {
             c.ensureCompleted()
       end completeChildrenIn
 
-      if is(Sealed) || isAllOf(JavaEnumTrait) then
+      if is(Sealed) || isAllOf(JavaEnum) && isClass then
         if !is(ChildrenQueried) then
           // Make sure all visible children are completed, so that
           // they show up in Child annotations. A possible child is visible if it

compiler/src/dotty/tools/dotc/core/tasty/Attributes.scala

@@ -11,6 +11,8 @@ class Attributes private[tasty](
   def explicitNulls: Boolean = booleanTags(EXPLICITNULLSattr)
   def captureChecked: Boolean = booleanTags(CAPTURECHECKEDattr)
   def withPureFuns: Boolean = booleanTags(WITHPUREFUNSattr)
+  def isJava: Boolean = booleanTags(JAVAattr)
+  def isOutline: Boolean = booleanTags(OUTLINEattr)
 }
 
 object Attributes:
@@ -19,12 +21,16 @@ object Attributes:
     explicitNulls: Boolean,
     captureChecked: Boolean,
     withPureFuns: Boolean,
+    isJava: Boolean,
+    isOutline: Boolean,
   ): Attributes =
     val booleanTags = BitSet.newBuilder
     if scala2StandardLibrary then booleanTags += SCALA2STANDARDLIBRARYattr
     if explicitNulls then booleanTags += EXPLICITNULLSattr
     if captureChecked then booleanTags += CAPTURECHECKEDattr
     if withPureFuns then booleanTags += WITHPUREFUNSattr
+    if isJava then booleanTags += JAVAattr
+    if isOutline then booleanTags += OUTLINEattr
     new Attributes(booleanTags.result())
   end apply