Add a YAML header comment syntax for configuration of .mill- files, import $ivy, mill-version (#4969)

This PR adds a YAML 1.2 header comment syntax that subsumes many of the
ad-hoc configuration mechanisms we have today: `.mill-version`,
`.mill-opts`, `.mill-jvm-opts`, `.mill-jvm-version`, `import $ivy`, and
possibly more meta-build overrides or other configuration options in
future.

```scala
//| mill-version: 0.13.0
//| mill-jvm-version: 17
//| repositories: [$PWD_URI/custom-repo]
//| mvnDeps: [org.thymeleaf:thymeleaf:3.1.1.RELEASE]
package build
import mill._, javalib._
import org.thymeleaf.TemplateEngine
import org.thymeleaf.context.Context

object foo extends JavaModule {
  def htmlSnippet = Task {
    val context = new Context
    context.setVariable("heading", "hello")
    context.setVariable("paragraph", "world")
    new TemplateEngine().process(
      """<div><h1 th:text="${heading}"></h1><p th:text="${paragraph}"></p></div>""",
      context
    )
  }

  def resources = Task {
    os.write(Task.dest / "snippet.txt", htmlSnippet())
    super.resources() ++ Seq(PathRef(Task.dest))
  }
}
```

The YAML header for any `.mill` file must be a line-comment block
starting from the first line of the file and prefixed by `//|`. The
prefix is chosen to avoid conflicts with normal comments (`//`),
Scala-CLI directives (`//>`), and Java markdown comments (`///`), and to
be reminiscent of Scala `|`s commonly used in `.stripMargin`

We use YAML 1.2 (via `snakeyaml-engine`) rather than ScalaCLI directives
because:

- It can map more directly to Mill concepts: `mill-version`,
`mill-jvm-version`, `mvnDeps`, and other config keys can be directly in
YAML without any mapping.

- YAML's hierarchical list/dict/primitive data model maps clearly to
Mill's JSON data model for tasks, whereas Scala-CLI directives do not
have such a clean correspondence and will require that we manually
maintain an explicit mapping

- YAML will also be more familiar to non-Scala developers than
Scala-CLI's directive syntax, including all the edge cases that are
moderately well defined (quoting? escaping? comments? type coercions?
multi-line strings?)

- Processing by third-party tools will be easier: anyone can write a
Python script to strip the prefixes and parse out the YAML using pyyaml
or yamlcore, or their equivalents in Node.js/Ruby/Rust/Go/whatever,
whereas with Scala-CLI directives you are restricted to a single JVM
implementation without spec or documentation

All of these concerns were raised in the original Scala-CLI directives
discussion but were ignored
([link](https://contributors.scala-lang.org/t/pre-sip-using-directives/5700/5?u=lihaoyi),
[link](https://contributors.scala-lang.org/t/pre-sip-scala-cli-as-new-scala-command/5628/92?u=lihaoyi)),
so that brings us to today where we have to diverge from their syntax.

The choice of YAML over TOML was largely arbitrary. Python chose TOML
for their script-header metadata in
[PEP-723](https://peps.python.org/pep-0723/), whereas all Markdown
implementations
([Github](https://docs.github.com/en/contributing/writing-for-github-docs/using-yaml-frontmatter),
[RMarkdown](https://zsmith27.github.io/rmarkdown_crash-course/lesson-4-yaml-headers.html),
[Jekyll](https://jekyllrb.com/docs/front-matter/)) chose YAML for their
header-metadata syntax. YAML 1.2 seems to fix/mitigate most of the
problems with YAML 1.1 (e.g. the `no == false` thing). Kotlin has their
own syntax for file-level metadata
[`@file:JvmName("Foo")`](https://kotlinlang.org/docs/annotations.html#annotation-use-site-targets),
as does Swift [`@Metadata { @DocumentationExtension(mergeBehavior:
override) }`](https://www.swift.org/documentation/docc/metadata)

Builds on top of #4624

---------

Co-authored-by: Tobias Roeser <le.petit.fou@web.de>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>

Author: 3 people

.config/mill-version

@@ -1,3 +1,5 @@
+//| mill-version: 0.13.0-M1-43-b217bc
+
 package build
 // imports
 //import com.github.lolgab.mill.mima.Mima
@@ -230,6 +232,7 @@ object Deps {
   val hiltGradlePlugin = mvn"com.google.dagger:hilt-android-gradle-plugin:2.56"
 
   val sbt = mvn"org.scala-sbt:sbt:1.10.10"
+  val snakeyamlEngine = mvn"org.snakeyaml:snakeyaml-engine:2.9"
 
   object RuntimeDeps {
     val dokkaVersion = "2.0.0"

build.mill

@@ -1,3 +1,18 @@
+diff --git a/build.mill b/build.mill
+index d815508132a..ba62e147383 100644
+--- a/build.mill
++++ b/build.mill
+@@ -19,9 +19,8 @@ import mill.define.Cross
+ import scala.util.Properties
+ 
+ // plugins and dependencies
+-import $meta._
++//| meta: true
+ 
+-import $packages._
+ 
+ object Settings {
+   val pomOrg = "com.lihaoyi"
 diff --git a/mill-build/build.mill b/mill-build/build.mill
 index f32b3cbcf65..3fc9b0097c8 100644
 --- a/mill-build/build.mill

ci/mill-bootstrap.patch

@@ -14,9 +14,6 @@ import mill.T
 import mill.define.Cross
 import build.Deps
 
-// plugins and dependencies
-import $meta._
-
 /**
  * [[build.contrib]] contains user-contributed Mill plugins that satisfy
  * a range of use cases not covered by the core Mill builtins. `contrib`

contrib/package.mill

@@ -9,7 +9,7 @@ import mill.scalalib._
  * server, and the rest of the Mill codebase.
  */
 object `package` extends RootModule with build.MillPublishJavaModule with BuildInfo {
-  def buildInfoPackageName = "mill.cosntants"
+  def buildInfoPackageName = "mill.constants"
   def buildInfoMembers = Seq(BuildInfo.Value("millVersion", build.millVersion(), "Mill version."))
 
   object test extends JavaTests with TestModule.Junit4 {

core/constants/package.mill

@@ -9,6 +9,10 @@
  * home folder so you can find them
  */
 public class DebugLog {
+  public static synchronized void apply(String s) {
+    println(s);
+  }
+
   public static synchronized void println(String s) {
     Path path = Paths.get(System.getProperty("user.home"), "mill-debug-log.txt");
     try {

core/constants/src/mill/constants/DebugLog.java

@@ -21,10 +21,6 @@ public class EnvVars {
    */
   public static final String MILL_SERVER_TIMEOUT_MILLIS = "MILL_SERVER_TIMEOUT_MILLIS";
 
-  public static final String MILL_JVM_VERSION_PATH = "MILL_JVM_VERSION_PATH";
-  public static final String MILL_JVM_OPTS_PATH = "MILL_JVM_OPTS_PATH";
-  public static final String MILL_OPTS_PATH = "MILL_OPTS_PATH";
-
   /**
    * Output directory where Mill workers' state and Mill tasks output should be
    * written to

core/constants/src/mill/constants/EnvVars.java

@@ -7,6 +7,9 @@
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.Locale;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 public class Util {
 
@@ -71,4 +74,48 @@ public static String hexArray(byte[] bytes) {
   public static boolean hasConsole() {
     return hasConsole0;
   }
+
+  public static String readYamlHeader(java.nio.file.Path buildFile) throws java.io.IOException {
+    java.util.List<String> lines = java.nio.file.Files.readAllLines(buildFile);
+    String yamlString = lines.stream()
+        .filter(line -> line.startsWith("//|"))
+        .map(line -> line.substring(4)) // Remove the `//|` prefix
+        .collect(java.util.stream.Collectors.joining("\n"));
+
+    return yamlString;
+  }
+
+  private static String envInterpolatorPattern0 = "(\\$|[A-Z_][A-Z0-9_]*)";
+  private static Pattern envInterpolatorPattern =
+      Pattern.compile("\\$\\{" + envInterpolatorPattern0 + "\\}|\\$" + envInterpolatorPattern0);
+
+  /**
+   * Interpolate variables in the form of <code>${VARIABLE}</code> based on the given Map <code>env</code>.
+   * Missing vars will be replaced by the empty string.
+   */
+  public static String interpolateEnvVars(String input, Map<String, String> env0) {
+    Matcher matcher = envInterpolatorPattern.matcher(input);
+    // StringBuilder to store the result after replacing
+    StringBuffer result = new StringBuffer();
+
+    Map<String, String> env = new java.util.HashMap<>();
+    env.putAll(env0);
+
+    env.put("MILL_VERSION", mill.constants.BuildInfo.millVersion);
+    while (matcher.find()) {
+      String match = matcher.group(1);
+      if (match == null) match = matcher.group(2);
+      if (match.equals("$")) {
+        matcher.appendReplacement(result, "\\$");
+      } else {
+        String envVarValue;
+        mill.constants.DebugLog.println("MATCH " + match);
+        envVarValue = env.containsKey(match) ? env.get(match) : "";
+        matcher.appendReplacement(result, envVarValue);
+      }
+    }
+
+    matcher.appendTail(result); // Append the remaining part of the string
+    return result.toString();
+  }
 }

core/constants/src/mill/constants/Util.java

@@ -8,4 +8,5 @@ import mill._
  */
 object `package` extends RootModule with build.MillStableScalaModule {
   def moduleDeps = Seq(build.core.define, build.core.internal)
+  def mvnDeps = Seq(build.Deps.snakeyamlEngine)
 }

core/exec/package.mill

@@ -37,7 +37,8 @@ private[mill] case class Execution(
     systemExit: Int => Nothing,
     exclusiveSystemStreams: SystemStreams,
     getEvaluator: () => EvaluatorApi,
-    offline: Boolean
+    offline: Boolean,
+    headerData: String
 ) extends GroupExecution with AutoCloseable {
 
   // this (shorter) constructor is used from [[MillBuildBootstrap]] via reflection
@@ -57,7 +58,8 @@ private[mill] case class Execution(
       systemExit: Int => Nothing,
       exclusiveSystemStreams: SystemStreams,
       getEvaluator: () => EvaluatorApi,
-      offline: Boolean
+      offline: Boolean,
+      headerData: String
   ) = this(
     baseLogger,
     new JsonArrayLogger.ChromeProfile(os.Path(outPath) / millChromeProfile),
@@ -76,7 +78,8 @@ private[mill] case class Execution(
     systemExit,
     exclusiveSystemStreams,
     getEvaluator,
-    offline
+    offline,
+    headerData
   )
 
   def withBaseLogger(newBaseLogger: Logger) = this.copy(baseLogger = newBaseLogger)

core/exec/src/mill/exec/Execution.scala

@@ -33,6 +33,40 @@ private trait GroupExecution {
   def systemExit: Int => Nothing
   def exclusiveSystemStreams: SystemStreams
   def getEvaluator: () => EvaluatorApi
+  def headerData: String
+  lazy val parsedHeaderData: Map[String, ujson.Value] = {
+    import org.snakeyaml.engine.v2.api.{Load, LoadSettings}
+    val loaded = new Load(LoadSettings.builder().build()).loadFromString(headerData)
+    // recursively convert java data structure to ujson.Value
+    val envWithPwd = env ++ Seq(
+      "PWD" -> workspace.toString,
+      "PWD_URI" -> workspace.toNIO.toUri.toString
+    )
+    def rec(x: Any): ujson.Value = {
+      import collection.JavaConverters._
+      x match {
+        case d: java.util.Date => ujson.Str(d.toString)
+        case s: String => ujson.Str(mill.constants.Util.interpolateEnvVars(s, envWithPwd.asJava))
+        case d: Double => ujson.Num(d)
+        case d: Int => ujson.Num(d)
+        case d: Long => ujson.Num(d)
+        case true => ujson.True
+        case false => ujson.False
+        case null => ujson.Null
+        case m: java.util.Map[Object, Object] =>
+          import collection.JavaConverters._
+          val scalaMap = m.asScala
+          ujson.Obj.from(scalaMap.map { case (k, v) => (k.toString, rec(v)) })
+        case l: java.util.List[Object] =>
+          import collection.JavaConverters._
+          val scalaList: collection.Seq[Object] = l.asScala
+          ujson.Arr.from(scalaList.map(rec))
+      }
+    }
+
+    rec(loaded).objOpt.getOrElse(Map.empty[String, ujson.Value]).toMap
+  }
+
   lazy val constructorHashSignatures: Map[String, Seq[(String, Int)]] =
     CodeSigUtils.constructorHashSignatures(codeSignatures)
 
@@ -85,79 +119,95 @@ private trait GroupExecution {
       terminal match {
 
         case labelled: NamedTask[_] =>
-          val out = if (!labelled.ctx.external) outPath else externalOutPath
-          val paths = ExecutionPaths.resolve(out, labelled.ctx.segments)
-          val cached = loadCachedJson(logger, inputsHash, labelled, paths)
-
-          // `cached.isEmpty` means worker metadata file removed by user so recompute the worker
-          val upToDateWorker = loadUpToDateWorker(logger, inputsHash, labelled, cached.isEmpty)
-
-          val cachedValueAndHash =
-            upToDateWorker.map((_, inputsHash))
-              .orElse(cached.flatMap { case (inputHash, valOpt, valueHash) =>
-                valOpt.map((_, valueHash))
-              })
-
-          cachedValueAndHash match {
-            case Some((v, hashCode)) =>
-              val res = ExecResult.Success((v, hashCode))
-              val newResults: Map[Task[?], ExecResult[(Val, Int)]] =
-                Map(labelled -> res)
-
+          labelled.ctx.segments.value match {
+            case Seq(Segment.Label(single)) if parsedHeaderData.contains(single) =>
+              val jsonData = parsedHeaderData(single)
+              val resultData = upickle.default.read[Any](jsonData)(
+                using labelled.readWriterOpt.get.asInstanceOf[upickle.default.Reader[Any]]
+              )
               GroupExecution.Results(
-                newResults,
+                Map(labelled -> ExecResult.Success(Val(resultData), resultData.##)),
                 Nil,
                 cached = true,
                 inputsHash,
                 -1,
-                valueHashChanged = false
+                false
               )
-
             case _ =>
-              // uncached
-              if (!labelled.persistent) os.remove.all(paths.dest)
-
-              val (newResults, newEvaluated) =
-                executeGroup(
-                  group = group,
-                  results = results,
-                  inputsHash = inputsHash,
-                  paths = Some(paths),
-                  maybeTargetLabel = Some(terminal.toString),
-                  counterMsg = countMsg,
-                  reporter = zincProblemReporter,
-                  testReporter = testReporter,
-                  logger = logger,
-                  executionContext = executionContext,
-                  exclusive = exclusive,
-                  isCommand = labelled.isInstanceOf[Command[?]],
-                  deps = deps,
-                  offline = offline
-                )
-
-              val valueHash = newResults(labelled) match {
-                case ExecResult.Success((v, _)) =>
-                  val valueHash = getValueHash(v, terminal, inputsHash)
-                  handleTaskResult(v, valueHash, paths.meta, inputsHash, labelled)
-                  valueHash
+              val out = if (!labelled.ctx.external) outPath else externalOutPath
+              val paths = ExecutionPaths.resolve(out, labelled.ctx.segments)
+              val cached = loadCachedJson(logger, inputsHash, labelled, paths)
+
+              // `cached.isEmpty` means worker metadata file removed by user so recompute the worker
+              val upToDateWorker = loadUpToDateWorker(logger, inputsHash, labelled, cached.isEmpty)
+
+              val cachedValueAndHash =
+                upToDateWorker.map((_, inputsHash))
+                  .orElse(cached.flatMap { case (inputHash, valOpt, valueHash) =>
+                    valOpt.map((_, valueHash))
+                  })
+
+              cachedValueAndHash match {
+                case Some((v, hashCode)) =>
+                  val res = ExecResult.Success((v, hashCode))
+                  val newResults: Map[Task[?], ExecResult[(Val, Int)]] =
+                    Map(labelled -> res)
+
+                  GroupExecution.Results(
+                    newResults,
+                    Nil,
+                    cached = true,
+                    inputsHash,
+                    -1,
+                    valueHashChanged = false
+                  )
 
                 case _ =>
-                  // Wipe out any cached meta.json file that exists, so
-                  // a following run won't look at the cached metadata file and
-                  // assume it's associated with the possibly-borked state of the
-                  // destPath after an evaluation failure.
-                  os.remove.all(paths.meta)
-                  0
-              }
+                  // uncached
+                  if (!labelled.persistent) os.remove.all(paths.dest)
+
+                  val (newResults, newEvaluated) =
+                    executeGroup(
+                      group = group,
+                      results = results,
+                      inputsHash = inputsHash,
+                      paths = Some(paths),
+                      maybeTargetLabel = Some(terminal.toString),
+                      counterMsg = countMsg,
+                      reporter = zincProblemReporter,
+                      testReporter = testReporter,
+                      logger = logger,
+                      executionContext = executionContext,
+                      exclusive = exclusive,
+                      isCommand = labelled.isInstanceOf[Command[?]],
+                      deps = deps,
+                      offline = offline
+                    )
+
+                  val valueHash = newResults(labelled) match {
+                    case ExecResult.Success((v, _)) =>
+                      val valueHash = getValueHash(v, terminal, inputsHash)
+                      handleTaskResult(v, valueHash, paths.meta, inputsHash, labelled)
+                      valueHash
+
+                    case _ =>
+                      // Wipe out any cached meta.json file that exists, so
+                      // a following run won't look at the cached metadata file and
+                      // assume it's associated with the possibly-borked state of the
+                      // destPath after an evaluation failure.
+                      os.remove.all(paths.meta)
+                      0
+                  }
 
-              GroupExecution.Results(
-                newResults,
-                newEvaluated.toSeq,
-                cached = if (labelled.isInstanceOf[InputImpl[?]]) null else false,
-                inputsHash,
-                cached.map(_._1).getOrElse(-1),
-                !cached.map(_._3).contains(valueHash)
-              )
+                  GroupExecution.Results(
+                    newResults,
+                    newEvaluated.toSeq,
+                    cached = if (labelled.isInstanceOf[InputImpl[?]]) null else false,
+                    inputsHash,
+                    cached.map(_._1).getOrElse(-1),
+                    !cached.map(_._3).contains(valueHash)
+                  )
+              }
           }
         case task =>
           val (newResults, newEvaluated) = executeGroup(
@@ -328,7 +378,7 @@ private trait GroupExecution {
   // invalidate workers in the scenario where a worker classloader is
   // re-created - so the worker *class* changes - but the *value* inputs to the
   // worker does not change. This typically happens when the worker class is
-  // brought in via `import $ivy`, since the class then comes from the
+  // brought in via `//| mvnDeps:`, since the class then comes from the
   // non-bootstrap classloader which can be re-created when the `build.mill` file
   // changes.
   //

core/exec/src/mill/exec/GroupExecution.scala

@@ -18,6 +18,7 @@ object `package` extends RootModule with build.MillPublishScalaModule {
     build.Deps.mainargs,
     build.Deps.upickle,
     build.Deps.pprint,
-    build.Deps.fansi
+    build.Deps.fansi,
+    build.Deps.snakeyamlEngine
   )
 }