Add a YAML header comment syntax for configuration of .mill- files, import $ivy, mill-version (#4969)

This PR adds a YAML 1.2 header comment syntax that subsumes many of the
ad-hoc configuration mechanisms we have today: `.mill-version`,
`.mill-opts`, `.mill-jvm-opts`, `.mill-jvm-version`, `import $ivy`, and
possibly more meta-build overrides or other configuration options in
future.

```scala
//| mill-version: 0.13.0
//| mill-jvm-version: 17
//| repositories: [$PWD_URI/custom-repo]
//| mvnDeps: [org.thymeleaf:thymeleaf:3.1.1.RELEASE]
package build
import mill._, javalib._
import org.thymeleaf.TemplateEngine
import org.thymeleaf.context.Context

object foo extends JavaModule {
  def htmlSnippet = Task {
    val context = new Context
    context.setVariable("heading", "hello")
    context.setVariable("paragraph", "world")
    new TemplateEngine().process(
      """<div><h1 th:text="${heading}"></h1><p th:text="${paragraph}"></p></div>""",
      context
    )
  }

  def resources = Task {
    os.write(Task.dest / "snippet.txt", htmlSnippet())
    super.resources() ++ Seq(PathRef(Task.dest))
  }
}
```

The YAML header for any `.mill` file must be a line-comment block
starting from the first line of the file and prefixed by `//|`. The
prefix is chosen to avoid conflicts with normal comments (`//`),
Scala-CLI directives (`//>`), and Java markdown comments (`///`), and to
be reminiscent of Scala `|`s commonly used in `.stripMargin`

We use YAML 1.2 (via `snakeyaml-engine`) rather than ScalaCLI directives
because:

- It can map more directly to Mill concepts: `mill-version`,
`mill-jvm-version`, `mvnDeps`, and other config keys can be directly in
YAML without any mapping.

- YAML's hierarchical list/dict/primitive data model maps clearly to
Mill's JSON data model for tasks, whereas Scala-CLI directives do not
have such a clean correspondence and will require that we manually
maintain an explicit mapping

- YAML will also be more familiar to non-Scala developers than
Scala-CLI's directive syntax, including all the edge cases that are
moderately well defined (quoting? escaping? comments? type coercions?
multi-line strings?)

- Processing by third-party tools will be easier: anyone can write a
Python script to strip the prefixes and parse out the YAML using pyyaml
or yamlcore, or their equivalents in Node.js/Ruby/Rust/Go/whatever,
whereas with Scala-CLI directives you are restricted to a single JVM
implementation without spec or documentation

All of these concerns were raised in the original Scala-CLI directives
discussion but were ignored
([link](https://contributors.scala-lang.org/t/pre-sip-using-directives/5700/5?u=lihaoyi),
[link](https://contributors.scala-lang.org/t/pre-sip-scala-cli-as-new-scala-command/5628/92?u=lihaoyi)),
so that brings us to today where we have to diverge from their syntax.

The choice of YAML over TOML was largely arbitrary. Python chose TOML
for their script-header metadata in
[PEP-723](https://peps.python.org/pep-0723/), whereas all Markdown
implementations
([Github](https://docs.github.com/en/contributing/writing-for-github-docs/using-yaml-frontmatter),
[RMarkdown](https://zsmith27.github.io/rmarkdown_crash-course/lesson-4-yaml-headers.html),
[Jekyll](https://jekyllrb.com/docs/front-matter/)) chose YAML for their
header-metadata syntax. YAML 1.2 seems to fix/mitigate most of the
problems with YAML 1.1 (e.g. the `no == false` thing). Kotlin has their
own syntax for file-level metadata
[`@file:JvmName("Foo")`](https://kotlinlang.org/docs/annotations.html#annotation-use-site-targets),
as does Swift [`@Metadata { @DocumentationExtension(mergeBehavior:
override) }`](https://www.swift.org/documentation/docc/metadata)

Builds on top of #4624

---------

Co-authored-by: Tobias Roeser <le.petit.fou@web.de>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>

Author: 3 people

.config/mill-version

@@ -1,3 +1,5 @@
+//| mill-version: 0.13.0-M1-43-b217bc
+
 package build
 // imports
 //import com.github.lolgab.mill.mima.Mima
@@ -230,6 +232,7 @@ object Deps {
   val hiltGradlePlugin = mvn"com.google.dagger:hilt-android-gradle-plugin:2.56"
 
   val sbt = mvn"org.scala-sbt:sbt:1.10.10"
+  val snakeyamlEngine = mvn"org.snakeyaml:snakeyaml-engine:2.9"
 
   object RuntimeDeps {
     val dokkaVersion = "2.0.0"

build.mill

@@ -1,3 +1,18 @@
+diff --git a/build.mill b/build.mill
+index d815508132a..ba62e147383 100644
+--- a/build.mill
++++ b/build.mill
+@@ -19,9 +19,8 @@ import mill.define.Cross
+ import scala.util.Properties
+ 
+ // plugins and dependencies
+-import $meta._
++//| meta: true
+ 
+-import $packages._
+ 
+ object Settings {
+   val pomOrg = "com.lihaoyi"
 diff --git a/mill-build/build.mill b/mill-build/build.mill
 index f32b3cbcf65..3fc9b0097c8 100644
 --- a/mill-build/build.mill

ci/mill-bootstrap.patch

@@ -14,9 +14,6 @@ import mill.T
 import mill.define.Cross
 import build.Deps
 
-// plugins and dependencies
-import $meta._
-
 /**
  * [[build.contrib]] contains user-contributed Mill plugins that satisfy
  * a range of use cases not covered by the core Mill builtins. `contrib`

contrib/package.mill

@@ -9,7 +9,7 @@ import mill.scalalib._
  * server, and the rest of the Mill codebase.
  */
 object `package` extends RootModule with build.MillPublishJavaModule with BuildInfo {
-  def buildInfoPackageName = "mill.cosntants"
+  def buildInfoPackageName = "mill.constants"
   def buildInfoMembers = Seq(BuildInfo.Value("millVersion", build.millVersion(), "Mill version."))
 
   object test extends JavaTests with TestModule.Junit4 {

core/constants/package.mill

@@ -9,6 +9,10 @@
  * home folder so you can find them
  */
 public class DebugLog {
+  public static synchronized void apply(String s) {
+    println(s);
+  }
+
   public static synchronized void println(String s) {
     Path path = Paths.get(System.getProperty("user.home"), "mill-debug-log.txt");
     try {

core/constants/src/mill/constants/DebugLog.java

@@ -21,10 +21,6 @@ public class EnvVars {
    */
   public static final String MILL_SERVER_TIMEOUT_MILLIS = "MILL_SERVER_TIMEOUT_MILLIS";
 
-  public static final String MILL_JVM_VERSION_PATH = "MILL_JVM_VERSION_PATH";
-  public static final String MILL_JVM_OPTS_PATH = "MILL_JVM_OPTS_PATH";
-  public static final String MILL_OPTS_PATH = "MILL_OPTS_PATH";
-
   /**
    * Output directory where Mill workers' state and Mill tasks output should be
    * written to

core/constants/src/mill/constants/EnvVars.java

@@ -7,6 +7,9 @@
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.Locale;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 public class Util {
 
@@ -71,4 +74,48 @@ public static String hexArray(byte[] bytes) {
   public static boolean hasConsole() {
     return hasConsole0;
   }
+
+  public static String readYamlHeader(java.nio.file.Path buildFile) throws java.io.IOException {
+    java.util.List<String> lines = java.nio.file.Files.readAllLines(buildFile);
+    String yamlString = lines.stream()
+        .filter(line -> line.startsWith("//|"))
+        .map(line -> line.substring(4)) // Remove the `//|` prefix
+        .collect(java.util.stream.Collectors.joining("\n"));
+
+    return yamlString;
+  }
+
+  private static String envInterpolatorPattern0 = "(\\$|[A-Z_][A-Z0-9_]*)";
+  private static Pattern envInterpolatorPattern =
+      Pattern.compile("\\$\\{" + envInterpolatorPattern0 + "\\}|\\$" + envInterpolatorPattern0);
+
+  /**
+   * Interpolate variables in the form of <code>${VARIABLE}</code> based on the given Map <code>env</code>.
+   * Missing vars will be replaced by the empty string.
+   */
+  public static String interpolateEnvVars(String input, Map<String, String> env0) {
+    Matcher matcher = envInterpolatorPattern.matcher(input);
+    // StringBuilder to store the result after replacing
+    StringBuffer result = new StringBuffer();
+
+    Map<String, String> env = new java.util.HashMap<>();
+    env.putAll(env0);
+
+    env.put("MILL_VERSION", mill.constants.BuildInfo.millVersion);
+    while (matcher.find()) {
+      String match = matcher.group(1);
+      if (match == null) match = matcher.group(2);
+      if (match.equals("$")) {
+        matcher.appendReplacement(result, "\\$");
+      } else {
+        String envVarValue;
+        mill.constants.DebugLog.println("MATCH " + match);
+        envVarValue = env.containsKey(match) ? env.get(match) : "";
+        matcher.appendReplacement(result, envVarValue);
+      }
+    }
+
+    matcher.appendTail(result); // Append the remaining part of the string
+    return result.toString();
+  }
 }

core/constants/src/mill/constants/Util.java

@@ -8,4 +8,5 @@ import mill._
  */
 object `package` extends RootModule with build.MillStableScalaModule {
   def moduleDeps = Seq(build.core.define, build.core.internal)
+  def mvnDeps = Seq(build.Deps.snakeyamlEngine)
 }

core/exec/package.mill

@@ -37,7 +37,8 @@ private[mill] case class Execution(
     systemExit: Int => Nothing,
     exclusiveSystemStreams: SystemStreams,
     getEvaluator: () => EvaluatorApi,
-    offline: Boolean
+    offline: Boolean,
+    headerData: String
 ) extends GroupExecution with AutoCloseable {
 
   // this (shorter) constructor is used from [[MillBuildBootstrap]] via reflection
@@ -57,7 +58,8 @@ private[mill] case class Execution(
       systemExit: Int => Nothing,
       exclusiveSystemStreams: SystemStreams,
       getEvaluator: () => EvaluatorApi,
-      offline: Boolean
+      offline: Boolean,
+      headerData: String
   ) = this(
     baseLogger,
     new JsonArrayLogger.ChromeProfile(os.Path(outPath) / millChromeProfile),
@@ -76,7 +78,8 @@ private[mill] case class Execution(
     systemExit,
     exclusiveSystemStreams,
     getEvaluator,
-    offline
+    offline,
+    headerData
   )
 
   def withBaseLogger(newBaseLogger: Logger) = this.copy(baseLogger = newBaseLogger)