Experimental support for Markdown (#1268)

* Markdown support PoC

* Markdown support fixes
 - Hide markdown support behind a feature flag (`--enable-markdown`)
 - still process explicitly passed Markdown files
 - remove some PoC simplifications and workarounds
 - misc refactors

* Add Markdown guide

Co-authored-by: Marcin K <github@anubis.email>

Author: Gedochao

modules/build/src/main/scala/scala/build/CrossSources.scala

@@ -156,7 +156,8 @@ object CrossSources {
         .flatMap(_.options)
         .flatMap(_.internal.extraSourceFiles)
         .distinct
-    val inputsElemFromDirectives          = value(resolveInputsFromSources(sourcesFromDirectives))
+    val inputsElemFromDirectives =
+      value(resolveInputsFromSources(sourcesFromDirectives, inputs.enableMarkdown))
     val preprocessedSourcesFromDirectives = value(preprocessSources(inputsElemFromDirectives))
     val allInputs                         = inputs.add(inputsElemFromDirectives)
 
@@ -228,15 +229,17 @@ object CrossSources {
     (CrossSources(paths, inMemory, defaultMainClassOpt, resourceDirs, buildOptions), allInputs)
   }
 
-  private def resolveInputsFromSources(sources: Seq[Positioned[os.Path]]) =
+  private def resolveInputsFromSources(sources: Seq[Positioned[os.Path]], enableMarkdown: Boolean) =
     sources.map { source =>
       val sourcePath   = source.value
       lazy val dir     = sourcePath / os.up
       lazy val subPath = sourcePath.subRelativeTo(dir)
-      if (os.isDir(sourcePath)) Right(Inputs.singleFilesFromDirectory(Inputs.Directory(sourcePath)))
+      if (os.isDir(sourcePath))
+        Right(Inputs.singleFilesFromDirectory(Inputs.Directory(sourcePath), enableMarkdown))
       else if (sourcePath.ext == "scala") Right(Seq(Inputs.ScalaFile(dir, subPath)))
       else if (sourcePath.ext == "sc") Right(Seq(Inputs.Script(dir, subPath)))
       else if (sourcePath.ext == "java") Right(Seq(Inputs.JavaFile(dir, subPath)))
+      else if (sourcePath.ext == "md") Right(Seq(Inputs.MarkdownFile(dir, subPath)))
       else {
         val msg =
           if (os.exists(sourcePath))

modules/build/src/main/scala/scala/build/Inputs.scala

@@ -22,6 +22,7 @@ final case class Inputs(
   baseProjectName: String,
   mayAppendHash: Boolean,
   workspaceOrigin: Option[WorkspaceOrigin],
+  enableMarkdown: Boolean,
   withRestrictedFeatures: Boolean
 ) {
 
@@ -31,7 +32,7 @@ final case class Inputs(
   def singleFiles(): Seq[Inputs.SingleFile] =
     elements.flatMap {
       case f: Inputs.SingleFile        => Seq(f)
-      case d: Inputs.Directory         => Inputs.singleFilesFromDirectory(d)
+      case d: Inputs.Directory         => Inputs.singleFilesFromDirectory(d, enableMarkdown)
       case _: Inputs.ResourceDirectory => Nil
       case _: Inputs.Virtual           => Nil
     }
@@ -52,7 +53,7 @@ final case class Inputs(
   def flattened(): Seq[Inputs.SingleElement] =
     elements.flatMap {
       case f: Inputs.SingleFile        => Seq(f)
-      case d: Inputs.Directory         => Inputs.singleFilesFromDirectory(d)
+      case d: Inputs.Directory         => Inputs.singleFilesFromDirectory(d, enableMarkdown)
       case _: Inputs.ResourceDirectory => Nil
       case v: Inputs.Virtual           => Seq(v)
     }
@@ -110,7 +111,7 @@ final case class Inputs(
       case elem: Inputs.OnDisk =>
         val content = elem match {
           case dirInput: Inputs.Directory =>
-            Seq("dir:") ++ Inputs.singleFilesFromDirectory(dirInput)
+            Seq("dir:") ++ Inputs.singleFilesFromDirectory(dirInput, enableMarkdown)
               .map(file => s"${file.path}:" + os.read(file.path))
           case resDirInput: Inputs.ResourceDirectory =>
             // Resource changes for SN require relinking, so they should also be hashed
@@ -199,6 +200,10 @@ object Inputs {
       extends OnDisk with SourceFile with Compiled {
     lazy val path: os.Path = base / subPath
   }
+  final case class MarkdownFile(base: os.Path, subPath: os.SubPath)
+      extends OnDisk with SourceFile {
+    lazy val path: os.Path = base / subPath
+  }
   final case class Directory(path: os.Path)         extends OnDisk with Compiled
   final case class ResourceDirectory(path: os.Path) extends OnDisk
 
@@ -218,7 +223,10 @@ object Inputs {
   final case class VirtualData(content: Array[Byte], source: String)
       extends Virtual
 
-  def singleFilesFromDirectory(d: Inputs.Directory): Seq[Inputs.SingleFile] = {
+  def singleFilesFromDirectory(
+    d: Inputs.Directory,
+    enableMarkdown: Boolean
+  ): Seq[Inputs.SingleFile] = {
     import Ordering.Implicits.seqOrdering
     os.walk.stream(d.path, skip = _.last.startsWith("."))
       .filter(os.isFile(_))
@@ -229,6 +237,8 @@ object Inputs {
           Inputs.ScalaFile(d.path, p.subRelativeTo(d.path))
         case p if p.last.endsWith(".sc") =>
           Inputs.Script(d.path, p.subRelativeTo(d.path))
+        case p if p.last.endsWith(".md") && enableMarkdown =>
+          Inputs.MarkdownFile(d.path, p.subRelativeTo(d.path))
       }
       .toVector
       .sortBy(_.subPath.segments)
@@ -244,6 +254,7 @@ object Inputs {
           case _: Inputs.JavaFile          => "java:"
           case _: Inputs.ScalaFile         => "scala:"
           case _: Inputs.Script            => "sc:"
+          case _: Inputs.MarkdownFile      => "md:"
         }
         Iterator(prefix, elem.path.toString, "\n").map(bytes)
       case v: Inputs.Virtual =>
@@ -268,6 +279,7 @@ object Inputs {
     baseProjectName: String,
     directories: Directories,
     forcedWorkspace: Option[os.Path],
+    enableMarkdown: Boolean,
     withRestrictedFeatures: Boolean
   ): Inputs = {
 
@@ -313,7 +325,8 @@ object Inputs {
       baseProjectName,
       mayAppendHash = needsHash,
       workspaceOrigin = Some(workspaceOrigin0),
-      withRestrictedFeatures
+      enableMarkdown = enableMarkdown,
+      withRestrictedFeatures = withRestrictedFeatures
     )
   }
 
@@ -410,6 +423,7 @@ object Inputs {
       else if (arg.endsWith(".sc")) Right(Seq(Script(dir, subPath)))
       else if (arg.endsWith(".scala")) Right(Seq(ScalaFile(dir, subPath)))
       else if (arg.endsWith(".java")) Right(Seq(JavaFile(dir, subPath)))
+      else if (arg.endsWith(".md")) Right(Seq(MarkdownFile(dir, subPath)))
       else if (os.isDir(path)) Right(Seq(Directory(path)))
       else if (acceptFds && arg.startsWith("/dev/fd/")) {
         val content = os.read.bytes(os.Path(arg, cwd))
@@ -436,6 +450,7 @@ object Inputs {
     javaSnippetList: List[String],
     acceptFds: Boolean,
     forcedWorkspace: Option[os.Path],
+    enableMarkdown: Boolean,
     withRestrictedFeatures: Boolean
   ): Either[BuildException, Inputs] = {
     val validatedArgs: Seq[Either[String, Seq[Element]]] =
@@ -457,6 +472,7 @@ object Inputs {
         baseProjectName,
         directories,
         forcedWorkspace,
+        enableMarkdown,
         withRestrictedFeatures
       ))
     }
@@ -477,13 +493,14 @@ object Inputs {
     javaSnippetList: List[String] = List.empty,
     acceptFds: Boolean = false,
     forcedWorkspace: Option[os.Path] = None,
+    enableMarkdown: Boolean = false,
     withRestrictedFeatures: Boolean
   ): Either[BuildException, Inputs] =
     if (
       args.isEmpty && scriptSnippetList.isEmpty && scalaSnippetList.isEmpty && javaSnippetList.isEmpty
     )
       defaultInputs().toRight(new InputsException(
-        "No inputs provided (expected files with .scala or .sc extensions, and / or directories)."
+        "No inputs provided (expected files with .scala, .sc, .java or .md extensions, and / or directories)."
       ))
     else
       forNonEmptyArgs(
@@ -498,22 +515,25 @@ object Inputs {
         javaSnippetList,
         acceptFds,
         forcedWorkspace,
+        enableMarkdown,
         withRestrictedFeatures
       )
 
   def default(): Option[Inputs] =
     None
 
-  def empty(workspace: os.Path): Inputs =
+  def empty(workspace: os.Path, enableMarkdown: Boolean): Inputs =
     Inputs(
       elements = Nil,
       defaultMainClassElement = None,
       workspace = workspace,
       baseProjectName = "project",
       mayAppendHash = true,
       workspaceOrigin = None,
+      enableMarkdown = enableMarkdown,
       withRestrictedFeatures = false
     )
 
-  def empty(projectName: String) = Inputs(Nil, None, os.pwd, projectName, false, None, false)
+  def empty(projectName: String): Inputs =
+    Inputs(Nil, None, os.pwd, projectName, false, None, true, false)
 }

modules/build/src/main/scala/scala/build/Sources.scala

@@ -92,6 +92,7 @@ object Sources {
   ): Seq[Preprocessor] =
     Seq(
       ScriptPreprocessor(codeWrapper),
+      MarkdownPreprocessor,
       JavaPreprocessor(archiveCache, javaClassNameVersionOpt, javaCommand),
       ScalaPreprocessor,
       DataPreprocessor

modules/build/src/main/scala/scala/build/internal/markdown/MarkdownCodeBlock.scala

@@ -0,0 +1,82 @@
+package scala.build.internal.markdown
+
+import scala.annotation.tailrec
+import scala.collection.mutable
+import scala.jdk.CollectionConverters.*
+
+/** Representation for a (closed) code block contained in Markdown
+  *
+  * @param info
+  *   a list of tags tied to a given code block
+  * @param body
+  *   the code block content
+  * @param startLine
+  *   starting line on which the code block was defined (excluding backticks)
+  * @param endLine
+  *   end line on which the code block was closed (excluding backticks)
+  */
+case class MarkdownCodeBlock(
+  info: Seq[String],
+  body: String,
+  startLine: Int,
+  endLine: Int
+) {
+
+  /** @return
+    *   `true` if this snippet should be ignored, `false` otherwise
+    */
+  def shouldIgnore: Boolean = info.head != "scala" || info.contains("ignore")
+
+  /** @return
+    *   `true` if this snippet should have its scope reset, `false` otherwise
+    */
+  def resetScope: Boolean = info.contains("reset")
+
+  /** @return
+    *   `true` if this snippet is a test snippet, `false` otherwise
+    */
+  def isTest: Boolean = info.contains("test")
+
+  /** @return
+    *   `true` if this snippet is a raw snippet, `false` otherwise
+    */
+  def isRaw: Boolean = info.contains("raw")
+}
+
+object MarkdownCodeBlock {
+
+  /** Finds all code snippets in given input
+    *
+    * @param md
+    *   Markdown file in a `String` format
+    * @return
+    *   list of all found snippets
+    */
+  def findCodeBlocks(md: String): Seq[MarkdownCodeBlock] = {
+    val allLines = md
+      .lines()
+      .toList
+      .asScala
+    @tailrec
+    def findCodeBlocksRec(
+      lines: Seq[String],
+      closedFences: Seq[MarkdownCodeBlock] = Seq.empty,
+      maybeOpenFence: Option[MarkdownOpenFence] = None,
+      currentIndex: Int = 0
+    ): Seq[MarkdownCodeBlock] = if lines.isEmpty then closedFences
+    else {
+      val currentLine = lines.head
+      val (newClosedFences, newOpenFence) = maybeOpenFence match {
+        case None => closedFences -> MarkdownOpenFence.maybeFence(currentLine, currentIndex)
+        case mof @ Some(openFence) =>
+          val backticksStart = currentLine.indexOf(openFence.backticks)
+          if backticksStart == openFence.indent &&
+            currentLine.forall(c => c == '`' || c.isWhitespace)
+          then (closedFences :+ openFence.closeFence(currentIndex, allLines.toArray)) -> None
+          else closedFences                                                           -> mof
+      }
+      findCodeBlocksRec(lines.tail, newClosedFences, newOpenFence, currentIndex + 1)
+    }
+    findCodeBlocksRec(allLines.toSeq).filter(!_.shouldIgnore)
+  }
+}