Merge pull request #1598 from Gedochao/fix-md-using-directives

Gedochao · web-flow · commit 1be0e6d54ec8 · 2022-11-22T22:28:47.000+08:00
Fix `using` directives for Markdown inputs
diff --git a/modules/build/src/main/scala/scala/build/internal/markdown/MarkdownCodeBlock.scala b/modules/build/src/main/scala/scala/build/internal/markdown/MarkdownCodeBlock.scala
@@ -61,7 +61,7 @@ object MarkdownCodeBlock {
   def findCodeBlocks(
     subPath: os.SubPath,
     md: String,
-    maybeRecoverOnError: BuildException => Option[BuildException]
+    maybeRecoverOnError: BuildException => Option[BuildException] = Some(_)
   ): Either[BuildException, Seq[MarkdownCodeBlock]] = {
     val allLines = md
       .lines()
diff --git a/modules/build/src/main/scala/scala/build/internal/markdown/MarkdownCodeWrapper.scala b/modules/build/src/main/scala/scala/build/internal/markdown/MarkdownCodeWrapper.scala
@@ -1,42 +1,47 @@
 package scala.build.internal.markdown
 
 import scala.annotation.tailrec
-import scala.build.EitherCps.{either, value}
 import scala.build.errors.BuildException
 import scala.build.internal.markdown.MarkdownCodeBlock
 import scala.build.internal.{AmmUtil, Name}
+import scala.build.preprocessing.{
+  ExtractedDirectives,
+  PreprocessedMarkdown,
+  PreprocessedMarkdownCodeBlocks
+}
 
 /** A util for extraction and wrapping of code blocks in Markdown files.
   */
 object MarkdownCodeWrapper {
 
+  case class WrappedMarkdownCode(
+    code: String,
+    directives: ExtractedDirectives = ExtractedDirectives.empty
+  )
+
   /** Extracts scala code blocks from Markdown snippets, divides them into 3 categories and wraps
     * when necessary.
     *
     * @param subPath
     *   the project [[os.SubPath]] to the Markdown file
-    * @param content
-    *   Markdown code
+    * @param markdown
+    *   preprocessed Markdown code blocks
     * @return
-    *   a tuple of (Option(simple scala snippets code), Option(raw scala snippets code), Option(test
-    *   scala snippets code))
+    *   a tuple of (Option(simple scala code blocks), Option(raw scala snippets code blocks),
+    *   Option(test scala snippets code blocks))
     */
   def apply(
     subPath: os.SubPath,
-    content: String,
-    maybeRecoverOnError: BuildException => Option[BuildException] = b => Some(b)
-  ): Either[BuildException, (Option[String], Option[String], Option[String])] = either {
+    markdown: PreprocessedMarkdown
+  ): (Option[WrappedMarkdownCode], Option[WrappedMarkdownCode], Option[WrappedMarkdownCode]) = {
     val (pkg, wrapper) = AmmUtil.pathToPackageWrapper(subPath)
     val maybePkgString =
       if pkg.isEmpty then None else Some(s"package ${AmmUtil.encodeScalaSourcePath(pkg)}")
-    val allSnippets = value(MarkdownCodeBlock.findCodeBlocks(subPath, content, maybeRecoverOnError))
-    val (rawSnippets, processedSnippets) = allSnippets.partition(_.isRaw)
-    val (testSnippets, mainSnippets)     = processedSnippets.partition(_.isTest)
-    val wrapperName                      = s"${wrapper.raw}_md"
+    val wrapperName = s"${wrapper.raw}_md"
     (
-      wrapScalaCode(mainSnippets, wrapperName, maybePkgString),
-      rawScalaCode(rawSnippets),
-      rawScalaCode(testSnippets)
+      wrapScalaCode(markdown.scriptCodeBlocks, wrapperName, maybePkgString),
+      rawScalaCode(markdown.rawCodeBlocks),
+      rawScalaCode(markdown.testCodeBlocks)
     )
   }
 
@@ -75,20 +80,20 @@ object MarkdownCodeWrapper {
     *   an option of the wrapped code String
     */
   def wrapScalaCode(
-    snippets: Seq[MarkdownCodeBlock],
+    preprocessed: PreprocessedMarkdownCodeBlocks,
     wrapperName: String,
     pkg: Option[String]
-  ): Option[String] =
+  ): Option[WrappedMarkdownCode] =
     code(
-      snippets,
+      preprocessed.codeBlocks,
       s => {
         val packageDirective = pkg.map(_ + "; ").getOrElse("")
         val noWarnAnnotation = """@annotation.nowarn("msg=pure expression does nothing")"""
         val firstLine =
           s"""${packageDirective}object $wrapperName { $noWarnAnnotation def main(args: Array[String]): Unit = { """
         s.indices.foldLeft(0 -> firstLine) {
           case ((nextScopeIndex, sum), index) =>
-            if snippets(index).resetScope || index == 0 then
+            if preprocessed.codeBlocks(index).resetScope || index == 0 then
               nextScopeIndex + 1 -> (sum :++ s"${scopeObjectName(nextScopeIndex)}; ")
             else nextScopeIndex  -> sum // that class hasn't been created
         }
@@ -97,7 +102,7 @@ object MarkdownCodeWrapper {
           .:++(generateMainScalaLines(s, 0, 0, 0))
           .:++("}")
       }
-    )
+    ).map(c => WrappedMarkdownCode(c, preprocessed.extractedDirectives))
 
   @tailrec
   private def generateMainScalaLines(
@@ -137,8 +142,9 @@ object MarkdownCodeWrapper {
     * @return
     *   an option of the resulting code String
     */
-  def rawScalaCode(snippets: Seq[MarkdownCodeBlock]): Option[String] =
-    code(snippets, generateRawScalaLines(_, 0, 0))
+  def rawScalaCode(preprocessed: PreprocessedMarkdownCodeBlocks): Option[WrappedMarkdownCode] =
+    code(preprocessed.codeBlocks, generateRawScalaLines(_, 0, 0))
+      .map(c => WrappedMarkdownCode(c, preprocessed.extractedDirectives))
 
   @tailrec
   private def generateRawScalaLines(
diff --git a/modules/build/src/main/scala/scala/build/internal/markdown/MarkdownOpenFence.scala b/modules/build/src/main/scala/scala/build/internal/markdown/MarkdownOpenFence.scala
@@ -39,7 +39,7 @@ case class MarkdownOpenFence(
     val bodyLines: Array[String] = lines.slice(start, tickEndLine)
     MarkdownCodeBlock(
       info.split("\\s+").toList, // strip info by whitespaces
-      bodyLines.tail.foldLeft(bodyLines.head)((body, line) => body.:++("\n" + line)),
+      bodyLines.mkString("\n"),
       start,          // snippet has to begin in the new line
       tickEndLine - 1 // ending backticks have to be placed below the snippet
     )
diff --git a/modules/build/src/main/scala/scala/build/preprocessing/ExtractedDirectives.scala b/modules/build/src/main/scala/scala/build/preprocessing/ExtractedDirectives.scala
@@ -9,6 +9,7 @@ import com.virtuslab.using_directives.custom.model.{
 import com.virtuslab.using_directives.custom.utils.ast.{UsingDef, UsingDefs}
 import com.virtuslab.using_directives.{Context, UsingDirectivesProcessor}
 
+import scala.annotation.targetName
 import scala.build.errors.*
 import scala.build.preprocessing.directives.{DirectiveUtil, ScopedDirective, StrictDirective}
 import scala.build.{Logger, Position}
@@ -18,7 +19,11 @@ import scala.jdk.CollectionConverters.*
 case class ExtractedDirectives(
   offset: Int,
   directives: Seq[StrictDirective]
-)
+) {
+  @targetName("append")
+  def ++(other: ExtractedDirectives): ExtractedDirectives =
+    ExtractedDirectives(offset, directives ++ other.directives)
+}
 
 object ExtractedDirectives {
 
diff --git a/modules/build/src/main/scala/scala/build/preprocessing/MarkdownCodeBlockProcessor.scala b/modules/build/src/main/scala/scala/build/preprocessing/MarkdownCodeBlockProcessor.scala
@@ -0,0 +1,47 @@
+package scala.build.preprocessing
+
+import com.virtuslab.using_directives.custom.model.UsingDirectiveKind
+
+import scala.build.EitherCps.{either, value}
+import scala.build.Logger
+import scala.build.errors.BuildException
+import scala.build.internal.markdown.MarkdownCodeBlock
+
+object MarkdownCodeBlockProcessor {
+  def process(
+    codeBlocks: Seq[MarkdownCodeBlock],
+    reportingPath: Either[String, os.Path],
+    scopePath: ScopePath,
+    logger: Logger,
+    maybeRecoverOnError: BuildException => Option[BuildException]
+  ): Either[BuildException, PreprocessedMarkdown] = either {
+    val (rawCodeBlocks, remaining)         = codeBlocks.partition(_.isRaw)
+    val (testCodeBlocks, scriptCodeBlocks) = remaining.partition(_.isTest)
+    def preprocessCodeBlocks(cbs: Seq[MarkdownCodeBlock])
+      : Either[BuildException, PreprocessedMarkdownCodeBlocks] = either {
+      val mergedDirectives: ExtractedDirectives = cbs
+        .map { cb =>
+          value {
+            ExtractedDirectives.from(
+              contentChars = cb.body.toCharArray,
+              path = reportingPath,
+              logger = logger,
+              supportedDirectives = UsingDirectiveKind.values(),
+              cwd = scopePath / os.up,
+              maybeRecoverOnError = maybeRecoverOnError
+            )
+          }
+        }
+        .fold(ExtractedDirectives.empty)(_ ++ _)
+      PreprocessedMarkdownCodeBlocks(
+        cbs,
+        mergedDirectives
+      )
+    }
+    PreprocessedMarkdown(
+      value(preprocessCodeBlocks(scriptCodeBlocks)),
+      value(preprocessCodeBlocks(rawCodeBlocks)),
+      value(preprocessCodeBlocks(testCodeBlocks))
+    )
+  }
+}
diff --git a/modules/build/src/main/scala/scala/build/preprocessing/MarkdownPreprocessor.scala b/modules/build/src/main/scala/scala/build/preprocessing/MarkdownPreprocessor.scala
@@ -6,7 +6,7 @@ import scala.build.EitherCps.{either, value}
 import scala.build.Logger
 import scala.build.errors.BuildException
 import scala.build.input.{Inputs, MarkdownFile, SingleElement, VirtualMarkdownFile}
-import scala.build.internal.markdown.MarkdownCodeWrapper
+import scala.build.internal.markdown.{MarkdownCodeBlock, MarkdownCodeWrapper}
 import scala.build.internal.{AmmUtil, CodeWrapper, CustomCodeWrapper, Name}
 import scala.build.options.{BuildOptions, BuildRequirements}
 import scala.build.preprocessing.ScalaPreprocessor.ProcessingOutput
@@ -67,24 +67,25 @@ case object MarkdownPreprocessor extends Preprocessor {
     allowRestrictedFeatures: Boolean
   ): Either[BuildException, List[PreprocessedSource.InMemory]] = either {
     def preprocessSnippets(
-      maybeCode: Option[String],
+      maybeWrapper: Option[MarkdownCodeWrapper.WrappedMarkdownCode],
       generatedSourceNameSuffix: String
     ): Either[BuildException, Option[PreprocessedSource.InMemory]] =
       either {
-        maybeCode
-          .map { code =>
-            val processingOutput =
+        maybeWrapper
+          .map { wrappedMarkdown =>
+            val processingOutput: ProcessingOutput =
               value {
                 ScalaPreprocessor.process(
-                  content = code,
+                  content = wrappedMarkdown.code,
+                  extractedDirectives = wrappedMarkdown.directives,
                   path = reportingPath,
                   scopeRoot = scopePath / os.up,
                   logger = logger,
                   maybeRecoverOnError = maybeRecoverOnError,
                   allowRestrictedFeatures = allowRestrictedFeatures
                 )
               }.getOrElse(ProcessingOutput(BuildRequirements(), Nil, BuildOptions(), None))
-            val processedCode = processingOutput.updatedContent.getOrElse(code)
+            val processedCode = processingOutput.updatedContent.getOrElse(wrappedMarkdown.code)
             PreprocessedSource.InMemory(
               originalPath = reportingPath.map(subPath -> _),
               relPath = os.rel / (subPath / os.up) / s"${subPath.last}$generatedSourceNameSuffix",
@@ -99,8 +100,19 @@ case object MarkdownPreprocessor extends Preprocessor {
           }
       }
 
+    val codeBlocks: Seq[MarkdownCodeBlock] =
+      value(MarkdownCodeBlock.findCodeBlocks(subPath, content, maybeRecoverOnError))
+    val preprocessedMarkdown: PreprocessedMarkdown =
+      value(MarkdownCodeBlockProcessor.process(
+        codeBlocks,
+        reportingPath,
+        scopePath,
+        logger,
+        maybeRecoverOnError
+      ))
+
     val (mainScalaCode, rawScalaCode, testScalaCode) =
-      value(MarkdownCodeWrapper(subPath, content, maybeRecoverOnError))
+      MarkdownCodeWrapper(subPath, preprocessedMarkdown)
 
     val maybeMainFile = value(preprocessSnippets(mainScalaCode, ".scala"))
     val maybeRawFile  = value(preprocessSnippets(rawScalaCode, ".raw.scala"))
diff --git a/modules/build/src/main/scala/scala/build/preprocessing/PreprocessedMarkdown.scala b/modules/build/src/main/scala/scala/build/preprocessing/PreprocessedMarkdown.scala
@@ -0,0 +1,23 @@
+package scala.build.preprocessing
+
+import scala.build.internal.markdown.MarkdownCodeBlock
+
+case class PreprocessedMarkdownCodeBlocks(
+  codeBlocks: Seq[MarkdownCodeBlock],
+  extractedDirectives: ExtractedDirectives = ExtractedDirectives.empty
+)
+
+object PreprocessedMarkdownCodeBlocks {
+  def empty: PreprocessedMarkdownCodeBlocks =
+    PreprocessedMarkdownCodeBlocks(Seq.empty, ExtractedDirectives.empty)
+}
+
+case class PreprocessedMarkdown(
+  scriptCodeBlocks: PreprocessedMarkdownCodeBlocks = PreprocessedMarkdownCodeBlocks.empty,
+  rawCodeBlocks: PreprocessedMarkdownCodeBlocks = PreprocessedMarkdownCodeBlocks.empty,
+  testCodeBlocks: PreprocessedMarkdownCodeBlocks = PreprocessedMarkdownCodeBlocks.empty
+)
+
+object PreprocessedMarkdown {
+  def empty: PreprocessedMarkdown = PreprocessedMarkdown()
+}
diff --git a/modules/build/src/main/scala/scala/build/preprocessing/ScalaPreprocessor.scala b/modules/build/src/main/scala/scala/build/preprocessing/ScalaPreprocessor.scala
@@ -174,15 +174,43 @@ case object ScalaPreprocessor extends Preprocessor {
     logger: Logger,
     maybeRecoverOnError: BuildException => Option[BuildException],
     allowRestrictedFeatures: Boolean
+  ): Either[BuildException, Option[ProcessingOutput]] = either {
+    val extractedDirectives = value(ExtractedDirectives.from(
+      content.toCharArray,
+      path,
+      logger,
+      UsingDirectiveKind.values(),
+      scopeRoot,
+      maybeRecoverOnError
+    ))
+    value(process(
+      content,
+      extractedDirectives,
+      path,
+      scopeRoot,
+      logger,
+      maybeRecoverOnError,
+      allowRestrictedFeatures
+    ))
+  }
+
+  def process(
+    content: String,
+    extractedDirectives: ExtractedDirectives,
+    path: Either[String, os.Path],
+    scopeRoot: ScopePath,
+    logger: Logger,
+    maybeRecoverOnError: BuildException => Option[BuildException],
+    allowRestrictedFeatures: Boolean
   ): Either[BuildException, Option[ProcessingOutput]] = either {
     val (content0, isSheBang) = SheBang.ignoreSheBangLines(content)
     val afterStrictUsing: StrictDirectivesProcessingOutput =
       value(processStrictUsing(
         content0,
+        extractedDirectives,
         path,
         scopeRoot,
         logger,
-        maybeRecoverOnError,
         allowRestrictedFeatures
       ))
 
@@ -298,22 +326,14 @@ case object ScalaPreprocessor extends Preprocessor {
 
   private def processStrictUsing(
     content: String,
+    extractedDirectives: ExtractedDirectives,
     path: Either[String, os.Path],
     cwd: ScopePath,
     logger: Logger,
-    maybeRecoverOnError: BuildException => Option[BuildException] = e => Some(e),
     allowRestrictedFeatures: Boolean
   ): Either[BuildException, StrictDirectivesProcessingOutput] = either {
-    val contentChars = content.toCharArray
-    val ExtractedDirectives(codeOffset, directives0) =
-      value(ExtractedDirectives.from(
-        contentChars,
-        path,
-        logger,
-        UsingDirectiveKind.values(),
-        cwd,
-        maybeRecoverOnError
-      ))
+    val contentChars                                 = content.toCharArray
+    val ExtractedDirectives(codeOffset, directives0) = extractedDirectives
 
     val updatedOptions = value {
       DirectivesProcessor.process(
diff --git a/modules/build/src/test/scala/scala/build/tests/markdown/MarkdownCodeBlockTests.scala b/modules/build/src/test/scala/scala/build/tests/markdown/MarkdownCodeBlockTests.scala
diff --git a/modules/build/src/test/scala/scala/build/tests/markdown/MarkdownCodeWrapperTests.scala b/modules/build/src/test/scala/scala/build/tests/markdown/MarkdownCodeWrapperTests.scala
diff --git a/modules/build/src/test/scala/scala/build/tests/markdown/MarkdownTestUtil.scala b/modules/build/src/test/scala/scala/build/tests/markdown/MarkdownTestUtil.scala
diff --git a/modules/integration/src/test/scala/scala/cli/integration/MarkdownTests.scala b/modules/integration/src/test/scala/scala/cli/integration/MarkdownTests.scala

Original file line number	Diff line number	Diff line change
`@@ -39,7 +39,7 @@ case class MarkdownOpenFence(`
`39`	`39`	`val bodyLines: Array[String] = lines.slice(start, tickEndLine)`
`40`	`40`	`MarkdownCodeBlock(`
`41`	`41`	`info.split("\\s+").toList, // strip info by whitespaces`
`42`		`- bodyLines.tail.foldLeft(bodyLines.head)((body, line) => body.:++("\n" + line)),`
	`42`	`+ bodyLines.mkString("\n"),`
`43`	`43`	`start, // snippet has to begin in the new line`
`44`	`44`	`tickEndLine - 1 // ending backticks have to be placed below the snippet`
`45`	`45`	`)`