Merge pull request #1531 from alexarchambault/package-python

alexarchambault · web-flow · commit 1becd2df6cdb · 2022-11-29T12:18:46.000+01:00
Make package --python work
diff --git a/build.sc b/build.sc
@@ -181,6 +181,12 @@ object dummy extends Module {
       Deps.scalafmtCli
     )
   }
+  object pythonInterface extends JavaModule with Bloop.Module {
+    def skipBloop = true
+    def ivyDeps = Agg(
+      Deps.pythonInterface
+    )
+  }
 }
 
 trait BuildMacros extends ScalaCliSbtModule
@@ -664,6 +670,9 @@ trait Cli extends SbtModule with ProtoBuildModule with CliLaunchers
          |
          |/** Build-time constants. Generated by mill. */
          |object Constants {
+         |  def pythonInterfaceOrg       = "${Deps.pythonInterface.dep.module.organization.value}"
+         |  def pythonInterfaceName      = "${Deps.pythonInterface.dep.module.name.value}"
+         |  def pythonInterfaceVersion   = "${Deps.pythonInterface.dep.version}"
          |  def launcherTypeResourcePath = "${launcherTypeResourcePath.toString}"
          |  def defaultFilesResourcePath = "$defaultFilesResourcePath"
          |  def maxAmmoniteScala3Version = "${Scala.maxAmmoniteScala3Version}"
diff --git a/modules/cli/src/main/scala/scala/cli/commands/package0/Package.scala b/modules/cli/src/main/scala/scala/cli/commands/package0/Package.scala
@@ -3,6 +3,7 @@ package scala.cli.commands.package0
 import ai.kien.python.Python
 import caseapp.*
 import coursier.launcher.*
+import dependency.*
 import packager.config.*
 import packager.deb.DebianPackage
 import packager.docker.DockerPackage
@@ -36,7 +37,7 @@ import scala.cli.commands.util.BuildCommandHelpers
 import scala.cli.commands.{CommandUtils, ScalaCommand, WatchUtil}
 import scala.cli.config.{ConfigDb, Keys}
 import scala.cli.errors.ScalaJsLinkingError
-import scala.cli.internal.{CachedBinary, ProcUtil, ScalaJsLinker}
+import scala.cli.internal.{CachedBinary, Constants, ProcUtil, ScalaJsLinker}
 import scala.cli.packaging.{Library, NativeImage}
 import scala.util.Properties
 
@@ -300,7 +301,7 @@ object Package extends ScalaCommand[PackageOptions] with BuildCommandHelpers {
 
       val outputPath = packageType match {
         case PackageType.Bootstrap =>
-          value(bootstrap(build, destPath, value(mainClass), () => alreadyExistsCheck()))
+          value(bootstrap(build, destPath, value(mainClass), () => alreadyExistsCheck(), logger))
           destPath
         case PackageType.LibraryJar =>
           val content = Library.libraryJar(build)
@@ -391,7 +392,15 @@ object Package extends ScalaCommand[PackageOptions] with BuildCommandHelpers {
 
         case nativePackagerType: PackageType.NativePackagerType =>
           val bootstrapPath = os.temp.dir(prefix = "scala-packager") / "app"
-          value(bootstrap(build, bootstrapPath, value(mainClass), () => alreadyExistsCheck()))
+          value {
+            bootstrap(
+              build,
+              bootstrapPath,
+              value(mainClass),
+              () => alreadyExistsCheck(),
+              logger
+            )
+          }
           val sharedSettings = SharedSettings(
             sourceAppPath = bootstrapPath,
             version = packageOptions.packageVersion,
@@ -627,7 +636,7 @@ object Package extends ScalaCommand[PackageOptions] with BuildCommandHelpers {
 
     val appPath = os.temp.dir(prefix = "scala-cli-docker") / "app"
     build.options.platform.value match {
-      case Platform.JVM => value(bootstrap(build, appPath, mainClass, () => Right(())))
+      case Platform.JVM => value(bootstrap(build, appPath, mainClass, () => Right(()), logger))
       case Platform.JS  => buildJs(build, appPath, mainClass, logger)
       case Platform.Native =>
         val dest = value(buildNative(build, mainClass, logger))
@@ -669,7 +678,8 @@ object Package extends ScalaCommand[PackageOptions] with BuildCommandHelpers {
     build: Build.Successful,
     destPath: os.Path,
     mainClass: String,
-    alreadyExistsCheck: () => Either[BuildException, Unit]
+    alreadyExistsCheck: () => Either[BuildException, Unit],
+    logger: Logger
   ): Either[BuildException, Unit] = either {
     val byteCodeZipEntries = os.walk(build.output)
       .filter(os.isFile(_))
@@ -708,10 +718,41 @@ object Package extends ScalaCommand[PackageOptions] with BuildCommandHelpers {
       .withOsKind(Properties.isWin)
       .callsItself(Properties.isWin)
       .withJavaOpts(build.options.javaOptions.javaOpts.toSeq.map(_.value.value))
-    val params = Parameters.Bootstrap(Seq(loaderContent), mainClass)
+    val baseParams = Parameters.Bootstrap(Seq(loaderContent), mainClass)
       .withDeterministic(true)
       .withPreamble(preamble)
 
+    val params =
+      if (build.options.notForBloopOptions.doSetupPython.getOrElse(false)) {
+        val res = value {
+          Artifacts.fetch(
+            Positioned.none(Seq(
+              dep"${Constants.pythonInterfaceOrg}:${Constants.pythonInterfaceName}:${Constants.pythonInterfaceVersion}"
+            )),
+            Nil,
+            None,
+            logger,
+            build.options.finalCache,
+            None,
+            Some(_)
+          )
+        }
+        val entries = res.artifacts.map {
+          case (a, f) =>
+            val path = os.Path(f)
+            if (build.options.notForBloopOptions.packageOptions.isStandalone)
+              ClassPathEntry.Resource(path.last, os.mtime(path), os.read.bytes(path))
+            else
+              ClassPathEntry.Url(a.url)
+        }
+        val pythonContent = Seq(
+          ClassLoaderContent(entries)
+        )
+        baseParams.addExtraContent("python", pythonContent).withPython(true)
+      }
+      else
+        baseParams
+
     value(alreadyExistsCheck())
     BootstrapGenerator.generate(params, destPath.toNIO)
     ProcUtil.maybeUpdatePreamble(destPath)
diff --git a/modules/cli/src/main/scala/scala/cli/commands/publish/Publish.scala b/modules/cli/src/main/scala/scala/cli/commands/publish/Publish.scala
@@ -70,7 +70,6 @@ object Publish extends ScalaCommand[PublishOptions] with BuildCommandHelpers {
     publishParams: PublishParamsOptions,
     sharedPublish: SharedPublishOptions,
     publishRepo: PublishRepositoryOptions,
-    sharedPython: SharedPythonOptions,
     scalaSigning: PgpScalaSigningOptions,
     mainClass: MainClassOptions,
     ivy2LocalLike: Option[Boolean]
@@ -156,10 +155,7 @@ object Publish extends ScalaCommand[PublishOptions] with BuildCommandHelpers {
             useJvm = scalaSigning.forceJvmSigningCli,
             javaArgs = scalaSigning.signingCliJavaArg
           )
-        ),
-        python = sharedPython.python,
-        pythonSetup = sharedPython.pythonSetup,
-        scalaPyVersion = sharedPython.scalaPyVersion
+        )
       )
     )
   }
@@ -191,7 +187,6 @@ object Publish extends ScalaCommand[PublishOptions] with BuildCommandHelpers {
       options.publishParams,
       options.sharedPublish,
       options.publishRepo,
-      options.sharedPython,
       options.signingCli,
       options.mainClass,
       options.ivy2LocalLike
diff --git a/modules/cli/src/main/scala/scala/cli/commands/publish/PublishLocal.scala b/modules/cli/src/main/scala/scala/cli/commands/publish/PublishLocal.scala
@@ -37,7 +37,6 @@ object PublishLocal extends ScalaCommand[PublishLocalOptions] {
       options.publishParams,
       options.sharedPublish,
       PublishRepositoryOptions(),
-      options.sharedPython,
       options.scalaSigning,
       options.mainClass,
       None
diff --git a/modules/cli/src/main/scala/scala/cli/commands/publish/PublishLocalOptions.scala b/modules/cli/src/main/scala/scala/cli/commands/publish/PublishLocalOptions.scala
@@ -26,8 +26,6 @@ final case class PublishLocalOptions(
     publishParams: PublishParamsOptions = PublishParamsOptions(),
   @Recurse
     sharedPublish: SharedPublishOptions = SharedPublishOptions(),
-  @Recurse
-    sharedPython: SharedPythonOptions = SharedPythonOptions(),
   @Recurse
     scalaSigning: PgpScalaSigningOptions = PgpScalaSigningOptions(),
 ) extends HasSharedOptions
diff --git a/modules/cli/src/main/scala/scala/cli/commands/publish/PublishOptions.scala b/modules/cli/src/main/scala/scala/cli/commands/publish/PublishOptions.scala
@@ -21,8 +21,6 @@ final case class PublishOptions(
     publishRepo: PublishRepositoryOptions = PublishRepositoryOptions(),
   @Recurse
     sharedPublish: SharedPublishOptions = SharedPublishOptions(),
-  @Recurse
-    sharedPython: SharedPythonOptions = SharedPythonOptions(),
   @Recurse
     signingCli: PgpScalaSigningOptions = PgpScalaSigningOptions(),
 
diff --git a/modules/cli/src/main/scala/scala/cli/commands/repl/Repl.scala b/modules/cli/src/main/scala/scala/cli/commands/repl/Repl.scala
@@ -78,9 +78,6 @@ object Repl extends ScalaCommand[ReplOptions] {
           ammoniteVersionOpt = ammoniteVersion.map(_.trim).filter(_.nonEmpty),
           ammoniteArgs = ammoniteArg
         ),
-        python = sharedPython.python,
-        pythonSetup = sharedPython.pythonSetup,
-        scalaPyVersion = sharedPython.scalaPyVersion,
         addRunnerDependencyOpt = baseOptions.notForBloopOptions.addRunnerDependencyOpt
           .orElse(Some(false))
       )
diff --git a/modules/cli/src/main/scala/scala/cli/commands/repl/SharedReplOptions.scala b/modules/cli/src/main/scala/scala/cli/commands/repl/SharedReplOptions.scala
@@ -19,8 +19,6 @@ final case class SharedReplOptions(
     watch: SharedWatchOptions = SharedWatchOptions(),
   @Recurse
     compileCross: CrossOptions = CrossOptions(),
-  @Recurse
-    sharedPython: SharedPythonOptions = SharedPythonOptions(),
 
   @Group("Repl")
   @Tag(tags.restricted)
diff --git a/modules/cli/src/main/scala/scala/cli/commands/run/Run.scala b/modules/cli/src/main/scala/scala/cli/commands/run/Run.scala
@@ -89,9 +89,6 @@ object Run extends ScalaCommand[RunOptions] with BuildCommandHelpers {
       ),
       notForBloopOptions = baseOptions.notForBloopOptions.copy(
         runWithManifest = options.sharedRun.useManifest,
-        python = options.sharedRun.sharedPython.python,
-        pythonSetup = options.sharedRun.sharedPython.pythonSetup,
-        scalaPyVersion = options.sharedRun.sharedPython.scalaPyVersion,
         addRunnerDependencyOpt = baseOptions.notForBloopOptions.addRunnerDependencyOpt.orElse {
           runMode(options) match {
             case _: RunMode.Spark | RunMode.HadoopJar =>
diff --git a/modules/cli/src/main/scala/scala/cli/commands/run/SharedRunOptions.scala b/modules/cli/src/main/scala/scala/cli/commands/run/SharedRunOptions.scala
@@ -25,8 +25,6 @@ final case class SharedRunOptions(
     compileCross: CrossOptions = CrossOptions(),
   @Recurse
     mainClass: MainClassOptions = MainClassOptions(),
-  @Recurse
-    sharedPython: SharedPythonOptions = SharedPythonOptions(),
   @Group("Run")
   @Hidden
   @Tag(tags.experimental)
diff --git a/modules/cli/src/main/scala/scala/cli/commands/shared/SharedOptions.scala b/modules/cli/src/main/scala/scala/cli/commands/shared/SharedOptions.scala
@@ -63,6 +63,8 @@ final case class SharedOptions(
     coursier: CoursierOptions = CoursierOptions(),
   @Recurse
     workspace: SharedWorkspaceOptions = SharedWorkspaceOptions(),
+  @Recurse
+    sharedPython: SharedPythonOptions = SharedPythonOptions(),
 
   @Group("Scala")
   @HelpMessage(s"Set the Scala version (${Constants.defaultScalaVersion} by default)")
@@ -321,7 +323,10 @@ final case class SharedOptions(
       ),
       notForBloopOptions = bo.PostBuildOptions(
         scalaJsLinkerOptions = linkerOptions(js),
-        addRunnerDependencyOpt = runner
+        addRunnerDependencyOpt = runner,
+        python = sharedPython.python,
+        pythonSetup = sharedPython.pythonSetup,
+        scalaPyVersion = sharedPython.scalaPyVersion
       )
     )
   }
diff --git a/modules/cli/src/test/scala/cli/commands/tests/ReplOptionsTests.scala b/modules/cli/src/test/scala/cli/commands/tests/ReplOptionsTests.scala
@@ -3,14 +3,14 @@ package scala.cli.commands.tests
 import com.eed3si9n.expecty.Expecty.{assert => expect}
 
 import scala.cli.commands.repl.{Repl, ReplOptions, SharedReplOptions}
-import scala.cli.commands.shared.SharedPythonOptions
+import scala.cli.commands.shared.{SharedOptions, SharedPythonOptions}
 
 class ReplOptionsTests extends munit.FunSuite {
 
   test("ScalaPy version") {
     val ver = "X.Y.Z"
     val replOptions = ReplOptions(
-      sharedRepl = SharedReplOptions(
+      shared = SharedOptions(
         sharedPython = SharedPythonOptions(
           scalaPyVersion = Some(ver)
         )
diff --git a/modules/cli/src/test/scala/cli/commands/tests/RunOptionsTests.scala b/modules/cli/src/test/scala/cli/commands/tests/RunOptionsTests.scala
@@ -3,14 +3,14 @@ package scala.cli.commands.tests
 import com.eed3si9n.expecty.Expecty.{assert => expect}
 
 import scala.cli.commands.run.{Run, RunOptions, SharedRunOptions}
-import scala.cli.commands.shared.SharedPythonOptions
+import scala.cli.commands.shared.{SharedOptions, SharedPythonOptions}
 
 class RunOptionsTests extends munit.FunSuite {
 
   test("ScalaPy version") {
     val ver = "X.Y.Z"
     val runOptions = RunOptions(
-      sharedRun = SharedRunOptions(
+      shared = SharedOptions(
         sharedPython = SharedPythonOptions(
           scalaPyVersion = Some(ver)
         )
diff --git a/modules/integration/src/test/scala/scala/cli/integration/CompileTestDefinitions.scala b/modules/integration/src/test/scala/scala/cli/integration/CompileTestDefinitions.scala
@@ -509,4 +509,35 @@ abstract class CompileTestDefinitions(val scalaVersionOpt: Option[String])
         .call(cwd = root, stdin = os.Inherit, stdout = os.Inherit)
     }
   }
+
+  test("scalapy") {
+
+    def maybeScalapyPrefix =
+      if (actualScalaVersion.startsWith("2.13.")) ""
+      else "import me.shadaj.scalapy.py" + System.lineSeparator()
+
+    val inputs = TestInputs(
+      os.rel / "Hello.scala" ->
+        s"""$maybeScalapyPrefix
+           |object Hello {
+           |  def main(args: Array[String]): Unit = {
+           |    py.Dynamic.global.print("Hello from Python", flush = true)
+           |  }
+           |}
+           |""".stripMargin
+    )
+
+    inputs.fromRoot { root =>
+      val res =
+        os.proc(TestUtil.cli, "compile", "--python", "--print-class-path", ".", extraOptions)
+          .call(cwd = root)
+      val classPath = res.out.trim().split(File.pathSeparator)
+      val outputDir = os.Path(classPath.head, root)
+      val classFiles = os.walk(outputDir)
+        .filter(_.last.endsWith(".class"))
+        .filter(os.isFile(_))
+        .map(_.relativeTo(outputDir))
+      expect(classFiles.contains(os.rel / "Hello.class"))
+    }
+  }
 }
diff --git a/modules/integration/src/test/scala/scala/cli/integration/PackageTestDefinitions.scala b/modules/integration/src/test/scala/scala/cli/integration/PackageTestDefinitions.scala
@@ -894,4 +894,36 @@ abstract class PackageTestDefinitions(val scalaVersionOpt: Option[String])
       )
     }
   }
+
+  test("scalapy") {
+
+    def maybeScalapyPrefix =
+      if (actualScalaVersion.startsWith("2.13.")) ""
+      else "import me.shadaj.scalapy.py" + System.lineSeparator()
+
+    val inputs = TestInputs(
+      os.rel / "Hello.scala" ->
+        s"""$maybeScalapyPrefix
+           |object Hello {
+           |  def main(args: Array[String]): Unit = {
+           |    py.Dynamic.global.print("Hello from Python", flush = true)
+           |  }
+           |}
+           |""".stripMargin
+    )
+
+    val dest =
+      if (Properties.isWin) "hello.bat"
+      else "hello"
+
+    inputs.fromRoot { root =>
+      os.proc(TestUtil.cli, "package", "--python", ".", "-o", dest, extraOptions)
+        .call(cwd = root, stdin = os.Inherit, stdout = os.Inherit)
+
+      val launcher = root / dest
+      val res      = os.proc(launcher).call(cwd = root)
+      val output   = res.out.trim()
+      expect(output == "Hello from Python")
+    }
+  }
 }
diff --git a/project/deps.sc b/project/deps.sc
@@ -139,6 +139,7 @@ object Deps {
   def organizeImports            = ivy"com.github.liancheng::organize-imports:0.6.0"
   def osLib                      = ivy"com.lihaoyi::os-lib:0.8.1"
   def pprint                     = ivy"com.lihaoyi::pprint:0.8.0"
+  def pythonInterface            = ivy"io.github.alexarchambault.python:interface:0.1.0"
   def pythonNativeLibs           = ivy"ai.kien::python-native-libs:0.2.4"
   def scala3Compiler(sv: String) = ivy"org.scala-lang:scala3-compiler_3:$sv"
   def scalaAsync         = ivy"org.scala-lang.modules::scala-async:1.0.1".exclude("*" -> "*")
diff --git a/website/docs/reference/cli-options.md b/website/docs/reference/cli-options.md
@@ -1007,7 +1007,7 @@ Dummy mode - don't upload any secret to GitHub
 
 Available in commands:
 
-[`publish`](./commands.md#publish), [`publish local`](./commands.md#publish-local), [`repl` , `console`](./commands.md#repl), [`run`](./commands.md#run), [`shebang`](./commands.md#shebang)
+[`bsp`](./commands.md#bsp), [`compile`](./commands.md#compile), [`dependency-update`](./commands.md#dependency-update), [`doc`](./commands.md#doc), [`export`](./commands.md#export), [`fmt` , `format` , `scalafmt`](./commands.md#fmt), [`browse` , `metabrowse`](./commands.md#browse), [`package`](./commands.md#package), [`publish`](./commands.md#publish), [`publish local`](./commands.md#publish-local), [`repl` , `console`](./commands.md#repl), [`run`](./commands.md#run), [`setup-ide`](./commands.md#setup-ide), [`shebang`](./commands.md#shebang), [`test`](./commands.md#test)
 
 <!-- Automatically generated, DO NOT EDIT MANUALLY -->
 
diff --git a/website/docs/reference/commands.md b/website/docs/reference/commands.md
diff --git a/website/docs/reference/scala-command/commands.md b/website/docs/reference/scala-command/commands.md

Original file line number	Diff line number	Diff line change
`@@ -181,6 +181,12 @@ object dummy extends Module {`
`181`	`181`	`Deps.scalafmtCli`
`182`	`182`	`)`
`183`	`183`	`}`
	`184`	`+ object pythonInterface extends JavaModule with Bloop.Module {`
	`185`	`+ def skipBloop = true`
	`186`	`+ def ivyDeps = Agg(`
	`187`	`+ Deps.pythonInterface`
	`188`	`+ )`
	`189`	`+ }`
`184`	`190`	`}`
`185`	`191`
`186`	`192`	`trait BuildMacros extends ScalaCliSbtModule`
`@@ -664,6 +670,9 @@ trait Cli extends SbtModule with ProtoBuildModule with CliLaunchers`
`664`	`670`	`\|`
`665`	`671`	`\|/** Build-time constants. Generated by mill. */`
`666`	`672`	`\|object Constants {`
	`673`	`+ \| def pythonInterfaceOrg = "${Deps.pythonInterface.dep.module.organization.value}"`
	`674`	`+ \| def pythonInterfaceName = "${Deps.pythonInterface.dep.module.name.value}"`
	`675`	`+ \| def pythonInterfaceVersion = "${Deps.pythonInterface.dep.version}"`
`667`	`676`	`\| def launcherTypeResourcePath = "${launcherTypeResourcePath.toString}"`
`668`	`677`	`\| def defaultFilesResourcePath = "$defaultFilesResourcePath"`
`669`	`678`	`\| def maxAmmoniteScala3Version = "${Scala.maxAmmoniteScala3Version}"`