feat: Include waved binary in Conda distributions #2266

.github/workflows/publish-conda.yml

@@ -8,7 +8,6 @@ on:
         required: true
 
 env:
-  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   VERSION: "${{ github.event.inputs.version }}"
 
 jobs:
@@ -20,13 +19,39 @@ jobs:
         with:
           token: ${{ secrets.GIT_TOKEN }}
 
-      - name: Publish to Conda
-        uses: marek-mihok/publish_conda_package_action@v1.0.1
+      - uses: actions/setup-go@v1
         with:
-          CondaDir: 'py/h2o_wave/conda'
-          Platforms: 'noarch'
-          CondaUsername: ${{ secrets.CONDA_USERNAME }}
-          CondaPassword: ${{ secrets.CONDA_PASSWORD }}
+          go-version: '1.19.4'
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version-file: '.nvmrc'
+
+      - name: Setup
+        run: make setup
+
+      - uses: r-lib/actions/setup-r@v2
+      - uses: r-lib/actions/setup-r-dependencies@v2
+        with:
+          working-directory: ./r
+
+      - name: Setup Miniconda
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          auto-activate-base: false
+          activate-environment: ""
+          conda-build-version: '24.3.0'
+
+      - name: Build Release
+        run: make release
         env:
           # VERSION clashes with conda build. Use PKG_VERSION instead.
           PKG_VERSION: ${{ env.VERSION }}
+          NODE_OPTIONS: '--max-old-space-size=8192'
+
+      - name: Publish to Conda
+        uses: marek-mihok/upload_to_anaconda_cloud_action@v1.0.0
+        with:
+          PackagesDir: 'py/h2o_wave/sdist'
+          CondaUsername: ${{ secrets.CONDA_USERNAME }}
+          CondaPassword: ${{ secrets.CONDA_PASSWORD }}

.github/workflows/release-wave.yml

@@ -41,9 +41,18 @@ jobs:
         with:
           working-directory: ./r
 
+      - name: Setup Miniconda
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          auto-activate-base: false
+          activate-environment: ""
+          conda-build-version: '24.3.0'
+
       - name: Build Release
         run: make release
         env:
+          # VERSION clashes with conda build. Use PKG_VERSION instead.
+          PKG_VERSION: ${{ env.VERSION }}
           NODE_OPTIONS: '--max-old-space-size=8192'
 
       - name: Unit Test
@@ -79,15 +88,11 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Publish to Conda
-        uses: marek-mihok/publish_conda_package_action@v1.0.1
+        uses: marek-mihok/upload_to_anaconda_cloud_action@v1.0.0
         with:
-          CondaDir: 'py/h2o_wave/conda'
-          Platforms: 'noarch'
+          PackagesDir: 'py/h2o_wave/sdist'
           CondaUsername: ${{ secrets.CONDA_USERNAME }}
           CondaPassword: ${{ secrets.CONDA_PASSWORD }}
-        env:
-          # VERSION clashes with conda build. Use PKG_VERSION instead.
-          PKG_VERSION: ${{ env.VERSION }}
 
       - name: Publish to PyPI
         uses: pypa/gh-action-pypi-publish@master

py/h2o_wave/Makefile

@@ -1,3 +1,19 @@
+build_conda_package = \
+	if [ "$(1)" != "noarch" ]; then \
+		mkdir -p conda/temp; \
+		cp -a ../../build/wave-$(VERSION)-$(1)/. conda/temp; \
+		NO_ARCH=0 conda build --output-folder temp_build/ conda; \
+		find temp_build/ -name *.tar.bz2 | while read -r file; do \
+			conda convert --force --platform $(2) "$$file" -o temp_build/; \
+		done \
+	fi && \
+	if [ "$(1)" == "noarch" ]; then \
+		NO_ARCH=1 conda build --output-folder temp_build/ conda; \
+	fi && \
+	mkdir -p sdist/$(2) && \
+	mv temp_build/$(2)/*.tar.bz2 sdist/$(2) && \
+	rm -rf temp_build conda/temp
+
 .PHONY: build
 build: purge
 	H2O_WAVE_PLATFORM=win_amd64 ../venv/bin/python3 -m build --wheel
@@ -6,7 +22,11 @@ build: purge
 	H2O_WAVE_PLATFORM=macosx_11_0_arm64 ../venv/bin/python3 -m build --wheel
 	H2O_WAVE_PLATFORM=macosx_12_0_arm64 ../venv/bin/python3 -m build --wheel
 	../venv/bin/python3 -m build --wheel
+	$(call build_conda_package,darwin-arm64,osx-arm64)
+	$(call build_conda_package,darwin-amd64,osx-64)
+	$(call build_conda_package,linux-amd64,linux-64)
+	$(call build_conda_package,windows-amd64,win-64)
+	$(call build_conda_package,noarch,noarch)
 
 purge: ## Purge previous build
-	rm -f h2o_wave/metadata.py
-	rm -rf build dist h2o_wave.egg-info
+	rm -rf build dist sdist h2o_wave.egg-info temp_build conda/temp h2o_wave/metadata.py

py/h2o_wave/conda/conda_build_config.yaml

@@ -0,0 +1,8 @@
+# TODO: Build a single package for all python versions once this is resolved: https://github.com/conda/conda-build/issues/2611#issuecomment-1239338538 
+python_version:
+  - 3.8
+  - 3.9
+  - 3.10
+  - 3.11
+  - 3.12
+

py/h2o_wave/conda/meta.yaml

@@ -15,20 +15,33 @@ about:
 
 requirements:
   build: 
+    {% if NO_ARCH == "0" %}
+    - python={{ python_version }}
+    {% endif %}
     {% for dep in pyproject['build-system']['requires'] %}
     - {{ dep.lower() }}
     {% endfor %}
   run:
+    {% if NO_ARCH == "0" %}
+    - python={{ python_version }}
+    {% endif %}
     {% for dep in project['dependencies'] %}
     - {{ dep.lower() }}
     {% endfor %}
 source:
   path: ..
 
 build:
-  noarch: python
   script: python -m pip install . -vv
+  string: py{{ python_version | replace(".", "") }}
   entry_points:
     {% for module, entrypoints in project['scripts'].items() %}
       - {{ module }} = {{ entrypoints }}
-    {% endfor %}
+    {% endfor %}
+  include_recipe: False
+  script_env:
+    - NO_ARCH
+  {% if NO_ARCH == "1" %}
+  noarch: python
+  string: "0"
+  {% endif %}

py/h2o_wave/hatch_build.py

@@ -26,6 +26,10 @@ def initialize(self, _version, build_data):
         if not platform:
             # Create a default metadata file in case of noarch builds.
             create_metadata_file('linux', 'amd64')
+            # If conda build, copy binaries into package.
+            binaries_path = os.path.join('conda', 'temp')
+            if os.environ.get('CONDA_BUILD') and os.path.exists(binaries_path):
+                self.copy_binaries_into_package(binaries_path)
             return
 
         build_data['tag'] = f'py3-none-{platform}'
@@ -42,12 +46,7 @@ def initialize(self, _version, build_data):
         elif platform == 'manylinux1_x86_64':
             operating_system = 'linux'
 
-        binaries_path = os.path.join('..', '..', 'build', f'wave-{version}-{operating_system}-{arch}')
-        if not os.path.exists(binaries_path):
-            raise Exception(f'{binaries_path} does not exist. Run make release first to generate server binaries.')
-
-        self.copy_files(binaries_path, 'tmp', ['demo', 'examples', 'test'])
-        self.copy_files('project_templates', 'tmp', [], True)
+        self.copy_binaries_into_package(os.path.join('..', '..', 'build', f'wave-{version}-{operating_system}-{arch}'))
 
         create_metadata_file(operating_system, arch)
 
@@ -61,5 +60,13 @@ def copy_files(self, src, dst, ignore, keep_dir=False) -> None:
             elif os.path.isdir(src_file) and file_name not in ignore:
                 self.copy_files(src_file, dst_file, ignore)
 
+    def copy_binaries_into_package(self, binaries_path):
+        if not os.path.exists(binaries_path):
+            raise Exception(f'{binaries_path} does not exist. Run make release first to generate server binaries.')
+
+        self.copy_files(binaries_path, 'tmp', ['demo', 'examples', 'test'])
+        self.copy_files('project_templates', 'tmp', [], True)
+
     def finalize(self, version: str, build_data: Dict[str, Any], artifact_path: str) -> None:
         shutil.rmtree('tmp', ignore_errors=True)
+

website/docs/installation.mdx

@@ -22,9 +22,6 @@ pip install h2o-wave
 conda config --append channels conda-forge
 conda install -c h2oai h2o-wave
 ```
-:::info
-Conda packaging does not contain Wave server which means, you will still need to run the wave server (waved) [separately](/docs/tutorial-hello#step-1-start-the-wave-server).
-:::
 
 </TabItem>
 
@@ -64,9 +61,6 @@ conda config --append channels conda-forge
 conda install -c h2oai h2o-wave
 ```
 
-:::info
-Conda packaging does not contain Wave server which means, you will still need to run the wave server (waved) [separately](/docs/tutorial-hello#step-1-start-the-wave-server).
-:::
 
 </TabItem>
 </Tabs>