ci: release libdatahike artifacts (#749)

* fix: libdatahike build

* test: testing libdatahike compiled binary

* fix: persisting to workspace

* ci: graalvm home set

* fixup! ci: graalvm home set

* fixup! ci: graalvm home set

* fixup! ci: graalvm home set

* fixup! ci: graalvm home set

* fixup! ci: graalvm home set

* fixup! ci: graalvm home set

Author: TimoKramer

.circleci/config.yml

@@ -190,8 +190,10 @@ jobs:
           at: /home/circleci
       - run:
           name: Release pod on pod registry
-          command: bb release pod
-
+          command: |
+            if ["$CIRCLE_ORGANIZATION_ID" = "replikativ"]; then
+              bb release pod
+            fi
 
 workflows:
   build-test-and-deploy:

.circleci/scripts/gen_ci.clj

@@ -5,7 +5,7 @@
    [clojure.string :as str]
    [flatland.ordered.map :refer [ordered-map]]))
 
-(def graalvm-version "24.0.1")
+(def graalvm-version "24.0.2")
 
 (defn run
   ([cmd-name cmd]
@@ -39,6 +39,7 @@
       :resource_class resource-class}
      :working_directory "/home/circleci/replikativ"
      :environment {:GRAALVM_VERSION graalvm-version
+                   :GRAALVM_HOME "/home/circleci/graalvm"
                    :DTHK_PLATFORM "linux"
                    :DTHK_ARCH arch
                    :PATH "/bin:/home/circleci/graalvm/bin:/home/circleci/clojure/bin:/home/circleci/bin"
@@ -69,17 +70,26 @@ sudo update-alternatives --set javac /home/circleci/graalvm/bin/javac"
       (run "Build native image"
            "cd /home/circleci/replikativ
 bb ni-cli")
+      (run "Build libdatahike"
+           "cd /home/circleci/replikativ
+bb ni-compile")
       (run "Test native image"
            "cd /home/circleci/replikativ
 bb test native-image")
+      (run "Test babashka pod"
+           "cd /home/circleci/replikativ
+bb test bb-pod")
+      (run "Test libdatahike"
+           "cd /home/circleci/replikativ
+bb test libdatahike")
       {:persist_to_workspace
        {:root "/home/circleci/"
-        :paths ["replikativ/dthk"]}}
+        :paths ["replikativ/dthk" "replikativ/libdatahike/target"]}}
       {:save_cache
        {:paths ["~/.m2" "~/graalvm"]
         :key cache-key}}])))
 
-(defn release-native-image
+(defn release-artifacts
   [arch]
   (let [cache-key (str arch "-deps-linux-{{ checksum \"deps.edn\" }}")]
     (ordered-map
@@ -94,6 +104,9 @@ bb test native-image")
       (run "Release native image"
            "cd /home/circleci/replikativ
 bb release native-image")
+      (run "Release libdatahike"
+           "cd /home/circleci/replikativ
+bb release libdatahike")
       {:persist_to_workspace
        {:root "/home/circleci/"
         :paths ["replikativ/dthk"]}}
@@ -116,8 +129,8 @@ bb release native-image")
    :jobs (ordered-map
           :build-linux-amd64 (build-native-image "amd64" "large")
           :build-linux-aarch64 (build-native-image "aarch64" "arm.large")
-          :release-linux-amd64 (release-native-image "amd64")
-          :release-linux-aarch64 (release-native-image "aarch64"))
+          :release-linux-amd64 (release-artifacts "amd64")
+          :release-linux-aarch64 (release-artifacts "aarch64"))
    :workflows (ordered-map
                :version 2
                :native-images

.cirrus.yml

@@ -9,8 +9,8 @@ build-task:
     DTHK_PLATFORM: macos
     DTHK_ARCH: aarch64
     INSTALL_DIR: ${HOME}
-    GRAALVM_VERSION: 22.0.2
-    GRAALVM_SLUG: 9.1
+    GRAALVM_VERSION: 24.0.2
+    GRAALVM_SLUG: 11.1
     GRAALVM_HOME: ${HOME}/graalvm-community-openjdk-${GRAALVM_VERSION}+${GRAALVM_SLUG}/Contents/Home/
     GITHUB_TOKEN: ENCRYPTED[!86a3791406be6ffc9490f464846ec5385908ae8d3d2a9eeb1457d529060816e14e9d7b88ecd190970add6a5d5d4e7b59!]
   script: |
@@ -43,15 +43,18 @@ build-task:
     echo $PATH
     native-image --version
 
-    # compile
+    # compile native artifacts
     bb ni-cli
+    bb ni-compile
 
-    # test
+    # test native artifacts
     bb test native-image
     bb test bb-pod
+    bb test libdatahike
 
-    # upload artifact
+    # release native artifacts
     bb release native-image
+    bb release libdatahike
 
   binaries_artifacts:
     path: "dist/*"

.github/workflows/native-image.yml

@@ -24,7 +24,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     env:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      GRAALVM_VERSION: 24.0.1
+      GRAALVM_VERSION: 24.0.2
       DTHK_PLATFORM: macos
       DTHK_ARCH: amd64
     steps:
@@ -45,7 +45,7 @@ jobs:
         uses: graalvm/setup-graalvm@v1
         with:
           java-version: '24'
-          distribution: 'graalvm'
+          distribution: 'graalvm-community'
           github-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Install clojure tools
@@ -57,14 +57,24 @@ jobs:
       - name: Build native image
         run: bb ni-cli
 
+      - name: Build libdatahike
+        run: bb ni-compile
+
       - name: Test native image
-        run: |
-          bb test native-image
-          bb test bb-pod
+        run: bb test native-image
+
+      - name: Test babashka pod
+        run:  bb test bb-pod
+
+      - name: Test libdatahike
+        run:  bb test libdatahike
 
       - name: Release native-image
         run: bb release native-image
 
+      - name: Release libdatahike
+        run: bb release libdatahike
+
       - name: Save cache
         uses: actions/cache/save@v4
         id: save-cache

.gitignore

@@ -32,8 +32,10 @@ out/
 /datahike
 tests.user.edn
 libdatahike.zip
-dhi
-dhi.build_artifacts.txt
+dthk
+libdatahike*.zip
 trace.edn
 tmp.edn
-.shadow-cljs
+.shadow-cljs
+.claude
+repl-mcp.log

bb.edn

@@ -88,7 +88,7 @@
                                :depends [http-server-clean]
                                :task    (build/compile-java (-> config :build :http-server-clj))}
 
-         http-server-ccompile {:doc  "Compile clojure namespaces"
+         http-server-ccompile {:doc  "Compile clojure namespaces for http-server"
                                :task (build/compile-clojure (-> config :build :http-server-clj))}
 
          http-server-pom {:doc  "Create pom file"
@@ -119,7 +119,7 @@
                  :depends [jcompile ni-check]
                  :task (clojure "-M:native-cli")}
 
-         ni-ccompile {:doc "Create native cpp library"
+         ni-ccompile {:doc "Compile clojure namespaces for native-image"
                       :task (build/compile-clojure (-> config :build :native))}
 
          ni-uber {:doc "Build native image uber jar"
@@ -128,11 +128,4 @@
 
          ni-compile {:doc "Create native cpp library"
                      :depends [ni-uber]
-                     :task (build/native-compile config (-> config :build :native))}
-
-         ni-release {:doc "Build and release native assets to GitHub"
-                     :depends [ni-compile]
-                     :task (let [{:keys [artifact target-dir]} (-> config :build :native)]
-                             (fs/zip artifact [target-dir])
-                             (release/gh-release artifact config)
-                             (fs/delete artifact))}}}
+                     :task (build/native-compile config (-> config :build :native))}}}

bb/resources/native-image-tests/run-libdatahike-tests

@@ -0,0 +1,10 @@
+#!/bin/env bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+export LD_LIBRARY_PATH=./libdatahike/target
+
+rm -rf "/tmp/libdatahike-test"
+./libdatahike/target/test_cpp "{:store {:backend :file :path \"/tmp/libdatahike-test\"} :schema-flexibility :read}" "[{:name \"Alice\"}]" "[:find ?e ?v :where [?e :name ?v]]"

bb/src/tools/build.clj

@@ -54,6 +54,11 @@
                                            :repo repo-config
                                            :version-str (version/string repo-config)})))
 
+(defn lib-path [repo-config {:keys [target-dir jar-pattern] :as project-config}]
+  (str target-dir "/" (render jar-pattern {:project project-config
+                                           :repo repo-config
+                                           :version-str (version/string repo-config)})))
+
 (defn jar
   "Builds jar file"
   [repo-config {:keys [class-dir target-dir src-dirs resource-dir] :as project-config}]
@@ -89,6 +94,7 @@
                (str "-o " project-name)
                "--shared"
                "-H:+ReportExceptionStackTraces"
+               "-H:+GenerateBuildArtifactsFile"
                "-J-Dclojure.spec.skip-macros=true"
                "-J-Dclojure.compiler.direct-linking=true"
                (str "-H:IncludeResources=" (version/string repo-config))
@@ -101,8 +107,9 @@
       (fs/delete-tree project-target-dir)
       (fs/create-dir project-target-dir)
       (run! #(fs/move % project-target-dir)
-            (concat ["graal_isolate.h" "graal_isolate_dynamic.h"]
+            (concat ["graal_isolate.h" "graal_isolate_dynamic.h" "build-artifacts.json"]
                     (map (fn [ext] (str project-name ext))
-                         [".so" ".h" "_dynamic.h" ".build_artifacts.txt"]))))
+                         [".so" ".h" "_dynamic.h"]))))
     (do (println "GRAALVM_HOME not set!")
-        (println "Please set GRAALVM_HOME to the root of the graalvm directory on your system."))))
+        (println "Please set GRAALVM_HOME to the root of the graalvm directory on your system.")
+        (System/exit 1))))

bb/src/tools/release.clj

@@ -2,7 +2,6 @@
   (:require
    [babashka.fs :as fs]
    [babashka.http-client :as http]
-   [babashka.process :as p]
    [borkdude.gh-release-artifact :as gh]
    [cheshire.core :as json]
    [clojure.java.io :as io]
@@ -108,6 +107,16 @@
                                                :lib lib
                                                :version version})))))
 
+(defn zip-lib [config project]
+  (let [{:keys [target-dir zip-pattern]} (-> config :release project) lib "libdatahike"
+        version (version/string config)]
+    (if-not (fs/exists? target-dir)
+      (do (println (str "ERROR: " target-dir " path not found, please compile first."))
+          (System/exit 1))
+      (let [zip-name (zip-path lib version "." zip-pattern)]
+        (fs/zip zip-name [target-dir])
+        zip-name))))
+
 (defn zip-cli [config project]
   (let [{:keys [target-dir binary-name zip-pattern]} (-> config :release project)
         lib (:lib config)
@@ -117,15 +126,18 @@
       (do (println (str "ERROR: " binary-path " executable not found, please compile first."))
           (System/exit 1))
       (let [zip-name (zip-path lib version target-dir zip-pattern)]
-        (p/shell "zip" zip-name binary-name)
+        (fs/zip zip-name [binary-name])
         zip-name))))
 
 (defn -main [config args]
   (let [cmd (first args)]
     (case cmd
-      "jar" (gh-release config (build/jar-path config (-> config :build :clj)))
+      "jar" (->> (build/jar-path config (-> config :build :clj))
+                 (gh-release config))
       "native-image" (->> (zip-cli config :native-cli)
                           (gh-release config))
       "pod" (pod-release config)
+      "libdatahike" (->> (zip-lib config :libdatahike)
+                         (gh-release config))
       (do (println "ERROR: Command not found: " cmd)
           (System/exit 1)))))

bb/src/tools/test.clj

@@ -49,6 +49,12 @@
     (p/shell "./bb/resources/native-image-tests/run-native-image-tests")
     (println "Native image cli missing. Please run 'bb ni-cli' and try again.")))
 
+(defn libdatahike []
+  (if (fs/exists? "./libdatahike/target")
+    (do (p/shell "./libdatahike/compile-cpp")
+        (p/shell "./bb/resources/native-image-tests/run-libdatahike-tests"))
+    (println "libdatahike binaries missing. Please run 'bb ni-compile' and try again.")))
+
 (defn bb-pod []
   (if (fs/exists? "./dthk")
     (p/shell "./bb/resources/native-image-tests/run-bb-pod-tests.clj")
@@ -67,13 +73,15 @@
   (specs)
   (back-compat config)
   (native-image)
+  (libdatahike)
   (bb-pod)
   (cljs-compile-test))
 
 (defn -main [config & args]
   (if (seq args)
     (case (first args)
       "native-image" (native-image)
+      "libdatahike" (libdatahike)
       "bb-pod" (bb-pod)
       "back-compat" (back-compat config)
       "specs" (specs)