Merging release 0.4.0 from main

.dockerignore

@@ -1,12 +1,18 @@
 # Ignore local virtualenv (if present)
-.venv/
+.venv
+
+# Python bytecode and compiled files
+__pycache__/
+*.py[cod]
+*$py.class
+.Python
+*.so
 
 .env*
 **/.env*
 .invenio.private*
-docker/nginx/samlPrivateKey.key
-docker/nginx_containerized/samlPrivateKey.key
-docker/nginx_staging/samlPrivateKey.key
+docker/nginx_local/samlPrivateKey.key
+docker/nginx_local/samlCertificate.key
 docker-compose*
 
 .git/

.github/workflows/CI.yml → .github/workflows/build.yml

@@ -99,11 +99,11 @@ jobs:
         with:
           submodules: recursive
 
-      # Set up Python 3.9 environment
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v1
+      # Set up Python 3.12 environment
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.9"
+          python-version: "3.12.x"
 
       # Cache docker images so they don't rebuild every time
       # - name: Cache Local Images
@@ -142,14 +142,18 @@ jobs:
       - name: Start containers
         if: always()
         run: |
+          echo "[cli]" >> .invenio.private && \
+          echo "services_setup=False" >> .invenio.private && \
+          echo "instance_path=/opt/invenio/var/instance" >> .invenio.private
+          touch tests/.env
           touch .env
-          docker compose --file docker-compose.yml --file docker-compose.dev.yml up -d
+          docker compose --file docker-compose.yml up -d
 
       - name: Destroy containers
         if: always()
         run: |
           touch .env
-          docker compose --file docker-compose.yml --file docker-compose.dev.yml down
+          docker compose --file docker-compose.yml down
 
       - name: Prune Docker
         run: docker system prune -af

.github/workflows/documentation.yml

@@ -16,7 +16,7 @@ jobs:
       - uses: actions/setup-python@v5
       - name: Install dependencies
         run: |
-          pip install sphinx furo myst-parser
+          pip install sphinx furo myst-parser sphinx-copybutton
       - name: Sphinx build
         run: |
           sphinx-build docs/source docs/build

.github/workflows/tests.yml

@@ -2,9 +2,9 @@ name: Tests
 
 on:
   push:
-    branches: [ main ]
+    branches: [ main, dev ]
   pull_request:
-    branches: [ main ]
+    branches: [ main, dev ]
   schedule:
     # * is a special character in YAML so you have to quote this string
     - cron: "0 3 * * 6"
@@ -25,16 +25,26 @@ jobs:
         with:
           submodules: true
 
-      - name: Set up Python 3.9
+      - name: Set up Python 3.12
         uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
+          python-version: '3.12.x'
 
       - name: Install prerequisites
         run: |
           sudo apt-get update
           sudo apt-get install -y pkg-config libxml2-dev libxmlsec1-dev libxmlsec1-openssl
 
+      - name: Install uv
+        run: |
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+
+      - name: Install python dependencies
+        run: |
+          uv sync --frozen --all-extras
+          export CFLAGS="-Wno-error=incompatible-pointer-types"
+          uv pip install --reinstall --no-binary=lxml --no-binary=xmlsec "lxml==5.2.1" "xmlsec==1.3.14"
+
       - name: set up docker
         uses: docker/setup-buildx-action@v3
 
@@ -43,14 +53,6 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y docker-compose
 
-      - name: Install pipenv
-        run: |
-          python -m pip install --upgrade pip
-          pip install pipenv -vv
-
-      - name: Install python dependencies
-        run: pipenv install --dev
-
       - name: Use Node.js 20.x
         uses: actions/setup-node@v4
         with:
@@ -61,7 +63,7 @@ jobs:
           echo "[cli]" >> .invenio.private && \
           echo "services_setup=False" >> .invenio.private && \
           echo "instance_path=/opt/invenio/var/instance" >> .invenio.private
-          touch site/tests/.env
+          touch tests/.env
 
       - name: Run tests
         env:
@@ -82,9 +84,8 @@ jobs:
           SPARKPOST_API_KEY: ${{ secrets.TEST_SPARKPOST_API_KEY }}
           SPARKPOST_USERNAME: ${{ secrets.TEST_SPARKPOST_USERNAME }}
         run: |
-          chmod +x site/run_tests.sh
-          cd site
-          bash run_tests.sh
+          chmod +x run-tests.sh
+          bash run-tests.sh
 
       # TODO: Add frontend tests
       # - name: Run eslint test
@@ -93,14 +94,16 @@ jobs:
       # - name: Run translations test
       #   run: ./run-i18n-tests.sh
 
-      # - name: Install deps for frontend tests
-      #   working-directory: ./site/kcworks/assets/semantic-ui/js/kcworks
-      #   run: npm install
+      - name: Install deps for frontend tests
+        working-directory: .
+        run: npm install
 
       # - name: Install deps for frontend tests - translations
       #   working-directory: ./translations/kcworks
       #   run: npm install
 
-      # - name: Run frontend tests
-      #   working-directory: ./site/kcworks/assets/semantic-ui/js/kcworks
-      #   run: npm test
+      - name: Run frontend tests
+        working-directory: .
+        run: |
+          chmod +x run-js-tests.sh
+          bash run-js-tests.sh

.gitignore

@@ -1,8 +1,8 @@
-# Byte-compiled / optimized / DLL files
+# Python bytecode and compiled files
 __pycache__/
 *.py[cod]
-
-# C extensions
+*$py.class
+.Python
 *.so
 
 # vi(m) swap files
@@ -34,8 +34,10 @@ static/xsl/
 
 # Distribution / packaging
 .Python
+build/
 env/
 develop-eggs/
+dist/
 downloads/
 eggs/
 .eggs/
@@ -68,7 +70,7 @@ coverage.xml
 *,cover
 
 # Sphinx documentation
-# docs/_build/
+docs/build/
 
 # PyBuilder
 # target/
@@ -78,9 +80,11 @@ test.db
 .pytest_cache/
 
 # Environments
-# .venv/
+.venv
+.vscode
 
 # Logs
+logs/
 logs/*
 *.log
 **/logs/*
@@ -106,6 +110,7 @@ site/kcworks/assets/semantic-ui/less/site/globals/site.overrides
 # Docker private environment variables
 .env
 .env.private
+.env-tests
 
 # S3 default bucket location
 data/default/*

Dockerfile

@@ -9,56 +9,107 @@
 # Note: It is important to keep the commands in this file in sync with your
 # bootstrap script located in ./scripts/bootstrap.
 
-FROM registry.cern.ch/inveniosoftware/almalinux:1
+FROM ghcr.io/astral-sh/uv:python3.12-bookworm
 
-# TODO: Add env variables below?
-# ENV PYTHONDONTWRITEBYTECODE 1
-# ENV PYTHONFAULTHANDLER 1
+ENV INVENIO_INSTANCE_PATH=/opt/invenio/var/instance \
+    INVENIO_SITE_UI_URL=https://localhost \
+    INVENIO_SITE_API_URL=https://localhost \
+    LANG=en_US.UTF-8 \
+    LANGUAGE=en_US:en \
+    LC_ALL=en_US.UTF-8 \
+    UV_PROJECT_ENVIRONMENT=/opt/invenio/src/.venv \
+    VIRTUAL_ENV=/opt/invenio/src/.venv \
+    PATH="/opt/invenio/src/.venv/bin:${PATH}"
+
+# Create instance path and set working directory
+RUN mkdir -p ${INVENIO_INSTANCE_PATH} && \
+    mkdir -p /opt/invenio/src
+WORKDIR /opt/invenio/src
+
+# Install prerequisites and set up locales
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    python3-dev \
+    git \
+    libxml2 \
+    libxml2-dev \
+    libxmlsec1 \
+    libxmlsec1-dev \
+    libxmlsec1-openssl \
+    libxmlsec1-gnutls \
+    xmlsec1 \
+    libssl-dev \
+    libltdl-dev \
+    libpq-dev \
+    libpcre3-dev \
+    locales \
+    libpcre3 \
+    libpcre3-dev \
+    libssl-dev \
+    libffi-dev \
+    uuid-dev \
+    wget \
+    curl \
+    && rm -rf /var/lib/apt/lists/* \
+    && sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen \
+    && locale-gen \
+    && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
+    && apt-get update \
+    && apt-get install -y nodejs \
+    && rm -rf /var/lib/apt/lists/*
 
-# Install prerequisites for building xmlsec Python package
-# also adds ps command for debugging
-RUN dnf install libxml2-devel xmlsec1-devel xmlsec1-openssl-devel libtool-ltdl-devel procps -y
-# Copy over directory for kcr instance Python package
-COPY site ./site
+# Configure npm to install packages locally
+RUN npm config set prefix '/opt/invenio/src/node_modules' && \
+    npm config set global false
 
-COPY Pipfile Pipfile.lock ./
+# Copy all source files needed for dependencies and webpack
+COPY . .
 
-# NOTE: turned off --deploy for dev
-RUN pipenv install --system
+# Install python dependencies in virtual environment
+RUN uv venv && \
+    . .venv/bin/activate && \
+    # First install lxml and xmlsec with system libxml2
+    # Then install the rest of the dependencies without reinstall
+    uv sync --frozen --compile-bytecode && \
+    export CFLAGS="-Wno-error=incompatible-pointer-types" && \
+    uv pip install --reinstall --no-binary=lxml --no-binary=xmlsec "lxml==5.2.1" "xmlsec==1.3.14" && \
+    uv clean
 
 RUN echo "[cli]" >> .invenio.private && \
     echo "services_setup=False" >> .invenio.private && \
     echo "instance_path=/opt/invenio/var/instance" >> .invenio.private
 
-# Copying whole app directory into /opt/invenio/src
-# (WORKDIR is set to that folder in base image)
-COPY ./ .
-ENV INVENIO_INSTANCE_PATH=/opt/invenio/var/instance \
-    INVENIO_SITE_UI_URL=https://localhost \
-    INVENIO_SITE_API_URL=https://localhost \
-    MIGRATION_SERVER_DOMAIN=localhost \
-    MIGRATION_SERVER_PROTOCOL=http \
-    MIGRATION_API_TOKEN=changeme
-
-RUN cp -r ./docker/uwsgi/uwsgi_rest.ini ${INVENIO_INSTANCE_PATH}/uwsgi_rest.ini && \
-    cp -r ./docker/uwsgi/uwsgi_ui.ini ${INVENIO_INSTANCE_PATH}/uwsgi_ui.ini && \
-    cp -r ./docker/startup_ui.sh ${INVENIO_INSTANCE_PATH}/startup_ui.sh && \
-    cp -r ./docker/startup_api.sh ${INVENIO_INSTANCE_PATH}/startup_api.sh && \
-    cp -r ./docker/startup_worker.sh ${INVENIO_INSTANCE_PATH}/startup_worker.sh && \
+# Copy required files to instance path
+RUN cp ./docker/uwsgi/uwsgi_rest.ini ${INVENIO_INSTANCE_PATH}/uwsgi_rest.ini && \
+    cp ./docker/uwsgi/uwsgi_ui.ini ${INVENIO_INSTANCE_PATH}/uwsgi_ui.ini && \
+    cp ./docker/startup_*.sh ${INVENIO_INSTANCE_PATH}/ && \
+    chmod +x ${INVENIO_INSTANCE_PATH}/startup_*.sh && \
     cp ./invenio.cfg ${INVENIO_INSTANCE_PATH}/invenio.cfg && \
     cp -r ./templates ${INVENIO_INSTANCE_PATH}/templates && \
     cp -r ./app_data/ ${INVENIO_INSTANCE_PATH}/app_data
-RUN chmod +x ${INVENIO_INSTANCE_PATH}/startup_ui.sh && \
-    chmod +x ${INVENIO_INSTANCE_PATH}/startup_api.sh && \
-    chmod +x ${INVENIO_INSTANCE_PATH}/startup_worker.sh
 
-RUN invenio collect --verbose && invenio webpack clean create && \
+# Install local dependencies
+RUN . .venv/bin/activate && \
+    uv pip install --editable . && \
+    uv pip install --editable ./site/kcworks/dependencies/invenio-modular-deposit-form && \
+    uv pip install --editable ./site/kcworks/dependencies/invenio-group-collections-kcworks && \
+    uv pip install --editable ./site/kcworks/dependencies/invenio-modular-detail-page && \
+    uv pip install --editable ./site/kcworks/dependencies/invenio-record-importer-kcworks && \
+    uv pip install --editable ./site/kcworks/dependencies/invenio-remote-api-provisioner && \
+    uv pip install --editable ./site/kcworks/dependencies/invenio-remote-user-data-kcworks && \
+    uv pip install --editable ./site/kcworks/dependencies/invenio-communities && \
+    uv pip install --editable ./site/kcworks/dependencies/invenio-rdm-records && \
+    uv pip install --editable ./site/kcworks/dependencies/invenio-records-resources
+
+# Build assets
+RUN . .venv/bin/activate && \
+    invenio collect --verbose && \
+    invenio webpack clean create && \
     mkdir -p ${INVENIO_INSTANCE_PATH}/assets/less && \
-    cp ./assets/less/theme.config ${INVENIO_INSTANCE_PATH}/assets/less/theme.config && \
-    mkdir -p ${INVENIO_INSTANCE_PATH}/assets/templates/custom_fields && \
-    mkdir -p ${INVENIO_INSTANCE_PATH}/assets/templates/search && \
+    cp ./assets/less/theme.config ${INVENIO_INSTANCE_PATH}/assets/less/ && \
+    mkdir -p ${INVENIO_INSTANCE_PATH}/assets/templates/{custom_fields,search} && \
     invenio webpack install && \
-    invenio shell ./scripts/symlink_assets.py && \
+    invenio shell /opt/invenio/src/scripts/symlink_assets.py && \
     invenio webpack build
 
-ENTRYPOINT ["bash", "-c"]
+ENTRYPOINT ["/bin/bash", "-c"]