Add Docker compose settings. (#121)

* Remove psutil version limit

* Update docling and chonkie version.

* Bump version to 1.0.14

* Update docling core version.

* wip

* Fix the docker file path.

* wip

* Fix dockerfile path.

* wip

* Add docker-compose files.

* Add document home to docker mount.

* Add start and stop scripts.

* Fix white spaces.

Author: pengfeng

.github/workflows/build_llmeds_app.yml

@@ -64,6 +64,7 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: ./
+          file: docker/Dockerfile
           push: true
           tags: ${{ env.DOCKER_USERNAME }}/${{ env.APP_NAME }}:latest
           platforms: linux/amd64,linux/arm64

docker/README.md

@@ -0,0 +1,38 @@
+# Run LeetTools Docker containers
+
+We can use Docker Compose to start the LeetTools service and web UI.
+
+## Usage
+
+To build the Docker images, you can run the following command:
+```bash
+docker/build.sh
+```
+
+If you do not want to build the Docker images, you can pull the latest images from the 
+Docker Hub by running the following command:
+```bash
+docker compose --profile full pull
+```
+
+To start the LeetTools service 
+```bash
+docker/start.sh
+```
+
+Now you can go [http://localhost:3000](http://localhost:3000) to access the LeetTools web UI.
+
+By default this command will mount $LEET_HOME directory as read-write volume and 
+$DOCUMENETS_HOME directory as read-only volume to the Docker containers. The $DOCUMENETS_HOME
+is ~/Documents directory on Linux and macOS and C:\Users\<username>\Documents on Windows.
+
+If you need to change these behaviors, you can change the settings in the 
+[docker/.env](docker/.env) file.
+
+To stop the LeetTools service, you can run the following command:
+```bash
+docker/stop.sh
+```
+
+Note that the leettols-web application is currently under development and not open sourced
+yet. We plan to open source it in the near future.

docker/build.sh

@@ -5,6 +5,8 @@ set -e -u
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 BASE_DIR="$(cd "$DIR"/.. && pwd)"
 
+org_name="leettools"
+app_name="leettools"
 # read the version number project.toml file and use in the docker image
 version=$(grep "^version = " "$BASE_DIR"/pyproject.toml | cut -d'"' -f2)
 
@@ -23,9 +25,9 @@ cd "$BASE_DIR"
 cp "$DIR"/dockerignore.template "$BASE_DIR"/.dockerignore
 cat "$BASE_DIR"/.gitignore >> "$BASE_DIR"/.dockerignore
 
-docker build -t leettools-dev/leettools:"${version}" -f "$DIR"/Dockerfile "$BASE_DIR"
+docker build -t "${org_name}/${app_name}":"${version}" -f "$DIR"/Dockerfile "$BASE_DIR"
 
 # tag the image with the latest version
-docker tag leettools-dev/leettools:"${version}" leettools-dev/leettools:latest
+docker tag "${org_name}/${app_name}":"${version}" "${org_name}/${app_name}":latest
 
 popd > /dev/null

docker/docker-compose.yml

@@ -0,0 +1,45 @@
+version: "3.8"
+
+x-logging: &default-logging # Define a reusable anchor for logging configuration
+  driver: json-file
+  options:
+    max-size: "50m"
+    max-file: "10"
+
+services:
+  leettools:
+    image: leettools/leettools:${LEETTOOLS_VERSION}
+    logging: *default-logging
+    container_name: leettools
+    restart: always
+    ports:
+      - "8000:8000"
+    volumes:
+      - ${LEET_HOME}:/leettools
+      # mount any extra directories that have documents need to be processed
+      - ${DOCUMENETS_HOME}:/leettools/web/incoming/documents:ro
+
+    environment:
+      - LEET_HOME=/leettools
+    env_file:
+      - ${LEETTOOLS_ENV_FILE}
+    profiles: ["full", "compact"]
+
+  llmeds-web:
+    image: leettools/leettools-web:${LEETTOOLS_WEB_VERSION}
+    logging: *default-logging
+    restart: always
+    depends_on:
+      - leettools
+    ports:
+      - "3000:3000"
+    volumes:
+      - ${LEET_HOME}/web:/leettools/web
+      # mount any extra directories that have documents need to be processed
+      - ${DOCUMENETS_HOME}:/leettools/web/incoming/documents:ro
+
+    environment:
+      - LEET_HOME=/leettools
+      - SERVICE_DOMAIN=http://leettools:8000
+      - DEFAULT_LANGUAGE=${DEFAULT_LANGUAGE}
+    profiles: ["full", "compact"]

docker/env.template

@@ -0,0 +1,10 @@
+# env variables for docker-compose.yml
+
+# fix to a stable version if needed
+LEETTOOLS_VERSION=latest
+LEETTOOLS_WEB_VERSION=latest
+
+LEETTOOLS_ENV_FILE=../.env
+DEFAULT_LANGUAGE=en
+# specify the DOCUMENETS_HOME variable if you want to use a different directory
+# DOCUMENETS_HOME=~/documents

docker/start.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+set -e -u
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# check if the .env file exists in the DIR
+env_file="$DIR/.env"
+if [ ! -f "$env_file" ]; then
+    echo "Error: .env file not found in $DIR. Please create one from $DIR/.env.template"
+    exit 1
+fi
+
+# load the .env file into the environment by exporting the variables
+while IFS='=' read -r name value; do
+if [[ ! $name =~ ^\# ]] && [[ -n $name ]]; then
+    # we only export the variables that are not set in the environment
+    current_env_value="${!name:-}"
+    if [ -z "${current_env_value}" ]; then
+        export "$name=$value";
+    else
+        # print a warning message if the two values are different
+        if [ "${current_env_value}" != "${value}" ]; then
+            echo "[start.sh warning] $name is set to $current_env_value in the current environment, but the $env_file file has a different value: $value"
+        fi;
+    fi;
+fi;
+done < "${env_file}"
+
+# set DOCUMENETS_HOME to the Documents directory on different OS
+# windows: C:\Users\<username>\Documents
+# mac: ~/Documents
+# linux: ~/Documents
+
+# if no DOCUMENETS_HOME is set in the .env file, set it to the Documents directory on different OS
+if [ -z "${DOCUMENETS_HOME:-}" ]; then
+    case "$(uname -s)" in
+        Darwin|Linux)
+            export DOCUMENETS_HOME=~/Documents
+            ;;
+        CYGWIN*|MINGW*|MSYS*)
+            export DOCUMENETS_HOME="$USERPROFILE/Documents"
+            ;;
+        *)
+            echo "Unsupported operating system, using the value from .env file"
+            ;;
+    esac
+fi
+
+pushd . > /dev/null
+
+cd "$DIR"
+
+docker compose --profile full down
+docker compose --profile full up -d
+
+popd > /dev/null
+
+echo "LeetTools Docker containers started."

docker/start_docker.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+# this program is used to start the docker container for the leettools service
+
 set -e -u
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
@@ -53,7 +55,9 @@ if [ ${#MOUNT_DIRS[@]} -gt 0 ]; then
   done
 fi
 
-container_name="leettools-svc"
+org_name="leettools"
+app_name="leettools"
+container_name="leettools"
 
 # If force remove is enabled, stop and remove existing container
 if [ "$FORCE_REMOVE" = true ]; then
@@ -86,10 +90,10 @@ if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
 fi
 
 # check if the docker image exists
-if ! docker images leettools-dev/leettools:"${version}" | grep -q "${version}"; then
-    echo "Docker image leettools-dev/leettools:${version} does not exist"
+if ! docker images "${org_name}/${app_name}":"${version}" | grep -q "${version}"; then
+    echo "Docker image ${org_name}/${app_name}:${version} does not exist"
     version="latest"
-    if ! docker images leettools-dev/leettools:"${version}" | grep -q "${version}"; then
+    if ! docker images "${org_name}/${app_name}":"${version}" | grep -q "${version}"; then
         echo "Docker image leettools-dev/leettools:${version} does not exist"
         exit 1
     fi
@@ -143,7 +147,7 @@ echo "docker run -d ${envfile_opt} ${mount_opts} \\
     -v \"$LEET_HOME\":\"$leet_home_in_docker\" \\
     -v \"$EDS_DATA_DIR\":\"$leet_home_in_docker/data\" \\
     -v \"$EDS_LOG_DIR\":\"$leet_home_in_docker/logs\" \\
-    leettools-dev/leettools:\"${version}\""
+    ${org_name}/${app_name}:\"${version}\""
 
 # run the docker container
 # shellcheck disable=SC2086
@@ -154,4 +158,4 @@ docker run -d ${envfile_opt} ${mount_opts} \
     -v "$LEET_HOME":"$leet_home_in_docker" \
     -v "$EDS_DATA_DIR":"$leet_home_in_docker/data" \
     -v "$EDS_LOG_DIR":"$leet_home_in_docker/logs" \
-    leettools-dev/leettools:"${version}"
+    ${org_name}/${app_name}:"${version}"

docker/stop.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -e -u
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+pushd . > /dev/null 
+cd "$DIR"
+
+docker compose --profile full down
+
+popd > /dev/null
+
+echo "LeetTools Docker containers stopped."
+

scripts/run.sh

@@ -34,13 +34,13 @@ if [ -f "${env_file}" ]; then
     while IFS='=' read -r name value; do
     if [[ ! $name =~ ^\# ]] && [[ -n $name ]]; then
       # we only export the variables that are not set in the environment
-      current_value="${!name:-}"
-      if [ -z "${current_value}" ]; then
+      current_env_value="${!name:-}"
+      if [ -z "${current_env_value}" ]; then
         export "$name=$value";
       else
         # print a warning message if the two values are different
-        if [ "${current_value}" != "${value}" ]; then
-          echo "[run.sh warning] Variable $name is already set to $current_value, but the $env_file file has a different value: $value"
+        if [ "${current_env_value}" != "${value}" ]; then
+          echo "[run.sh warning] $name is set to $current_env_value in the current environment, but the $env_file file has a different value: $value"
         fi;
       fi;
     fi;