y-scope · anlowee · Jul 28, 2025 · Jul 27, 2025 · Jul 27, 2025 · Jul 27, 2025
diff --git a/tools/deployment/presto-clp/.env b/tools/deployment/presto-clp/.env
@@ -0,0 +1,35 @@
+# Coordinator common
+PRESTO_COORDINATOR_HTTPPORT="8080"
+PRESTO_COORDINATOR_SERVICENAME="presto-coordinator"
+
+# Coordinator clp.properties
+PRESTO_COORDINATOR_CONFIG_CLPPROPERTIES_METADATA_PROVIDERTYPE="mysql"
+PRESTO_COORDINATOR_CONFIG_CLPPROPERTIES_METADATA_DATABASE_URL="jdbc:mysql://localhost:6001"
+PRESTO_COORDINATOR_CONFIG_CLPPROPERTIES_METADATA_DATABASE_NAME="clp-db"
+PRESTO_COORDINATOR_CONFIG_CLPPROPERTIES_METADATA_DATABASE_USER="clp-user"
+PRESTO_COORDINATOR_CONFIG_CLPPROPERTIES_METADATA_DATABASE_PASSWORD="123456"
+PRESTO_COORDINATOR_CONFIG_CLPPROPERTIES_METADATA_TABLEPREFIX="clp_"
+PRESTO_COORDINATOR_CONFIG_CLPPROPERTIES_SPLITPROVIDER="mysql"
+
+# Coordinator config.properties
+PRESTO_COORDINATOR_CONFIG_CONFIGPROPERTIES_QUERY_MAXMEMORY="1GB"
+PRESTO_COORDINATOR_CONFIG_CONFIGPROPERTIES_QUERY_MAXMEMORYPERNODE="1GB"
+
+# Coordinator jvm.config
+PRESTO_COORDINATOR_CONFIG_JVMCONFIG_MAXHEAPSIZE="4G"
+PRESTO_COORDINATOR_CONFIG_JVMCONFIG_G1HEAPREGIONSIZE="32M"
+
+# Coordinator log.properties
+PRESTO_COORDINATOR_CONFIG_LOGPROPERTIES_LEVEL="DEBUG"
+
+# Coordinator node.properties
+PRESTO_COORDINATOR_CONFIG_NODEPROPERTIES_ENVIRONMENT="production"
+
+# Worker common
+PRESTO_WORKER_HTTPPORT="8080"
+
+# Worker node.properties
+PRESTO_WORKER_CONFIG_NODEPROPERTIES_LOCATION="worker-location"
+
+# CLP package archives
+CLP_PACKAGE_ARCHIVES=REPLACE_ME
diff --git a/tools/deployment/presto-clp/README.md b/tools/deployment/presto-clp/README.md
@@ -0,0 +1,76 @@
+# Setup local docker stack for presto + clp
+
+## Install docker
+
+Follow the guide here: [docker]
+
+# Launch clp-package
+
+1. Find the clp-package for test on our official website [clp-json-v0.4.0]. Here is a sample dataset for demo testing: [postgresql dataset].
+
+2. Untar the clp-package and the postgresql dataset.
+
+3. Replace the content of `/path/to/clp-json-package/etc/clp-config.yml` with the output of `demo-assets/init.sh generate_sample_clp_config`.
+
+4. Launch:
+
+```bash
+# You probably want to run a python 3.9 or newer virtual environment
+sbin/start-clp.sh
+```
+
+5. Compress:
+
+```bash
+# You can also use your own dataset
+sbin/compress.sh --timestamp-key 'timestamp' /path/to/postgresql.log
+```
+
+6. Use the following command to update `.env`:
+
+```bash
+demo-assets/init.sh update_metadata_config /path/to/clp-json-package
+```
+
+# Create Docker Cluster
+
+Create a local docker stack:
+
+```bash
+docker compose up
+```
+
+To create a docker stack with more than 1 worker (e.g., 3 workers):
+```
+docker compose up --scale presto-worker=3
+```
+
+# Use cli:
+
+After all containers are in "Started" states (check by `docker ps`):
+
+```bash
+# On your host
+docker exec -it compose-presto-coordinator-1 sh
+
+# In presto-coordinator container
+/opt/presto-cli --catalog clp --schema default --server localhost:8080
+```
+
+Example query:
+```sql
+SELECT * FROM default LIMIT 1;
+```
+
+# Delete docker Cluster
+
+```bash
+docker compose down
+```
+
+
+
+[clp-json-v0.4.0]: https://github.com/y-scope/clp/releases/tag/v0.4.0
+[docker]: https://docs.docker.com/engine/install
+[postgresql dataset]: https://zenodo.org/records/10516402
+
@@ -0,0 +1,10 @@
+connector.name=clp
+clp.metadata-provider-type=${PRESTO_COORDINATOR_CONFIG_CLPPROPERTIES_METADATA_PROVIDERTYPE}
+clp.metadata-db-url=${PRESTO_COORDINATOR_CONFIG_CLPPROPERTIES_METADATA_DATABASE_URL}
+clp.metadata-db-name=${PRESTO_COORDINATOR_CONFIG_CLPPROPERTIES_METADATA_DATABASE_NAME}
+clp.metadata-db-user=${PRESTO_COORDINATOR_CONFIG_CLPPROPERTIES_METADATA_DATABASE_USER}
+clp.metadata-db-password=${PRESTO_COORDINATOR_CONFIG_CLPPROPERTIES_METADATA_DATABASE_PASSWORD}
+clp.metadata-table-prefix=${PRESTO_COORDINATOR_CONFIG_CLPPROPERTIES_METADATA_TABLEPREFIX}
+clp.split-provider-type=${PRESTO_COORDINATOR_CONFIG_CLPPROPERTIES_SPLITPROVIDER}
+clp.metadata-filter-config=/opt/presto-server/etc/metadata-filter.json
+
@@ -0,0 +1,14 @@
+coordinator=true
+node-scheduler.include-coordinator=false
+http-server.http.port=${PRESTO_COORDINATOR_HTTPPORT}
+query.max-memory=${PRESTO_COORDINATOR_CONFIG_CONFIGPROPERTIES_QUERY_MAXMEMORY}
+query.max-memory-per-node=${PRESTO_COORDINATOR_CONFIG_CONFIGPROPERTIES_QUERY_MAXMEMORYPERNODE}
+discovery-server.enabled=true
+discovery.uri=http://${PRESTO_COORDINATOR_SERVICENAME}:${PRESTO_COORDINATOR_HTTPPORT}
+optimizer.optimize-hash-generation=false
+regex-library=RE2J
+use-alternative-function-signatures=true
+inline-sql-functions=false
+nested-data-serialization-enabled=false
+native-execution-enabled=true
+
@@ -0,0 +1,10 @@
+-server
+-Xmx${PRESTO_COORDINATOR_CONFIG_JVMCONFIG_MAXHEAPSIZE}
+-XX:+UseG1GC
+-XX:G1HeapRegionSize=${PRESTO_COORDINATOR_CONFIG_JVMCONFIG_G1HEAPREGIONSIZE}
+-XX:+UseGCOverheadLimit
+-XX:+ExplicitGCInvokesConcurrent
+-XX:+HeapDumpOnOutOfMemoryError
+-XX:+ExitOnOutOfMemoryError
+-Djdk.attach.allowAttachSelf=true
+
@@ -0,0 +1,2 @@
+com.facebook.presto=${PRESTO_COORDINATOR_CONFIG_LOGPROPERTIES_LEVEL}
-com.facebook.presto=${PRESTO_COORDINATOR_CONFIG_LOGPROPERTIES_LEVEL}
+com.facebook.presto=${PRESTO_COORDINATOR_CONFIG_LOGPROPERTIES_LEVEL:-INFO}
-com.facebook.presto=${PRESTO_COORDINATOR_CONFIG_LOGPROPERTIES_LEVEL}
+com.facebook.presto=${PRESTO_COORDINATOR_CONFIG_LOGPROPERTIES_LEVEL:-INFO}
+
@@ -0,0 +1,3 @@
+{
+}
+
@@ -0,0 +1,3 @@
+node.environment=${PRESTO_COORDINATOR_CONFIG_NODEPROPERTIES_ENVIRONMENT}
+node.id=${PRESTO_COORDINATOR_SERVICENAME}
+
-node.environment=${PRESTO_COORDINATOR_CONFIG_NODEPROPERTIES_ENVIRONMENT}
-node.id=${PRESTO_COORDINATOR_SERVICENAME}
+node.environment=${PRESTO_COORDINATOR_CONFIG_NODEPROPERTIES_ENVIRONMENT}
+# A unique ID per container – fall back to HOSTNAME if not supplied
+node.id=${PRESTO_COORDINATOR_NODE_ID:-${HOSTNAME}}
+# Mandatory data directory
+node.data-dir=/opt/presto-server/data
-node.environment=${PRESTO_COORDINATOR_CONFIG_NODEPROPERTIES_ENVIRONMENT}
-node.id=${PRESTO_COORDINATOR_SERVICENAME}
+node.environment=${PRESTO_COORDINATOR_CONFIG_NODEPROPERTIES_ENVIRONMENT}
+# A unique ID per container – fall back to HOSTNAME if not supplied
+node.id=${PRESTO_COORDINATOR_NODE_ID:-${HOSTNAME}}
+# Mandatory data directory
+node.data-dir=/opt/presto-server/data
diff --git a/tools/deployment/presto-clp/coordinator/scripts/$ b/tools/deployment/presto-clp/coordinator/scripts/$
@@ -0,0 +1,19 @@
+#!/bin/sh
+
+# Exit on error
+set -e
+
+PRESTO_CONFIG_DIR="/opt/presto-server/etc"
+
+# Substitute environemnt variables in config template
+find /configs -type f | while read -r f; do
+  ( echo "cat <<EOF"; cat $f; echo "EOF" ) | sh > "${PRESTO_CONFIG_DIR}/$(basename "$f")"
+done
+
+# Setup the config directory hierarchy
+rm -rf ${PRESTO_CONFIG_DIR}/catalog
+mkdir -p ${PRESTO_CONFIG_DIR}/catalog
+
+# Copy over files
+mv ${PRESTO_CONFIG_DIR}/clp.properties ${PRESTO_CONFIG_DIR}/catalog
-mv ${PRESTO_CONFIG_DIR}/clp.properties ${PRESTO_CONFIG_DIR}/catalog
+mv "${PRESTO_CONFIG_DIR}/clp.properties" "${PRESTO_CONFIG_DIR}/catalog"
-mv ${PRESTO_CONFIG_DIR}/clp.properties ${PRESTO_CONFIG_DIR}/catalog
+mv "${PRESTO_CONFIG_DIR}/clp.properties" "${PRESTO_CONFIG_DIR}/catalog"
+
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+# Exit on error
+set -e
+
+PRESTO_CONFIG_DIR="/opt/presto-server/etc"
+
+# Substitute environemnt variables in config template
+find /configs -type f | while read -r f; do
+  ( echo "cat <<EOF"; cat $f; echo "EOF" ) | sh > "${PRESTO_CONFIG_DIR}/$(basename "$f")"
+done
-# Substitute environemnt variables in config template
-find /configs -type f | while read -r f; do
-  ( echo "cat <<EOF"; cat $f; echo "EOF" ) | sh > "${PRESTO_CONFIG_DIR}/$(basename "$f")"
-done
+# Substitute environment variables in config template
+find /configs -type f | while read -r f; do
+  envsubst < "$f" > "${PRESTO_CONFIG_DIR}/$(basename "$f")"
+done
-# Substitute environemnt variables in config template
-find /configs -type f | while read -r f; do
-  ( echo "cat <<EOF"; cat $f; echo "EOF" ) | sh > "${PRESTO_CONFIG_DIR}/$(basename "$f")"
-done
+# Substitute environment variables in config template
+find /configs -type f | while read -r f; do
+  envsubst < "$f" > "${PRESTO_CONFIG_DIR}/$(basename "$f")"
+done
+
+# Setup the config directory hierarchy
+rm -f ${PRESTO_CONFIG_DIR}/catalog/*
+
+# Copy over files
+mv ${PRESTO_CONFIG_DIR}/clp.properties ${PRESTO_CONFIG_DIR}/catalog
+
diff --git a/tools/deployment/presto-clp/demo-assets/clp-config.yml b/tools/deployment/presto-clp/demo-assets/clp-config.yml
@@ -0,0 +1,39 @@
+package:
+  storage_engine: "clp-s"
+database:
+  type: "mariadb"
+  host: "${REPLACE_IP}"
+  port: 6001
+  name: "clp-db"
+query_scheduler:
+  host: "${REPLACE_IP}"
+  port: 6002
+  jobs_poll_delay: 0.1
+  num_archives_to_search_per_sub_job: 16
+  logging_level: "INFO"
+queue:
+  host: "${REPLACE_IP}"
+  port: 6003
+redis:
+  host: "${REPLACE_IP}"
+  port: 6004
+  query_backend_database: 0
+  compression_backend_database: 1
+reducer:
+  host: "${REPLACE_IP}"
+  base_port: 6100
+  logging_level: "INFO"
+  upsert_interval: 100
+results_cache:
+  host: "${REPLACE_IP}"
+  port: 6005
+  db_name: "clp-query-results"
+  stream_collection_name: "stream-files"
+webui:
+  host: "localhost"
+  port: 6000
+  logging_level: "INFO"
+log_viewer_webui:
+  host: "localhost"
+  port: 6006
+
diff --git a/tools/deployment/presto-clp/demo-assets/init.sh b/tools/deployment/presto-clp/demo-assets/init.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+SCRIPT_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+function generate_sample_clp_config {
+    local ip=$(hostname -i)
+    local file="${SCRIPT_PATH}/clp-config.yml"
+    cp "$file" "${file}.bak"
+    sed -i "s|\${REPLACE_IP}|$ip|g" "$file"
+    echo "Replaced \${REPLACE_IP} with $ip in $file"
+}
+
+function update_metadata_config {
+    if [[ $# -ne 1 ]]; then
+        echo "Usage: update_metadata_config </path/to/clp-package>"
+        return 1
+    fi
+
+    local clp_pkg_home=$1
+    local clp_config_path="$(readlink -f ${clp_pkg_home})/etc/clp-config.yml"
+    local credential_path="$(readlink -f ${clp_pkg_home})/etc/credentials.yml"
+    host=$(python3 -c 'import sys, yaml; print(yaml.load(sys.stdin)["database"]["host"])' < "$clp_config_path")
+    port=$(python3 -c 'import sys, yaml; print(yaml.load(sys.stdin)["database"]["port"])' < "$clp_config_path")
+    name=$(python3 -c 'import sys, yaml; print(yaml.load(sys.stdin)["database"]["name"])' < "$clp_config_path")
+    user=$(python3 -c 'import sys, yaml; print(yaml.load(sys.stdin)["database"]["user"])' < "$credential_path")
+    password=$(python3 -c 'import sys, yaml; print(yaml.load(sys.stdin)["database"]["password"])' < "$credential_path")
+    echo "Metadata database host: $host"
+    echo "Metadata database port: $port"
+    echo "Metadata database name: $name"
+    echo "Metadata database user: $user"
+    echo "Metadata database password: $password"
+
+    local env_path="${SCRIPT_PATH}/../.env"
+    cp "$env_path" "${env_path}.bak"
+    sed -i "s|^PRESTO_COORDINATOR_CONFIG_CLPPROPERTIES_METADATA_DATABASE_URL=.*|PRESTO_COORDINATOR_CONFIG_CLPPROPERTIES_METADATA_DATABASE_URL=\"jdbc:mysql://${host}:${port}\"|" "$env_path"
+    sed -i "s/^PRESTO_COORDINATOR_CONFIG_CLPPROPERTIES_METADATA_DATABASE_NAME=.*/PRESTO_COORDINATOR_CONFIG_CLPPROPERTIES_METADATA_DATABASE_NAME=\"${name}\"/" "$env_path"
+    sed -i "s/^PRESTO_COORDINATOR_CONFIG_CLPPROPERTIES_METADATA_DATABASE_USER=.*/PRESTO_COORDINATOR_CONFIG_CLPPROPERTIES_METADATA_DATABASE_USER=\"${user}\"/" "$env_path"
+    sed -i "s/^PRESTO_COORDINATOR_CONFIG_CLPPROPERTIES_METADATA_DATABASE_PASSWORD=.*/PRESTO_COORDINATOR_CONFIG_CLPPROPERTIES_METADATA_DATABASE_PASSWORD=\"${password}\"/" "$env_path"
+    sed -i "s|^CLP_PACKAGE_ARCHIVES=.*|CLP_PACKAGE_ARCHIVES=\"$(readlink -f ${clp_pkg_home})/var/data/archives/default\"|" "$env_path"
+}
+
+if declare -f "$1" > /dev/null; then
+    "$@"
+else
+    echo "Error: '$1' is not a valid function name."
+    echo "Available functions:"
+    declare -F | awk '{print $3}'
+    exit 1
+fi
+
@@ -0,0 +1,39 @@
+version: "3.9"
+
+services:
+  presto-coordinator:
+    image: ghcr.io/y-scope/presto/coordinator:dev
+    entrypoint: ["/bin/bash", "-c", "/scripts/generate-configs.sh && /opt/entrypoint.sh"]
+    env_file:
+      - .env
+    volumes:
+      - ./coordinator/scripts:/scripts:ro
+      - coordinator-config:/opt/presto-server/etc
+      - ./coordinator/config-template:/configs:ro
+    networks:
+      - presto
+
+  presto-worker:
+    image: ghcr.io/y-scope/presto/prestissimo-worker:dev
+    depends_on:
+      presto-coordinator:
+        condition: service_started
+    entrypoint: ["/bin/bash", "-c", "/scripts/generate-configs.sh && /opt/entrypoint.sh"]
+    env_file:
+      - .env
+    volumes:
+      - ./worker/scripts:/scripts:ro
+      - worker-config:/opt/presto-server/etc
+      - ./worker/config-template:/configs:ro
+      - "${CLP_PACKAGE_ARCHIVES}:${CLP_PACKAGE_ARCHIVES}"
+    networks:
+      - presto
+
+volumes:
+  coordinator-config:
+  worker-config:
+
+networks:
+  presto:
+    driver: bridge
+
@@ -0,0 +1,2 @@
+connector.name=clp
+
@@ -0,0 +1,7 @@
+discovery.uri=http://${PRESTO_COORDINATOR_SERVICENAME}:${PRESTO_COORDINATOR_HTTPPORT}
+presto.version=REPLACE_ME
-presto.version=REPLACE_ME
+presto.version=${PRESTO_VERSION}
-presto.version=REPLACE_ME
+presto.version=${PRESTO_VERSION}
+http-server.http.port=${PRESTO_WORKER_HTTPPORT}
+shutdown-onset-sec=1
+register-test-functions=false
+runtime-metrics-collection-enabled=false
+
@@ -0,0 +1,5 @@
+node.environment=${PRESTO_COORDINATOR_CONFIG_NODEPROPERTIES_ENVIRONMENT}
+node.internal-address=REPLACE_ME
+node.location=${PRESTO_WORKER_CONFIG_NODEPROPERTIES_LOCATION}
+node.id=REPLACE_ME
-node.environment=${PRESTO_COORDINATOR_CONFIG_NODEPROPERTIES_ENVIRONMENT}
-node.internal-address=REPLACE_ME
-node.location=${PRESTO_WORKER_CONFIG_NODEPROPERTIES_LOCATION}
-node.id=REPLACE_ME
+node.environment=${PRESTO_WORKER_CONFIG_NODEPROPERTIES_ENVIRONMENT}
+node.internal-address=REPLACE_ME
+node.location=${PRESTO_WORKER_CONFIG_NODEPROPERTIES_LOCATION}
+node.id=REPLACE_ME
-node.environment=${PRESTO_COORDINATOR_CONFIG_NODEPROPERTIES_ENVIRONMENT}
-node.internal-address=REPLACE_ME
-node.location=${PRESTO_WORKER_CONFIG_NODEPROPERTIES_LOCATION}
-node.id=REPLACE_ME
+node.environment=${PRESTO_WORKER_CONFIG_NODEPROPERTIES_ENVIRONMENT}
+node.internal-address=REPLACE_ME
+node.location=${PRESTO_WORKER_CONFIG_NODEPROPERTIES_LOCATION}
+node.id=REPLACE_ME
+
@@ -0,0 +1,2 @@
+mutable-config=true
+
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		{
		}
Comment on lines +1 to +2 Copy link Contributor coderabbitai bot Jul 27, 2025 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others. Learn more. 🧹 Nitpick (assertive) Provide a minimal example or document expected schema An empty JSON object is syntactically valid, but future maintainers may be unsure what keys are supported. A commented exemplar or pointer to docs beside this file would improve clarity without affecting runtime. 🤖 Prompt for AI Agents `In tools/deployment/presto-clp/coordinator/config-template/metadata-filter.json at lines 1 to 2, the JSON file is currently empty, which may confuse future maintainers about the expected keys. Add a minimal example JSON object with typical keys and values or include comments or a reference to documentation explaining the expected schema to improve clarity without impacting runtime behavior.` Comment on lines +1 to +2 Copy link Member kirkrodrigues Jul 28, 2025 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others. Learn more. @anlowee I think we should explain to the user how to configure this for the timestamp field in their logs, right? And also that it may need to be different for each dataset they compress. Copy link Contributor Author anlowee Jul 28, 2025 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others. Learn more. Can we direct them to the related presto-doc section? Copy link Member kirkrodrigues Jul 28, 2025 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others. Learn more. That's not published and is more general than they need, right? We should write a simplified section for them here.