docs: Add k8s setup instructions.

yscope-k8s/.gitignore

@@ -0,0 +1 @@
+demo-assets/clp-config.yml

yscope-k8s/.helmignore

@@ -0,0 +1,27 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
+
+# Demo assets
+demo-assets/

yscope-k8s/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: presto-velox
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.16.0"

yscope-k8s/README.md

@@ -0,0 +1,104 @@
+# Setup local K8s cluster for presto + clp
+
+## Install docker
+
+Follow the guide here: [docker]
+
+## Install kubectl
+
+`kubectl` is the command-line tool for interacting with Kubernetes clusters. You will use it to
+manage and inspect your k3d cluster.
+
+Follow the guide here: [kubectl]
+
+## Install k3d
+
+k3d is a lightweight wrapper to run k3s (Rancher Lab's minimal Kubernetes distribution) in docker.
+
+Follow the guide here: [k3d]
+
+## Install Helm
+
+Helm is the package manager for Kubernetes.
+
+Follow the guide here: [helm]
+
+# Launch clp-package
+1. Find the clp-package for test on our official website [clp-json-v0.4.0]. We also put the dataset for demo here: `mongod-256MB-presto-clp.log.tar.gz`.
+
+2. Untar it.
+
+3. Replace the content of `/path/to/clp-json-package/etc/clp-config.yml` with the output of `demo-assets/init.sh <ip_addr>` where the `<ip_addr>` is the IP address of the host that you are running the clp-package.
+
+4. Launch:
+```bash
+# You probably want to run in a 3.11 python environment
+sbin/start-clp.sh
+```
+
+5. Compress:
+```bash
+# You can also use your own dataset
+sbin/compress.sh --timestamp-key 't.dollar_sign_date' datasets/mongod-256MB-processed.log
+```
+
+6. Use the following command to update the CLP metadata database so that the worker can find the archives in right place:
+```bash
+# Install mysql-client if necessary
+sudo apt update && sudo apt install -y mysql-client
+# Find the user and password in /path/to/clp-json-package/etc/credential.yml
+mysql -h ${REPLACE_IP} -P 6001 -u ${REPLACE_USER} -p'${REPLACE_PASSWORD}' clp-db -e "UPDATE clp_datasets SET archive_storage_directory = '/var/data/archives/default';"
+```
+
+# Create k8s Cluster
+Create a local k8s cluster with port forwarding
+```bash
+k3d cluster create yscope --servers 1 --agents 1 -v $(readlink -f /path/to/clp-json-package/var/data/archives):/var/data/archives
+```
+
+# Working with helm chart
+## Install
+In `yscope-k8s/templates/presto/presto-coordinator-config.yaml` replace the `${REPLACE_IP}` in `clp.metadata-db-url=jdbc:mysql://${REPLACE_IP}:6001` with the IP address of the host you are running the clp-package (basially match the IP address that you configured in the `etc/clp-config.yml` of the clp-package).
+
+```bash
+cd yscope-k8s
+
+helm template . 
+
+helm install demo .
+```
+
+## Use cli:
+After all containers are in "Running" states (check by `kubectl get pods`):
+```bash
+kubectl port-forward service/presto-coordinator 8080:8080
+```
+
+Then you can further forward the 8080 port to your local laptop, to access the Presto's WebUI by e.g., http://localhost:8080
+
+To use presto-cli:
+```bash
+./presto-cli-0.293-executable.jar --catalog clp --schema default --server localhost:8080
+```
+
+Example query:
+```
+SELECT * FROM default LIMIT 1;
+```
+
+## Uninstall
+```bash
+helm uninstall demo
+```
+
+# Delete k8s Cluster
+```bash
+k3d cluster delete yscope
+```
+
+
+[clp-json-v0.4.0]: https://github.com/y-scope/clp/releases/tag/v0.4.0
+[docker]: https://docs.docker.com/engine/install
+[k3d]: https://k3d.io/stable/#installation
+[kubectl]: https://kubernetes.io/docs/tasks/tools/#kubectl
+[helm]: https://helm.sh/docs/intro/install/

yscope-k8s/demo-assets/clp-config.yml.bak

@@ -0,0 +1,38 @@
+package:
+  storage_engine: "clp-s"
+database:
+  type: "mariadb"
+  host: "${REPLACE_IP}"
+  port: 6001
+  name: "clp-db"
+query_scheduler:
+  host: "${REPLACE_IP}"
+  port: 6002
+  jobs_poll_delay: 0.1
+  num_archives_to_search_per_sub_job: 16
+  logging_level: "INFO"
+queue:
+  host: "${REPLACE_IP}"
+  port: 6003
+redis:
+  host: "${REPLACE_IP}"
+  port: 6004
+  query_backend_database: 0
+  compression_backend_database: 1
+reducer:
+  host: "${REPLACE_IP}"
+  base_port: 6100
+  logging_level: "INFO"
+  upsert_interval: 100
+results_cache:
+  host: "${REPLACE_IP}"
+  port: 6005
+  db_name: "clp-query-results"
+  stream_collection_name: "stream-files"
+webui:
+  host: "localhost"
+  port: 6000
+  logging_level: "INFO"
+log_viewer_webui:
+  host: "localhost"
+  port: 6006

yscope-k8s/demo-assets/init.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+if [ "$#" -ne 1 ]; then
+    echo "Usage: $0 <ip-address>"
+    exit 1
+fi
+
+SCRIPT_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+IP="$1"
+FILE="${SCRIPT_PATH}/clp-config.yml"
+
+cp "${FILE}.bak" "$FILE"
+
+sed -i "s|\${REPLACE_IP}|$IP|g" "$FILE"
+
+echo "Replaced \${REPLACE_IP} with $IP in $FILE"

yscope-k8s/templates/object-store/aws-credentials.yaml

@@ -0,0 +1,8 @@
+apiVersion: "v1"
+kind: "Secret"
+metadata:
+  name: "aws-credentials"
+  namespace: "default"
+type: "Opaque"
+data:
+  credentials: "W2RlZmF1bHRdCmF3c19hY2Nlc3Nfa2V5X2lkID0gbWluaW9hZG1pbgphd3Nfc2VjcmV0X2FjY2Vzc19rZXkgPSBtaW5pb2FkbWluCg=="

yscope-k8s/templates/object-store/bucket-creation.yaml

@@ -0,0 +1,157 @@
+kind: "Job"
+metadata:
+  name: "bucket-creation"
+spec:
+  template:
+    spec:
+      containers:
+        # Container to deploy the log viewer. To inspect logs, use the following command:
+        # `kubectl logs job.batch/bucket-creation`
+        - name: "bucket-creation"
+          image: "amazon/aws-cli:latest"
+          command:
+            - "/bin/bash"
+          args:
+            - "/scripts/bucket-creation.sh"
+          env:
+            - name: "AWS_ENDPOINT_URL"
+              value: "http://{{ .Values.objectStore.minio.serviceName }}.default.svc.cluster.local:{{ .Values.objectStore.minio.apiPort }}"
+            - name: "BUCKET_NAME"
+              value: "{{ .Values.objectStore.bucketCreation.bucketName }}"
+            - name: "PUBLIC"
+              value: "{{ .Values.objectStore.bucketCreation.public }}"
+          volumeMounts:
+            - name: "aws-credentials-volume"
+              mountPath: "/root/.aws"
+            - name:  "scripts-volume"
+              mountPath: "/scripts"
+          imagePullPolicy: "IfNotPresent"
+      restartPolicy: "Never"
+      volumes:
+        - name: "aws-credentials-volume"
+          secret:
+            secretName: "aws-credentials"
+        - name: "scripts-volume"
+          configMap:
+            name: "bucket-creation"
+---
+apiVersion: v1
+kind: "ConfigMap"
+metadata:
+  name: "bucket-creation"
+data:
+  bucket-creation.sh: |-
+    #!/usr/bin/env bash
+
+    # Create a bucket and optionally it configure with public read access
+    # on a S3-compatible object store such as MinIO
+    #
+    # Requirements:
+    #
+    # * AWS CLI authentication configured using any supported method---for example:
+    #   * A credentials file in $HOME/.aws/credentials
+    #   * AWS_CONFIG_FILE pointing to a custom credentials file
+    #   * Environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
+    # * Environment variables:
+    #   * AWS_ENDPOINT_URL: The S3-compatible object store endpoint URL
+    #   * BUCKET_NAME: The name of the bucket where the log viewer should be deployed
+    #     * NOTE: This script will make the bucket publicly readable.
+    #   * PUBLIC (Optional): If set to "true", configures bucket with public read policy
+    set -e
+    set -o pipefail
+    set -u
+
+    # Emits a log event to stderr with an auto-generated ISO timestamp as well as the given level
+    # and message.
+    #
+    # @param $1: Level string
+    # @param $2: Message to be logged
+    log() {
+        local -r LEVEL=$1
+        local -r MESSAGE=$2
+        echo "$(date --utc --date="now" +"%Y-%m-%dT%H:%M:%SZ") [${LEVEL}] ${MESSAGE}" >&2
+    }
+
+    # Waits for the S3 endpoint to be available, or exits if it's unavailable.
+    wait_for_s3_availability() {
+        # Check availability by listing available buckets
+        log "INFO" "Waiting until ${AWS_ENDPOINT_URL} endpoint becomes available."
+        local -r MAX_RETRIES=10
+        local -r RETRY_DELAY_IN_SECS=6
+        for ((retries = 0; retries < MAX_RETRIES; retries++)); do
+            if aws s3 ls --endpoint-url "$AWS_ENDPOINT_URL" >/dev/null; then
+                return
+            fi
+            log "WARN" "S3 API endpoint unavailable. Retrying in ${RETRY_DELAY_IN_SECS} seconds."
+
+            sleep "$RETRY_DELAY_IN_SECS"
+        done
+
+        if [[ $retries -eq $MAX_RETRIES ]]; then
+            log "ERROR" "Maximum retries reached. S3 API endpoint ${AWS_ENDPOINT_URL} didn't respond."
+            exit 1
+        fi
+    }
+
+    # Creates a bucket
+    create_bucket() {
+        # Create log-viewer bucket if it doesn't already exist
+        log "INFO" "Creating ${BUCKET_S3_URI} bucket."
+        if ! aws s3api head-bucket --endpoint-url "$AWS_ENDPOINT_URL" --bucket "$BUCKET_NAME" \
+            2>/dev/null; then
+            aws s3api create-bucket --endpoint-url "$AWS_ENDPOINT_URL" --bucket "$BUCKET_NAME"
+        fi
+    }
+
+    # Configures a bucket with public read access
+    configure_bucket() {
+        # Define and apply the bucket policy for public read access
+        log "INFO" "Applying public read access policy to ${BUCKET_S3_URI}"
+        local -r POLICY=$(
+            cat <<EOP
+    {
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+                "Effect": "Allow",
+                "Principal": "*",
+                "Action": "s3:GetObject",
+                "Resource": "arn:aws:s3:::${BUCKET_NAME}/*"
+            }
+        ]
+    }
+    EOP
+        )
+        if ! aws s3api put-bucket-policy \
+            --endpoint-url "$AWS_ENDPOINT_URL" \
+            --bucket "$BUCKET_NAME" \
+            --policy "$POLICY"; then
+            log "ERROR" "Failed to set bucket policy for ${BUCKET_S3_URI}"
+            exit 1
+        fi
+    }
+
+    # Validate required environment variables
+    readonly REQUIRED_ENV_VARS=(
+        # Example: "http://minio:9000"
+        "AWS_ENDPOINT_URL"
+
+        # Example: "logs"
+        "BUCKET_NAME"
+    )
+    for var in "${REQUIRED_ENV_VARS[@]}"; do
+        if ! [[ -v "$var" ]]; then
+            log "ERROR" "$var environment variable must be set."
+            exit 1
+        fi
+    done
+
+    readonly BUCKET_S3_URI="s3://${BUCKET_NAME}"
+
+    wait_for_s3_availability
+    create_bucket
+    if [[ "${PUBLIC:-false}" = "true" ]]; then
+        configure_bucket
+    fi
+
+    log "INFO" "Bucket ${BUCKET_NAME} created and configured successfully."