From 0e71970bfc8a05c226348270e48778c14840f694 Mon Sep 17 00:00:00 2001 From: Kasisnu <5044120+kasisnu@users.noreply.github.com> Date: Tue, 9 Apr 2024 19:31:41 +0530 Subject: [PATCH 1/2] an updated dockerfile --- Dockerfile | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3b4f8075..5cd962e5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:jammy +FROM x11docker/xfce # as rust-base # NB: This can be "stable" or another version. @@ -17,13 +17,23 @@ RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y \ build-essential \ cmake \ - libasound2-dev \ - libudev-dev \ + mesa-utils \ curl \ fontconfig \ libfontconfig1-dev \ libssl-dev \ - pkg-config + pkg-config \ + g++ \ + pkg-config \ + libx11-dev \ + libasound2-dev \ + libudev-dev \ + libxkbcommon-x11-0 \ + libwayland-dev \ + libxkbcommon-dev \ + mesa-vulkan-drivers \ + xterm + # Install Rust RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ @@ -50,3 +60,11 @@ RUN LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \ $HOME/.cargo/bin/cargo build \ --release \ --bin studio-headless +WORKDIR /workdir + +RUN mv $HOME/.cargo /workdir/.cargo && chmod -R 777 /workdir/.cargo + +COPY ./ . +RUN cp /tmp/target/release/studio-headless /workdir/studio-headless + +# CMD xterm \ No newline at end of file From 44e3fa1d06de7b7d93ced1e943fa728f88a1affa Mon Sep 17 00:00:00 2001 From: Madhukar Mishra Date: Fri, 12 Apr 2024 18:48:00 +0530 Subject: [PATCH 2/2] run x11docker as a systemd service --- Dockerfile | 39 ++++++++----- deploy/install.sh | 4 ++ deploy/pull_secrets.py | 24 ++++++++ deploy/vars.env | 16 ++++++ deploy/x11docker-inferencejob.service | 10 ++++ runner.sh | 79 +++++++++++++++++++++++++++ 6 files changed, 158 insertions(+), 14 deletions(-) create mode 100644 deploy/install.sh create mode 100644 deploy/pull_secrets.py create mode 100644 deploy/vars.env create mode 100644 deploy/x11docker-inferencejob.service create mode 100755 runner.sh diff --git a/Dockerfile b/Dockerfile index 5cd962e5..3a174227 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,8 +4,6 @@ FROM x11docker/xfce # NB: This can be "stable" or another version. ARG RUST_TOOLCHAIN="1.75.0" -WORKDIR /tmp - # NB: cmake is required for freetype-sys-0.13.1, which in turn has only been added for egui. # NB: fontconfig is required by servo-fontconfig-sys, which is in the dependency chain for egui. # NB: libfontconfig-dev is required by servo-fontconfig-sys, which is in the dependency chain for egui. @@ -34,6 +32,17 @@ RUN apt-get update \ mesa-vulkan-drivers \ xterm +ARG USERNAME=ubuntu +ARG USER_UID=1000 +ARG USER_GID=$USER_UID + +RUN groupadd --gid $USER_GID $USERNAME \ + && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME \ + # + && apt-get update \ + && apt-get install -y sudo \ + && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \ + && chmod 0440 /etc/sudoers.d/$USERNAME # Install Rust RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ @@ -44,27 +53,29 @@ RUN $HOME/.cargo/bin/rustup show RUN $HOME/.cargo/bin/rustc --version RUN $HOME/.cargo/bin/cargo --version -# Install NPM -# https://nodejs.org/en -# https://github.com/nodesource/distributions -#RUN curl -fsSL https://deb.nodesource.com/setup_21.x | bash - && \ -# apt-get install -y nodejs +WORKDIR /src -# NB: Now we build and test our code. COPY Cargo.lock . COPY Cargo.toml . -COPY ./ . +COPY . . + +ARG WORKDIR=/storyteller +ENV CARGO_TARGET_DIR=$WORKDIR RUN LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \ $HOME/.cargo/bin/cargo build \ --release \ --bin studio-headless -WORKDIR /workdir -RUN mv $HOME/.cargo /workdir/.cargo && chmod -R 777 /workdir/.cargo +RUN cp runner.sh $WORKDIR + +WORKDIR $WORKDIR + +RUN mv release/studio-headless $WORKDIR/studio-headless + +COPY --from=inference-job-local inference-job $WORKDIR/inference-job -COPY ./ . -RUN cp /tmp/target/release/studio-headless /workdir/studio-headless +RUN chown -R $USERNAME $WORKDIR -# CMD xterm \ No newline at end of file +USER $USERNAME diff --git a/deploy/install.sh b/deploy/install.sh new file mode 100644 index 00000000..b165523e --- /dev/null +++ b/deploy/install.sh @@ -0,0 +1,4 @@ +sudo cp x11docker-inferencejob.service /etc/systemd/system +sudo systemctl daemon-reload +sudo systemctl enable x11docker-inferencejob.service +sudo systemctl start x11docker-inferencejob.service diff --git a/deploy/pull_secrets.py b/deploy/pull_secrets.py new file mode 100644 index 00000000..dfb1c870 --- /dev/null +++ b/deploy/pull_secrets.py @@ -0,0 +1,24 @@ +import boto3 +from botocore.exceptions import ClientError +import json + +secret_name = "bvh_to_workflow_job" +region_name = "us-east-1" + +session = boto3.session.Session() +client = session.client( + service_name='secretsmanager', + region_name=region_name +) + +secret_response = client.get_secret_value( + SecretId=secret_name +) + +secret_string = secret_response["SecretString"] +secrets = json.loads(secret_string) + +with open("secrets-rendered.env", "w") as file: + for k, v in secrets.items(): + file.write(f"{k}={v}\n") + diff --git a/deploy/vars.env b/deploy/vars.env new file mode 100644 index 00000000..730b335b --- /dev/null +++ b/deploy/vars.env @@ -0,0 +1,16 @@ +PAUSE_FILE='/tmp/pause.txt' +ALWAYS_ALLOW_COLD_FILESYSTEM_CACHE='true' +SCOPED_EXECUTION_MODEL_TYPES='bvh_to_workflow' +SCOPED_TEMP_DIR_LONG_LIVED_DOWNLOADS='/tmp/downloads_long' +SCOPED_TEMP_DIR_SHORT_LIVED_DOWNLOADS='/tmp/downloads' +SCOPED_TEMP_DIR_WORK='/tmp/work' +BVH2WORKFLOW_TIMEOUT_SECONDS='300' +REGION_NAME='us-east1' +SEMI_PERSISTENT_DIR_ROOT='/tmp/shared_volume/cache' +PUBLIC_BUCKET_NAME=vocodes-public +PRIVATE_BUCKET_NAME=vocodes-private-uploads +BVH2WORKFLOW_DIRECTORY='/storyteller' +BVH2WORKFLOW_EXECUTABLE_OR_COMMAND='/storyteller/runner.sh' +STUDIO_HEADLESS_PATH='/storyteller/studio-headless' +REDIS_FOR_JOB_PROGRESS="" +REDIS_FOR_KEEPALIVE_URL="" diff --git a/deploy/x11docker-inferencejob.service b/deploy/x11docker-inferencejob.service new file mode 100644 index 00000000..568b9748 --- /dev/null +++ b/deploy/x11docker-inferencejob.service @@ -0,0 +1,10 @@ +# https://github.com/mviereck/x11docker/issues/154 +[Unit] +Description=run bevy with x11docker + +[Service] +# TODO: pre job pull down secrets, pull env vars also? +ExecStart=x11docker -D --xvfb --network --gpu=no --printenv --fallback=no --home=~/shared-with-x-serv --size=720x480 -- -e NVIDIA_DRIVER_CAPABILITIES=all --runtime=nvidia --gpus all --env-file /home/ubuntu/storyteller/storyteller-bevy/deploy/secrets.env -- docker.io/library/xfce-bevy-ubuntu-local /storyteller/inference-job + +[Install] +WantedBy=multi-user.target diff --git a/runner.sh b/runner.sh new file mode 100755 index 00000000..013cc954 --- /dev/null +++ b/runner.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash + + +set -ex + +# This script is used to run studio-headless + +#parse arguments from command line +# usage: runner.sh -i input_file -v output.mp4 -o output_dir.zip -f 1000 -b path/to/studio-headless + +while getopts "i:o:f:c:s:b:" opt; do + case $opt in + i) INPUT_FILE=$OPTARG ;; + o) OUTPUT_DIR=$OPTARG ;; + f) FRAME_COUNT=$OPTARG ;; + c) STUDIO_CAMERA_SETTING=$OPTARG ;; + s) STUDIO_CAMERA_SPEED=$OPTARG ;; + b) SKYBOX=$OPTARG ;; + \?) echo "Invalid option -$OPTARG" >&2 + ;; + esac +done + +# error out if input file is not provided +if [ -z "$INPUT_FILE" ]; then + echo "Input file is required" + exit 1 +fi + +# error out if output directory is not provided +if [ -z "$OUTPUT_DIR" ]; then + echo "Output directory is required" + exit 1 +fi + +#source secrets.env +source /home/ubuntu/secrets.env + +OUTPUT_DIR=${OUTPUT_DIR:-/tmp/studio-headless} +FPS=${FPS:-24} +RESOLUTION=${RESOLUTION:-1920x1080} +FRAME_COUNT=${FRAME_COUNT:-100} +STUDIO_HEADLESS_PATH=${STUDIO_HEADLESS_PATH:-/home/ubuntu/storyteller-bevy/target/release/studio-headless} +MAX_FRAME_COUNT=240 # 10 seconds @ 24 fps +STUDIO_CAMERA_SPEED=${STUDIO_CAMERA_SPEED:-1} +SKYBOX=${SKYBOX:-ff5555} +STUDIO_TIMEOUT_SECONDS=${STUDIO_TIMEOUT_SECONDS:-120} + +mkdir -p target/release/assets/ + +cp $INPUT_FILE target/release/assets/ + +# extract input file name without prefix +INPUT_FILE_NAME=$(basename $INPUT_FILE) + +mkdir -p $OUTPUT_DIR/frames + +# run studio-headless +#$STUDIO_HEADLESS_PATH bvh -o $OUTPUT_DIR/frames --frames $FRAME_COUNT $INPUT_FILE_NAME # NB(bt): commented out +# NB: Output directory might have to end in slash +#RUST_LOG="debug" $STUDIO_HEADLESS_PATH scene \ +#RUST_LOG="warn,bevy_gltf::loader=error" $STUDIO_HEADLESS_PATH scene \ +# NB(bt,2024-03-25): Removing the camera params since we no longer use the manual non-timeline camera + #--camera-motion $STUDIO_CAMERA_SETTING \ + #--camera-motion-speed $STUDIO_CAMERA_SPEED \ +#RUST_BACKTRACE=full RUST_LOG="warn,bevy_gltf::loader=error" $STUDIO_HEADLESS_PATH scene \ +RUST_BACKTRACE=full RUST_LOG="info" timeout --kill-after 30 $STUDIO_TIMEOUT_SECONDS $STUDIO_HEADLESS_PATH scene \ + -o $OUTPUT_DIR/frames/ \ + --frames $MAX_FRAME_COUNT \ + --skybox $SKYBOX \ + $INPUT_FILE_NAME + +# use ffmpeg to create a video from the images + +#ffmpeg -framerate $FPS -pattern_type glob -i "$OUTPUT_DIR/frames/*-img.png" -c:v libx264 -pix_fmt yuv420p $OUTPUT_DIR/../output.mp4 +ffmpeg -framerate $FPS -pattern_type glob -i "$OUTPUT_DIR/frames/*.png" -c:v libx264 -pix_fmt yuv420p $OUTPUT_DIR/../output.mp4 + +# zip the images and the video +zip -r $OUTPUT_DIR/../output.zip $OUTPUT_DIR/frames