diff --git a/Dockerfile b/Dockerfile index 3b4f8075..3a174227 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,9 @@ -FROM ubuntu:jammy +FROM x11docker/xfce # as rust-base # NB: This can be "stable" or another version. ARG RUST_TOOLCHAIN="1.75.0" -WORKDIR /tmp - # NB: cmake is required for freetype-sys-0.13.1, which in turn has only been added for egui. # NB: fontconfig is required by servo-fontconfig-sys, which is in the dependency chain for egui. # NB: libfontconfig-dev is required by servo-fontconfig-sys, which is in the dependency chain for egui. @@ -17,13 +15,34 @@ RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y \ build-essential \ cmake \ - libasound2-dev \ - libudev-dev \ + mesa-utils \ curl \ fontconfig \ libfontconfig1-dev \ libssl-dev \ - pkg-config + pkg-config \ + g++ \ + pkg-config \ + libx11-dev \ + libasound2-dev \ + libudev-dev \ + libxkbcommon-x11-0 \ + libwayland-dev \ + libxkbcommon-dev \ + mesa-vulkan-drivers \ + xterm + +ARG USERNAME=ubuntu +ARG USER_UID=1000 +ARG USER_GID=$USER_UID + +RUN groupadd --gid $USER_GID $USERNAME \ + && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME \ + # + && apt-get update \ + && apt-get install -y sudo \ + && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \ + && chmod 0440 /etc/sudoers.d/$USERNAME # Install Rust RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ @@ -34,19 +53,29 @@ RUN $HOME/.cargo/bin/rustup show RUN $HOME/.cargo/bin/rustc --version RUN $HOME/.cargo/bin/cargo --version -# Install NPM -# https://nodejs.org/en -# https://github.com/nodesource/distributions -#RUN curl -fsSL https://deb.nodesource.com/setup_21.x | bash - && \ -# apt-get install -y nodejs +WORKDIR /src -# NB: Now we build and test our code. COPY Cargo.lock . COPY Cargo.toml . -COPY ./ . +COPY . . + +ARG WORKDIR=/storyteller +ENV CARGO_TARGET_DIR=$WORKDIR RUN LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \ $HOME/.cargo/bin/cargo build \ --release \ --bin studio-headless + +RUN cp runner.sh $WORKDIR + +WORKDIR $WORKDIR + +RUN mv release/studio-headless $WORKDIR/studio-headless + +COPY --from=inference-job-local inference-job $WORKDIR/inference-job + +RUN chown -R $USERNAME $WORKDIR + +USER $USERNAME diff --git a/deploy/install.sh b/deploy/install.sh new file mode 100644 index 00000000..b165523e --- /dev/null +++ b/deploy/install.sh @@ -0,0 +1,4 @@ +sudo cp x11docker-inferencejob.service /etc/systemd/system +sudo systemctl daemon-reload +sudo systemctl enable x11docker-inferencejob.service +sudo systemctl start x11docker-inferencejob.service diff --git a/deploy/pull_secrets.py b/deploy/pull_secrets.py new file mode 100644 index 00000000..dfb1c870 --- /dev/null +++ b/deploy/pull_secrets.py @@ -0,0 +1,24 @@ +import boto3 +from botocore.exceptions import ClientError +import json + +secret_name = "bvh_to_workflow_job" +region_name = "us-east-1" + +session = boto3.session.Session() +client = session.client( + service_name='secretsmanager', + region_name=region_name +) + +secret_response = client.get_secret_value( + SecretId=secret_name +) + +secret_string = secret_response["SecretString"] +secrets = json.loads(secret_string) + +with open("secrets-rendered.env", "w") as file: + for k, v in secrets.items(): + file.write(f"{k}={v}\n") + diff --git a/deploy/vars.env b/deploy/vars.env new file mode 100644 index 00000000..730b335b --- /dev/null +++ b/deploy/vars.env @@ -0,0 +1,16 @@ +PAUSE_FILE='/tmp/pause.txt' +ALWAYS_ALLOW_COLD_FILESYSTEM_CACHE='true' +SCOPED_EXECUTION_MODEL_TYPES='bvh_to_workflow' +SCOPED_TEMP_DIR_LONG_LIVED_DOWNLOADS='/tmp/downloads_long' +SCOPED_TEMP_DIR_SHORT_LIVED_DOWNLOADS='/tmp/downloads' +SCOPED_TEMP_DIR_WORK='/tmp/work' +BVH2WORKFLOW_TIMEOUT_SECONDS='300' +REGION_NAME='us-east1' +SEMI_PERSISTENT_DIR_ROOT='/tmp/shared_volume/cache' +PUBLIC_BUCKET_NAME=vocodes-public +PRIVATE_BUCKET_NAME=vocodes-private-uploads +BVH2WORKFLOW_DIRECTORY='/storyteller' +BVH2WORKFLOW_EXECUTABLE_OR_COMMAND='/storyteller/runner.sh' +STUDIO_HEADLESS_PATH='/storyteller/studio-headless' +REDIS_FOR_JOB_PROGRESS="" +REDIS_FOR_KEEPALIVE_URL="" diff --git a/deploy/x11docker-inferencejob.service b/deploy/x11docker-inferencejob.service new file mode 100644 index 00000000..568b9748 --- /dev/null +++ b/deploy/x11docker-inferencejob.service @@ -0,0 +1,10 @@ +# https://github.com/mviereck/x11docker/issues/154 +[Unit] +Description=run bevy with x11docker + +[Service] +# TODO: pre job pull down secrets, pull env vars also? +ExecStart=x11docker -D --xvfb --network --gpu=no --printenv --fallback=no --home=~/shared-with-x-serv --size=720x480 -- -e NVIDIA_DRIVER_CAPABILITIES=all --runtime=nvidia --gpus all --env-file /home/ubuntu/storyteller/storyteller-bevy/deploy/secrets.env -- docker.io/library/xfce-bevy-ubuntu-local /storyteller/inference-job + +[Install] +WantedBy=multi-user.target diff --git a/runner.sh b/runner.sh new file mode 100755 index 00000000..013cc954 --- /dev/null +++ b/runner.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash + + +set -ex + +# This script is used to run studio-headless + +#parse arguments from command line +# usage: runner.sh -i input_file -v output.mp4 -o output_dir.zip -f 1000 -b path/to/studio-headless + +while getopts "i:o:f:c:s:b:" opt; do + case $opt in + i) INPUT_FILE=$OPTARG ;; + o) OUTPUT_DIR=$OPTARG ;; + f) FRAME_COUNT=$OPTARG ;; + c) STUDIO_CAMERA_SETTING=$OPTARG ;; + s) STUDIO_CAMERA_SPEED=$OPTARG ;; + b) SKYBOX=$OPTARG ;; + \?) echo "Invalid option -$OPTARG" >&2 + ;; + esac +done + +# error out if input file is not provided +if [ -z "$INPUT_FILE" ]; then + echo "Input file is required" + exit 1 +fi + +# error out if output directory is not provided +if [ -z "$OUTPUT_DIR" ]; then + echo "Output directory is required" + exit 1 +fi + +#source secrets.env +source /home/ubuntu/secrets.env + +OUTPUT_DIR=${OUTPUT_DIR:-/tmp/studio-headless} +FPS=${FPS:-24} +RESOLUTION=${RESOLUTION:-1920x1080} +FRAME_COUNT=${FRAME_COUNT:-100} +STUDIO_HEADLESS_PATH=${STUDIO_HEADLESS_PATH:-/home/ubuntu/storyteller-bevy/target/release/studio-headless} +MAX_FRAME_COUNT=240 # 10 seconds @ 24 fps +STUDIO_CAMERA_SPEED=${STUDIO_CAMERA_SPEED:-1} +SKYBOX=${SKYBOX:-ff5555} +STUDIO_TIMEOUT_SECONDS=${STUDIO_TIMEOUT_SECONDS:-120} + +mkdir -p target/release/assets/ + +cp $INPUT_FILE target/release/assets/ + +# extract input file name without prefix +INPUT_FILE_NAME=$(basename $INPUT_FILE) + +mkdir -p $OUTPUT_DIR/frames + +# run studio-headless +#$STUDIO_HEADLESS_PATH bvh -o $OUTPUT_DIR/frames --frames $FRAME_COUNT $INPUT_FILE_NAME # NB(bt): commented out +# NB: Output directory might have to end in slash +#RUST_LOG="debug" $STUDIO_HEADLESS_PATH scene \ +#RUST_LOG="warn,bevy_gltf::loader=error" $STUDIO_HEADLESS_PATH scene \ +# NB(bt,2024-03-25): Removing the camera params since we no longer use the manual non-timeline camera + #--camera-motion $STUDIO_CAMERA_SETTING \ + #--camera-motion-speed $STUDIO_CAMERA_SPEED \ +#RUST_BACKTRACE=full RUST_LOG="warn,bevy_gltf::loader=error" $STUDIO_HEADLESS_PATH scene \ +RUST_BACKTRACE=full RUST_LOG="info" timeout --kill-after 30 $STUDIO_TIMEOUT_SECONDS $STUDIO_HEADLESS_PATH scene \ + -o $OUTPUT_DIR/frames/ \ + --frames $MAX_FRAME_COUNT \ + --skybox $SKYBOX \ + $INPUT_FILE_NAME + +# use ffmpeg to create a video from the images + +#ffmpeg -framerate $FPS -pattern_type glob -i "$OUTPUT_DIR/frames/*-img.png" -c:v libx264 -pix_fmt yuv420p $OUTPUT_DIR/../output.mp4 +ffmpeg -framerate $FPS -pattern_type glob -i "$OUTPUT_DIR/frames/*.png" -c:v libx264 -pix_fmt yuv420p $OUTPUT_DIR/../output.mp4 + +# zip the images and the video +zip -r $OUTPUT_DIR/../output.zip $OUTPUT_DIR/frames