Skip to content

chore: Add kokoro automation config and scripts. #13

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Feb 5, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/ISSUE_TEMPLATE/bug_report.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ If you are still having issues, please be sure to include as much information as
- OS type and version:
- Python version: `python --version`
- pip version: `pip --version`
- `toolbox-langchain-sdk` version: `pip show toolbox-langchain-sdk`
- `toolbox-langchain` version: `pip show toolbox-langchain`

#### Steps to reproduce

Expand Down
4 changes: 2 additions & 2 deletions .github/release-please.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,5 +13,5 @@
# limitations under the License.

handleGHRelease: true
packageName: genai-toolbox-langchain-python
releaseType: python
packageName: toolbox-langchain
releaseType: python
2 changes: 1 addition & 1 deletion .github/workflows/lint.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -75,4 +75,4 @@ jobs:
- name: Run type-check
env:
MYPYPATH: './src'
run: mypy --install-types --non-interactive --cache-dir=.mypy_cache/ -p toolbox_langchain_sdk
run: mypy --install-types --non-interactive --cache-dir=.mypy_cache/ -p toolbox_langchain
59 changes: 59 additions & 0 deletions .kokoro/build.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
#!/bin/bash
# Copyright 2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -eo pipefail

if [[ -z "${PROJECT_ROOT:-}" ]]; then
PROJECT_ROOT="github/genai-toolbox-langchain-python"
fi

cd "${PROJECT_ROOT}"

# Disable buffering, so that the logs stream through.
export PYTHONUNBUFFERED=1

# Debug: show build environment
env | grep KOKORO

# Setup service account credentials.
export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json

# Setup project id.
export PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/project-id.json")

# Remove old nox
python3 -m pip uninstall --yes --quiet nox-automation

# Install nox
python3 -m pip install --upgrade --quiet nox
python3 -m nox --version

# If this is a continuous build, send the test log to the FlakyBot.
# See https://github.com/googleapis/repo-automation-bots/tree/main/packages/flakybot.
if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"continuous"* ]]; then
cleanup() {
chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot
$KOKORO_GFILE_DIR/linux_amd64/flakybot
}
trap cleanup EXIT HUP
fi

# If NOX_SESSION is set, it only runs the specified session,
# otherwise run all the sessions.
if [[ -n "${NOX_SESSION:-}" ]]; then
python3 -m nox -s ${NOX_SESSION:-}
else
python3 -m nox
fi
88 changes: 88 additions & 0 deletions .kokoro/docker/docs/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
# Copyright 2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from ubuntu:22.04

ENV DEBIAN_FRONTEND noninteractive

# Ensure local Python is preferred over distribution Python.
ENV PATH /usr/local/bin:$PATH

# Install dependencies.
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
apt-transport-https \
build-essential \
ca-certificates \
curl \
dirmngr \
git \
gpg-agent \
graphviz \
libbz2-dev \
libdb5.3-dev \
libexpat1-dev \
libffi-dev \
liblzma-dev \
libreadline-dev \
libsnappy-dev \
libssl-dev \
libsqlite3-dev \
portaudio19-dev \
python3-distutils \
redis-server \
software-properties-common \
ssh \
sudo \
tcl \
tcl-dev \
tk \
tk-dev \
uuid-dev \
wget \
zlib1g-dev \
&& add-apt-repository universe \
&& apt-get update \
&& apt-get -y install jq \
&& apt-get clean autoclean \
&& apt-get autoremove -y \
&& rm -rf /var/lib/apt/lists/* \
&& rm -f /var/cache/apt/archives/*.deb

ENV VERSION v4.2.0
ENV BINARY yq_linux_amd64
RUN wget https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY} -O /usr/bin/yq && \
chmod +x /usr/bin/yq

###################### Install python 3.9.13

# Download python 3.9.13
RUN wget https://www.python.org/ftp/python/3.9.13/Python-3.9.13.tgz

# Extract files
RUN tar -xvf Python-3.9.13.tgz

# Install python 3.9.13
RUN ./Python-3.9.13/configure --enable-optimizations
RUN make altinstall

###################### Install pip
RUN wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \
&& python3 /tmp/get-pip.py \
&& rm /tmp/get-pip.py

# Test pip
RUN python3 -m pip

CMD ["python3.8"]
1 change: 1 addition & 0 deletions .kokoro/docker/docs/requirements.in
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
nox
38 changes: 38 additions & 0 deletions .kokoro/docker/docs/requirements.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
#
# This file is autogenerated by pip-compile with Python 3.11
# by the following command:
#
# pip-compile --allow-unsafe --generate-hashes requirements.in
#
argcomplete==3.3.0 \
--hash=sha256:c168c3723482c031df3c207d4ba8fa702717ccb9fc0bfe4117166c1f537b4a54 \
--hash=sha256:fd03ff4a5b9e6580569d34b273f741e85cd9e072f3feeeee3eba4891c70eda62
# via nox
colorlog==6.8.2 \
--hash=sha256:3e3e079a41feb5a1b64f978b5ea4f46040a94f11f0e8bbb8261e3dbbeca64d44 \
--hash=sha256:4dcbb62368e2800cb3c5abd348da7e53f6c362dda502ec27c560b2e58a66bd33
# via nox
distlib==0.3.8 \
--hash=sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784 \
--hash=sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64
# via virtualenv
filelock==3.13.4 \
--hash=sha256:404e5e9253aa60ad457cae1be07c0f0ca90a63931200a47d9b6a6af84fd7b45f \
--hash=sha256:d13f466618bfde72bd2c18255e269f72542c6e70e7bac83a0232d6b1cc5c8cf4
# via virtualenv
nox==2024.4.15 \
--hash=sha256:6492236efa15a460ecb98e7b67562a28b70da006ab0be164e8821177577c0565 \
--hash=sha256:ecf6700199cdfa9e5ea0a41ff5e6ef4641d09508eda6edb89d9987864115817f
# via -r requirements.in
packaging==24.0 \
--hash=sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5 \
--hash=sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9
# via nox
platformdirs==4.2.1 \
--hash=sha256:031cd18d4ec63ec53e82dceaac0417d218a6863f7745dfcc9efe7793b7039bdf \
--hash=sha256:17d5a1161b3fd67b390023cb2d3b026bbd40abde6fdb052dfbd3a29c3ba22ee1
# via virtualenv
virtualenv==20.26.0 \
--hash=sha256:0846377ea76e818daaa3e00a4365c018bc3ac9760cbb3544de542885aad61fb3 \
--hash=sha256:ec25a9671a5102c8d2657f62792a27b48f016664c6873f6beed3800008577210
# via nox
67 changes: 67 additions & 0 deletions .kokoro/docs/common.cfg
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
# Format: //devtools/kokoro/config/proto/build.proto

# Build logs will be here
action {
define_artifacts {
regex: "**/*sponge_log.xml"
}
}

# Download trampoline resources.
gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"

# Use the trampoline script to run in docker.
build_file: "genai-toolbox-langchain-python/.kokoro/trampoline_v2.sh"

# Configure the docker image for kokoro-trampoline.
env_vars: {
key: "TRAMPOLINE_IMAGE"
value: "gcr.io/cloud-devrel-kokoro-resources/python-lib-docs"
}
env_vars: {
key: "TRAMPOLINE_BUILD_FILE"
value: "github/genai-toolbox-langchain-python/.kokoro/publish-docs.sh"
}

env_vars: {
key: "STAGING_BUCKET"
value: "docs-staging"
}

env_vars: {
key: "V2_STAGING_BUCKET"
# Push non-cloud library docs to `docs-staging-v2-staging` instead of the
# Cloud RAD bucket `docs-staging-v2`
value: "docs-staging-v2"
}

# It will upload the docker image after successful builds.
env_vars: {
key: "TRAMPOLINE_IMAGE_UPLOAD"
value: "true"
}

# It will always build the docker image.
env_vars: {
key: "TRAMPOLINE_DOCKERFILE"
value: ".kokoro/docker/docs/Dockerfile"
}

# Fetch the token needed for reporting release status to GitHub
before_action {
fetch_keystore {
keystore_resource {
keystore_config_id: 73713
keyname: "yoshi-automation-github-key"
}
}
}

before_action {
fetch_keystore {
keystore_resource {
keystore_config_id: 73713
keyname: "docuploader_service_account"
}
}
}
28 changes: 28 additions & 0 deletions .kokoro/docs/docs-presubmit.cfg
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
# Format: //devtools/kokoro/config/proto/build.proto

env_vars: {
key: "STAGING_BUCKET"
value: "gcloud-python-test"
}

env_vars: {
key: "V2_STAGING_BUCKET"
value: "gcloud-python-test"
}

# We only upload the image in the main `docs` build.
env_vars: {
key: "TRAMPOLINE_IMAGE_UPLOAD"
value: "false"
}

env_vars: {
key: "TRAMPOLINE_BUILD_FILE"
value: "github/genai-toolbox-langchain-python/.kokoro/build.sh"
}

# Only run this nox session.
env_vars: {
key: "NOX_SESSION"
value: "docs docfx"
}
1 change: 1 addition & 0 deletions .kokoro/docs/docs.cfg
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
# Format: //devtools/kokoro/config/proto/build.proto
43 changes: 43 additions & 0 deletions .kokoro/populate-secrets.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
#!/bin/bash
# Copyright 2025 Google LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -eo pipefail

function now { date +"%Y-%m-%d %H:%M:%S" | tr -d '\n' ;}
function msg { println "$*" >&2 ;}
function println { printf '%s\n' "$(now) $*" ;}


# Populates requested secrets set in SECRET_MANAGER_KEYS from service account:
# kokoro-trampoline@cloud-devrel-kokoro-resources.iam.gserviceaccount.com
SECRET_LOCATION="${KOKORO_GFILE_DIR}/secret_manager"
msg "Creating folder on disk for secrets: ${SECRET_LOCATION}"
mkdir -p ${SECRET_LOCATION}
for key in $(echo ${SECRET_MANAGER_KEYS} | sed "s/,/ /g")
do
msg "Retrieving secret ${key}"
docker run --entrypoint=gcloud \
--volume=${KOKORO_GFILE_DIR}:${KOKORO_GFILE_DIR} \
gcr.io/google.com/cloudsdktool/cloud-sdk \
secrets versions access latest \
--project cloud-devrel-kokoro-resources \
--secret ${key} > \
"${SECRET_LOCATION}/${key}"
if [[ $? == 0 ]]; then
msg "Secret written to ${SECRET_LOCATION}/${key}"
else
msg "Error retrieving secret ${key}"
fi
done
Loading