diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 7941bc9..518e334 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,42 +1,8 @@ -ARG PYTHON_VERSION=3.12 +FROM caltrans/pems:app -FROM python:${PYTHON_VERSION} - -ENV PYTHONDONTWRITEBYTECODE=1 \ - PYTHONUNBUFFERED=1 \ - USER=caltrans - -# create non-root $USER and home directory -RUN useradd --create-home --shell /bin/bash $USER && \ - chown -R $USER /home/$USER - -# switch to $USER -USER $USER - -# enter src directory -WORKDIR /home/$USER/src - -# update PATH for local pip installs -ENV PATH="$PATH:/home/$USER/.local/bin" - -# upgrade pip -RUN python -m pip install --upgrade pip - -# copy assets COPY . . - # install devcontainer requirements RUN pip install -e .[dev,test] # install docs requirements RUN pip install --no-cache-dir -r docs/requirements.txt - -# install pre-commit environments in throwaway Git repository -# https://stackoverflow.com/a/68758943 -RUN git init . && \ - pre-commit install-hooks && \ - rm -rf .git - -CMD sleep infinity - -ENTRYPOINT [] diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 915ec56..ac6c6c0 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -3,7 +3,7 @@ "dockerComposeFile": ["../compose.yml"], "service": "dev", "forwardPorts": ["docs:8000", "kibana:5601"], - "workspaceFolder": "/home/caltrans/src", + "workspaceFolder": "/caltrans/app", "postStartCommand": ["/bin/bash", "bin/reset_db.sh"], "postAttachCommand": ["/bin/bash", ".devcontainer/postAttach.sh"], "customizations": { diff --git a/.devcontainer/postAttach.sh b/.devcontainer/postAttach.sh index 2e748e6..ceb4aa8 100644 --- a/.devcontainer/postAttach.sh +++ b/.devcontainer/postAttach.sh @@ -3,4 +3,4 @@ set -eux # initialize pre-commit git config --global --add safe.directory /home/$USER/src -pre-commit install --overwrite +pre-commit install --install-hooks --overwrite diff --git a/.dockerignore b/.dockerignore index 3842b8d..897f101 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,2 @@ -.git/ *.egg-info *.db diff --git a/.env.sample b/.env.sample index 6bdc79f..45e563f 100644 --- a/.env.sample +++ b/.env.sample @@ -8,6 +8,9 @@ DJANGO_DB_RESET=true DJANGO_STORAGE_DIR=. DJANGO_DB_FILE=django.db +# Other Django config +DJANGO_DEBUG=true + # uncomment to start the elasticstack services with compose # COMPOSE_PROFILES=elasticstack diff --git a/.vscode/settings.json b/.vscode/settings.json index d34973e..a831bf7 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -20,6 +20,9 @@ "[python]": { "editor.defaultFormatter": "ms-python.black-formatter" }, + "[toml]": { + "editor.defaultFormatter": "tamasfe.even-better-toml" + }, "python.languageServer": "Pylance", "python.testing.pytestArgs": ["tests/pytest"], "python.testing.pytestEnabled": true, diff --git a/appcontainer/Dockerfile b/appcontainer/Dockerfile new file mode 100644 index 0000000..d1b7d33 --- /dev/null +++ b/appcontainer/Dockerfile @@ -0,0 +1,90 @@ +ARG PYTHON_VERSION=3.12 + +# multi-stage build +# +# stage 1: builds the pems package from source +# using the git metadata for version info +FROM python:${PYTHON_VERSION} AS build_wheel +WORKDIR /build + +# upgrade pip +RUN python -m pip install --upgrade pip && \ + pip install build + +# copy source files +COPY . . +RUN git config --global --add safe.directory /build + +# build package +RUN python -m build + +# multi-stage build +# +# stage 2: installs the pems package in a fresh base container +# using the pre-built package, and copying only needed source +FROM python:${PYTHON_VERSION} AS app_container + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + USER=caltrans + +EXPOSE 8000 + + # create non-root $USER and home directory +RUN useradd --create-home --shell /bin/bash $USER && \ + # setup $USER permissions for nginx + mkdir -p /var/cache/nginx && \ + chown -R $USER:$USER /var/cache/nginx && \ + mkdir -p /var/lib/nginx && \ + chown -R $USER:$USER /var/lib/nginx && \ + mkdir -p /var/log/nginx && \ + chown -R $USER:$USER /var/log/nginx && \ + touch /var/log/nginx/error.log && \ + chown $USER:$USER /var/log/nginx/error.log && \ + touch /var/run/nginx.pid && \ + chown -R $USER:$USER /var/run/nginx.pid && \ + # setup directories and permissions for gunicorn, (eventual) app + mkdir -p /$USER/app && \ + mkdir -p /$USER/run && \ + chown -R $USER:$USER /$USER && \ + # install server components + apt-get update && \ + apt-get install -qq --no-install-recommends build-essential nginx gettext && \ + python -m pip install --upgrade pip + +# enter source directory +WORKDIR /$USER + +COPY LICENSE app/LICENSE + +# switch to non-root $USER +USER $USER + +# update env for local pip installs +# see https://docs.python.org/3/using/cmdline.html#envvar-PYTHONUSERBASE +# since all `pip install` commands are in the context of $USER +# $PYTHONUSERBASE is the location used by default +ENV PATH="$PATH:/$USER/.local/bin" \ + PYTHONUSERBASE="/$USER/.local" + +# copy gunicorn config file +COPY appcontainer/gunicorn.conf.py run/gunicorn.conf.py +ENV GUNICORN_CONF "/$USER/run/gunicorn.conf.py" + +# overwrite default nginx.conf +COPY appcontainer/nginx.conf /etc/nginx/nginx.conf + +WORKDIR /$USER/app + +# copy runtime files +COPY --from=build_wheel /build/dist /build/dist +COPY manage.py manage.py +COPY bin bin +COPY pems pems + +# install source as a package +RUN pip install $(find /build/dist -name pems*.whl) + +# configure container executable +ENTRYPOINT ["/bin/bash"] +CMD ["bin/start.sh"] diff --git a/appcontainer/gunicorn.conf.py b/appcontainer/gunicorn.conf.py new file mode 100644 index 0000000..83a9adc --- /dev/null +++ b/appcontainer/gunicorn.conf.py @@ -0,0 +1,20 @@ +""" +The Gunicorn configuration file +More information: https://docs.gunicorn.org/en/stable/settings.html +""" + +import multiprocessing + +# the unix socket defined in nginx.conf +bind = "unix:/caltrans/run/gunicorn.sock" + +# Recommend (2 x $num_cores) + 1 as the number of workers to start off with +workers = multiprocessing.cpu_count() * 2 + 1 + +# send logs to stdout and stderr +accesslog = "-" +errorlog = "-" + +# Preloading can save some RAM resources as well as speed up server boot times, +# at the cost of not being able to reload app code by restarting workers +preload_app = True diff --git a/appcontainer/nginx.conf b/appcontainer/nginx.conf new file mode 100644 index 0000000..0f488e1 --- /dev/null +++ b/appcontainer/nginx.conf @@ -0,0 +1,80 @@ +worker_processes auto; +error_log stderr warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; + accept_mutex on; +} + +http { + include mime.types; + default_type application/octet-stream; + sendfile on; + gzip on; + keepalive_timeout 5; + + log_format main '$remote_addr - $remote_user [$time_local] ' + '"$request" $status $body_bytes_sent ' + '"$http_referer" "$http_user_agent" "$gzip_ratio"'; + + access_log /dev/stdout main; + + upstream app_server { + # fail_timeout=0 means we always retry an upstream even if it failed + # to return a good HTTP response + server unix:/caltrans/run/gunicorn.sock fail_timeout=0; + } + + server { + listen 8000; + + keepalive_timeout 65; + + # 404 known scraping path targets + # case-insensitive regex matches the given path fragment anywhere in the request path + location ~* /(\.?git|api|app|assets|ats|bootstrap|bower|cgi|content|cpanel|credentials|debug|docker|doc|env|example|jenkins|robots|swagger|web|yq) { + access_log off; + log_not_found off; + return 404; + } + + # 404 known scraping file targets + # case-insensitive regex matches the given file extension anywhere in the request path + location ~* /.*\.(ash|asp|axd|cgi|com|db|env|json|php|ping|sqlite|xml|ya?ml) { + access_log off; + log_not_found off; + return 404; + } + + location /favicon.ico { + access_log off; + log_not_found off; + expires 1y; + add_header Cache-Control public; + } + + # path for static files + location /static/ { + alias /caltrans/app/static/; + expires 1y; + add_header Cache-Control public; + } + + location / { + # checks for static file, if not found proxy to app + try_files $uri @proxy_to_app; + } + + # app path + location @proxy_to_app { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + proxy_pass http://app_server; + } + } +} diff --git a/bin/start.sh b/bin/start.sh new file mode 100755 index 0000000..eb54bb0 --- /dev/null +++ b/bin/start.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -eu + +# initialize Django + +bin/init.sh + +# start the web server + +nginx + +# start the application server + +python -m gunicorn -c $GUNICORN_CONF pems.wsgi diff --git a/compose.yml b/compose.yml index da61384..d554a91 100644 --- a/compose.yml +++ b/compose.yml @@ -1,23 +1,34 @@ name: pems services: + app: + build: + context: . + dockerfile: appcontainer/Dockerfile + image: caltrans/pems:app + env_file: .env + ports: + - "8000" + dev: build: context: . dockerfile: .devcontainer/Dockerfile - image: caltrans/pems:main + image: caltrans/pems:dev env_file: .env + # https://code.visualstudio.com/docs/remote/create-dev-container#_use-docker-compose + entrypoint: sleep infinity volumes: - - ./:/home/caltrans/src + - ./:/caltrans/app docs: - image: caltrans/pems:main + image: caltrans/pems:dev entrypoint: mkdocs command: serve --dev-addr "0.0.0.0:8000" ports: - "8000" volumes: - - ./:/home/caltrans/src + - ./:/caltrans/app esconfig: profiles: ["elasticstack"] diff --git a/manage.py b/manage.py index 12fddef..bdd5304 100755 --- a/manage.py +++ b/manage.py @@ -3,10 +3,17 @@ import os import sys +from pems import __version__ as VERSION + def main(): """Run administrative tasks.""" os.environ.setdefault("DJANGO_SETTINGS_MODULE", "pems.settings") + + if len(sys.argv) == 2 and sys.argv[1] == "--version" or sys.argv[1] == "-v": + print(f"pems, {VERSION}") + return + try: from django.core.management import execute_from_command_line except ImportError as exc: diff --git a/pyproject.toml b/pyproject.toml index 1ddcf1e..39505e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,32 +1,18 @@ [project] name = "pems" -version = "0.0.1" +dynamic = ["version"] description = "Caltrans Performance Measurement System (PeMS) is an application that enables access to traffic data collected by sensors that span the freeway system across all major metropolitan areas of the State of California." readme = "README.md" license = { file = "LICENSE" } classifiers = ["Programming Language :: Python :: 3 :: Only"] requires-python = ">=3.12" -maintainers = [ - { name = "Compiler LLC", email = "dev@compiler.la" } -] -dependencies = [ - "Django==5.1.4" -] +maintainers = [{ name = "Compiler LLC", email = "dev@compiler.la" }] +dependencies = ["Django==5.1.4", "gunicorn==23.0.0"] [project.optional-dependencies] -dev = [ - "black", - "flake8", - "pre-commit" -] - -test = [ - "coverage", - "pytest", - "pytest-django", - "pytest-mock", - "pytest-socket", -] +dev = ["black", "flake8", "pre-commit", "setuptools_scm>=8"] + +test = ["coverage", "pytest", "pytest-django", "pytest-mock", "pytest-socket"] [project.urls] Code = "https://github.com/compilerla/pems" @@ -34,12 +20,12 @@ Homepage = "https://compilerla.github.io/pems/" Issues = "https://github.com/compilerla/pems/issues" [build-system] -requires = ["setuptools>=75"] +requires = ["setuptools>=75", "setuptools_scm>=8"] build-backend = "setuptools.build_meta" [tool.black] line-length = 127 -target-version = ['py312'] +target-version = ["py312"] include = '\.pyi?$' [tool.coverage.run] @@ -53,3 +39,6 @@ DJANGO_SETTINGS_MODULE = "pems.settings" [tool.setuptools.packages.find] include = ["pems*"] namespaces = false + +[tool.setuptools_scm] +# intentionally left blank, but we need the section header to activate the tool