Skip to content

Commit 44d65d2

Browse files
Merge pull request #182 from basedosdados/staging
O PR mais aguardado do momento
2 parents efa3e21 + 4c8f8b0 commit 44d65d2

File tree

140 files changed

+15871
-1011
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

140 files changed

+15871
-1011
lines changed

.dockerignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -244,3 +244,4 @@ fabric.properties
244244
Dockerfile
245245
Makefile
246246
README.md
247+
*sqlite3

.editorconfig

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
root = true
2+
3+
[*]
4+
indent_style = space
5+
indent_size = 4
6+
charset = utf-8
7+
trim_trailing_whitespace = true
8+
insert_final_newline = true
9+
end_of_line = lf
10+
max_line_length = off

.env.example

Lines changed: 17 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,17 @@
1-
# bdd_app
2-
export API_HOST=0.0.0.0
3-
export API_PORT=8080
4-
export VAULT_ADDRESS=
5-
export VAULT_TOKEN=
6-
# bdd_postgres
7-
export POSTGRES_USER=postgres
8-
export POSTGRES_PASSWORD=postgres
9-
export POSTGRES_DB=api
10-
export POSTGRES_PORT=5432
11-
# bdd_pg_admin
12-
export PGADMIN_LISTEN_ADDRESS=0.0.0.0
13-
export PGADMIN_LISTEN_PORT=5050
14-
export PGADMIN_DEFAULT_EMAIL=postgres@postgres.com
15-
export PGADMIN_DEFAULT_PASSWORD=postgres
1+
# Mail notifications
2+
ADMINS="Gabriel Milan,gabriel.gazola@poli.ufrj.br;Guilherme Peres,contact@gperes.dev"
3+
EMAIL_HOST_USER="notifications@gmail.com"
4+
EMAIL_HOST_PASSWORD="password"
5+
# Django configurations
6+
DJANGO_SECRET_KEY="some-secret"
7+
DJANGO_SETTINGS_MODULE="basedosdados_api.settings.dev"
8+
# Django database
9+
DB_HOST="database"
10+
DB_PORT="5432"
11+
DB_NAME="postgres"
12+
DB_USER="postgres"
13+
DB_PASSWORD="postgres"
14+
# Postgres database
15+
POSTGRES_USER="postgres"
16+
POSTGRES_PASSWORD="postgres"
17+
POSTGRES_DB="postgres"

.flake8

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,4 @@ exclude =
33
.git,
44
__pycache__,
55
.venv
6-
max-line-length = 100
6+
max-line-length = 120

.github/workflows/cd-prod.yaml

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
name: Deployment - Production
2+
3+
on:
4+
workflow_run:
5+
workflows: ["Docker - Release image"]
6+
branches:
7+
- main
8+
types:
9+
- completed
10+
workflow_dispatch:
11+
12+
jobs:
13+
deploy-prod:
14+
runs-on: ubuntu-latest
15+
if: ${{ github.event.workflow_run.conclusion == 'success' }}
16+
environment:
17+
name: production
18+
url: https://api.basedosdados.org
19+
20+
steps:
21+
- name: Checkout
22+
uses: actions/checkout@v3
23+
with:
24+
ref: production
25+
26+
- name: Import secrets from Vault
27+
id: import_secrets
28+
uses: hashicorp/vault-action@v2.4.1
29+
with:
30+
url: https://vault.basedosdados.org
31+
token: ${{ secrets.VAULT_TOKEN }}
32+
secrets: |
33+
secret/data/gcp_credentials/basedosdados-dev GCP_PROJECT_ID | GCP_PROJECT_ID ;
34+
secret/data/gcp_credentials/basedosdados-dev GH_ACTIONS_SA | GCP_SA ;
35+
secret/data/gcp_credentials/basedosdados-dev GKE_CLUSTER_NAME | GKE_CLUSTER_NAME ;
36+
secret/data/gcp_credentials/basedosdados-dev GKE_CLUSTER_ZONE | GKE_CLUSTER_ZONE ;
37+
38+
- name: Setup Google Cloud CLI
39+
uses: google-github-actions/setup-gcloud@v0.2.1
40+
with:
41+
service_account_key: ${{ steps.import_secrets.outputs.GCP_SA }}
42+
project_id: ${{ steps.import_secrets.outputs.GCP_PROJECT_ID }}
43+
export_default_credentials: true
44+
45+
- name: Get GKE credentials
46+
uses: google-github-actions/get-gke-credentials@v0.2.1
47+
with:
48+
cluster_name: ${{ steps.import_secrets.outputs.GKE_CLUSTER_NAME }}
49+
location: ${{ steps.import_secrets.outputs.GKE_CLUSTER_ZONE }}
50+
credentials: ${{ steps.import_secrets.outputs.GCP_SA }}
51+
52+
- name: Write values.yaml file
53+
run: |
54+
cat << EOF > values.yaml
55+
api:
56+
name: "api-prod"
57+
image:
58+
name: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}"
59+
tag: "stable"
60+
pullPolicy: "Always"
61+
replicas: 1
62+
resources:
63+
limits:
64+
cpu: 2000m
65+
memory: 2048Mi
66+
requests:
67+
cpu: 1000m
68+
memory: 1024Mi
69+
env: []
70+
envFrom:
71+
- secretRef:
72+
name: api-prod-secrets
73+
settingsModule: "basedosdados_api.settings.prod"
74+
database:
75+
host: "cloud-sql-proxy"
76+
port: 5432
77+
name: "api"
78+
user: "api"
79+
passwordSecret: "api-prod-database-password"
80+
ingress:
81+
enabled: true
82+
host: "api.basedosdados.org"
83+
annotations:
84+
kubernetes.io/ingress.class: nginx
85+
nginx.ingress.kubernetes.io/rewrite-target: /
86+
cert-manager.io/issuer: "letsencrypt-production"
87+
nginx.ingress.kubernetes.io/ssl-redirect: "true"
88+
tls:
89+
- hosts:
90+
- api.basedosdados.org
91+
secretName: api-basedosdados-org-tls
92+
EOF
93+
94+
- name: Deploy using Helm
95+
run: |
96+
helm upgrade --install api-prod charts/basedosdados-api/. -n website -f values.yaml --wait

.github/workflows/cd-staging.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ jobs:
3232
secret/data/github_actions_deployment_secrets GKE_CLUSTER | GKE_CLUSTER ;
3333
secret/data/github_actions_deployment_secrets GKE_ZONE | GKE_ZONE ;
3434
35-
- name: Setup Google Cloud CLI
35+
- name: Setup Google Cloud CLI
3636
uses: google-github-actions/setup-gcloud@v0.2.1
3737
with:
3838
service_account_key: ${{ steps.import_secrets.outputs.GCP_SA_KEY }}
@@ -117,7 +117,7 @@ jobs:
117117
secret/data/gcp_credentials/basedosdados-dev GKE_CLUSTER_NAME | GKE_CLUSTER_NAME ;
118118
secret/data/gcp_credentials/basedosdados-dev GKE_CLUSTER_ZONE | GKE_CLUSTER_ZONE ;
119119
120-
- name: Setup Google Cloud CLI
120+
- name: Setup Google Cloud CLI
121121
uses: google-github-actions/setup-gcloud@v0.2.1
122122
with:
123123
service_account_key: ${{ steps.import_secrets.outputs.GCP_SA }}
@@ -173,6 +173,6 @@ jobs:
173173
secretName: staging-api-basedosdados-org-tls
174174
EOF
175175
176-
- name: Deploy using Helm
176+
- name: Deploy using Helm
177177
run: |
178178
helm upgrade --install api-staging charts/basedosdados-api/. -n website -f values.yaml --wait

.github/workflows/lint.yaml

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
name: CI - Lint
2+
3+
on:
4+
push:
5+
6+
jobs:
7+
lint:
8+
name: ${{ matrix.os }}${{ matrix.arch }} - Python ${{ matrix.python-version }} - lint
9+
runs-on: ${{ matrix.os }}
10+
strategy:
11+
matrix:
12+
os: [ubuntu-latest]
13+
arch: [x64]
14+
python-version: ["3.10.x"]
15+
steps:
16+
- name: Checkout
17+
uses: actions/checkout@v2
18+
19+
- name: Set up Python
20+
uses: actions/setup-python@v2
21+
with:
22+
python-version: ${{ matrix.python-version }}
23+
architecture: ${{ matrix.arch }}
24+
25+
- name: Set up Poetry and upgrade pip
26+
run: |
27+
pip install -U pip poetry
28+
- name: Install this package
29+
run: |
30+
poetry install
31+
- name: Run tests
32+
run: |
33+
poetry run lint

.github/workflows/release-docker.yaml

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,20 +4,22 @@ on:
44
push:
55
branches:
66
- main
7-
paths:
8-
- ".github/workflows/release-docker.yaml"
9-
- "app/**/*"
10-
- "Dockerfile"
7+
- staging
118
pull_request:
129
branches:
1310
- main
11+
- staging
1412
paths:
1513
- ".github/workflows/release-docker.yaml"
1614
- "app/**/*"
1715
- "Dockerfile"
16+
pull_request_review:
17+
types:
18+
- submitted
1819

1920
jobs:
2021
release-docker:
22+
name: Docker - Release image
2123
runs-on: ubuntu-latest
2224
steps:
2325
- name: Checkout
@@ -45,9 +47,24 @@ jobs:
4547
build-args: |
4648
BUILDKIT_INLINE_CACHE=1
4749
50+
- name: Build and push staging tagged image
51+
uses: docker/build-push-action@v2
52+
if: github.event_name == 'push' && github.ref == 'refs/heads/staging' # If it's a push to staging, build and push with the staging tag
53+
with:
54+
context: .
55+
file: ./Dockerfile
56+
push: true
57+
tags: |
58+
ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}:staging
59+
labels: |
60+
org.opencontainers.image.source=${{ github.event.repository.html_url }}
61+
org.opencontainers.image.revision=${{ github.sha }}
62+
build-args: |
63+
BUILDKIT_INLINE_CACHE=1
64+
4865
- name: Build and push stable tagged image
4966
uses: docker/build-push-action@v2
50-
if: github.event_name == 'push' # If it's a push to main, build and push with the stable tag
67+
if: github.event_name == 'push' && github.ref == 'refs/heads/main' # If it's a push to main, build and push with the stable tag
5168
with:
5269
context: .
5370
file: ./Dockerfile

.gitignore

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
# Byte-compiled / optimized / DLL files
2+
3+
credentials.json
4+
utils/migration/data/*
5+
**/data/*
6+
27
__pycache__/
38
*.py[cod]
49
*$py.class
@@ -236,3 +241,11 @@ fabric.properties
236241

237242
# Android studio 3.1+ serialized cache file
238243
.idea/caches/build_file_checksums.ser
244+
245+
*sqlite3
246+
247+
.vscode/
248+
.DS_Store
249+
/basedosdados_api/media/
250+
/basedosdados_api/notebooks/
251+
/basedosdados_api/staticfiles/

Dockerfile

Lines changed: 27 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -1,48 +1,27 @@
11
# Build arguments
2-
ARG PYTHON_VERSION=3.9
2+
ARG PYTHON_VERSION=3.10-slim
33

4-
# Start from the official Python base image on version: 3.9
5-
# First stage, build the application with the dependencies that
6-
# managed by Poetry.
7-
FROM python:${PYTHON_VERSION} as requirements-stage
8-
9-
# Set python virtual environment
10-
ENV VIRTUAL_ENV=/opt/venv
11-
RUN python -m venv $VIRTUAL_ENV
12-
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
13-
14-
# Set /tmp as the current working directory. Here's where we will
15-
# generate the file requirements.txt.
16-
WORKDIR /tmp
17-
18-
# Update pip and install Poetry in this Docker stage.
19-
RUN pip install --upgrade pip && \
20-
pip install poetry
21-
22-
# Copy the pyproject.toml and poetry.lock files to the /tmp directory.
23-
# Because it uses ./poetry.lock* (ending with a *), it won't crash if
24-
# that file is not available yet.
25-
ADD ./pyproject.toml ./poetry.lock* /tmp/
26-
27-
# Generate the requirements.txt file.
28-
RUN poetry export -f requirements.txt --output requirements.txt --without-hashes
29-
30-
# This is the final stage, anything here will be preserved in the final
31-
# container image.
324
FROM python:${PYTHON_VERSION}
335

34-
# Create the app user
35-
RUN addgroup --system app && adduser --no-create-home --system --group app
36-
37-
# Set python virtual environment
38-
ENV VIRTUAL_ENV=/opt/venv
39-
RUN python -m venv $VIRTUAL_ENV
40-
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
41-
42-
# Set the current working directory to /code. This is where
43-
# we'll put the code for our application.
44-
WORKDIR /code
45-
6+
# Install virtualenv and create a virtual environment
7+
RUN pip install --no-cache-dir -U virtualenv>=20.13.1 && virtualenv /env --python=python3.10
8+
ENV PATH /env/bin:$PATH
9+
10+
# Install pip requirements
11+
WORKDIR /app
12+
COPY . .
13+
RUN /env/bin/pip install --no-cache-dir . && \
14+
rm nginx.conf
15+
16+
# Install nginx and copy configuration
17+
RUN apt-get update && apt-get install -y --no-install-recommends nginx curl \
18+
&& apt-get clean \
19+
&& rm -rf /var/lib/apt/lists/* \
20+
&& rm /etc/nginx/sites-enabled/default \
21+
&& mkdir -p /var/log/django \
22+
&& touch /var/log/django/basedosdados_api.log \
23+
&& chown www-data:www-data /var/log/django/basedosdados_api.log
24+
COPY nginx.conf /etc/nginx/nginx.conf
4625
# https://docs.python.org/3/using/cmdline.html#envvar-PYTHONDONTWRITEBYTECODE
4726
# Prevents Python from writing .pyc files to disc
4827
ENV PYTHONDONTWRITEBYTECODE 1
@@ -52,25 +31,11 @@ ENV PYTHONDONTWRITEBYTECODE 1
5231
# in real time. Equivalent to python -u: https://docs.python.org/3/using/cmdline.html#cmdoption-u
5332
ENV PYTHONUNBUFFERED 1
5433

55-
# Copy the requirements.txt file from the requirements-stage to the
56-
# /code directory.
57-
COPY --from=requirements-stage /tmp/requirements.txt /code/requirements.txt
58-
59-
# Upgrade pip and install the dependencies from the requirements.txt file.
60-
RUN pip install --upgrade pip && \
61-
pip install --no-cache-dir --upgrade -r /code/requirements.txt
62-
63-
# Copy the api directory to the /code directory.
64-
ADD /app /code/app
65-
ADD run.sh /code/run.sh
66-
RUN chmod +x /code/run.sh
67-
68-
# chown all the files to the app user
69-
RUN chown -R app:app /code
70-
71-
# Change to the app user
72-
USER app
34+
# Copy app, generate static and set permissions
35+
RUN /env/bin/python manage.py collectstatic --no-input --settings=basedosdados_api.settings.base && \
36+
chown -R www-data:www-data /app
7337

74-
# Run the uvicorn command, telling it to use the app object imported
75-
# from api.main.
76-
CMD ["sh", "/code/run.sh"]
38+
# Expose and run app
39+
EXPOSE 80
40+
STOPSIGNAL SIGKILL
41+
CMD ["/app/start-server.sh"]

0 commit comments

Comments
 (0)