Dev

.env.docker

@@ -2,21 +2,40 @@
 ADMINS="Gabriel Milan,gabriel.gazola@poli.ufrj.br"
 EMAIL_HOST_USER="notifications@gmail.com"
 EMAIL_HOST_PASSWORD="password"
+
 # Django configurations
 DJANGO_SECRET_KEY="some-secret"
 DJANGO_SETTINGS_MODULE="backend.settings.local"
+
 # Logger
 LOGGER_LEVEL="DEBUG"
 LOGGER_IGNORE="faker,haystack"
 LOGGER_SERIALIZE=""
+
 # Database
 DB_HOST="database"
 DB_PORT="5432"
 DB_NAME="postgres"
 DB_USER="postgres"
 DB_PASSWORD="postgres"
+
 # Queue
 REDIS_HOST="queue"
 REDIS_PORT="6379"
+
 # Index
 ELASTICSEARCH_URL=http://index:9200
+
+# Chatbot
+CHATBOT_CREDENTIALS=
+BILLING_PROJECT_ID=
+QUERY_PROJECT_ID=
+MODEL_URI=
+EMBEDDING_MODEL=
+PGVECTOR_COLLECTION=
+LANGCHAIN_TRACING_V2=
+LANGCHAIN_API_KEY=
+
+# Local DB populating
+METABASE_USER=
+METABASE_PASSWORD=

.github/workflows/ci-python.yaml

@@ -18,6 +18,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+        with:
+          submodules: recursive
       - name: Set up poetry
         run: pipx install poetry
       - name: Set up python

.github/workflows/deploy-dev.yaml

@@ -56,10 +56,10 @@ jobs:
             resources:
               limits:
                 cpu: 500m
-                memory: 1Gi
+                memory: 4096Mi
               requests:
                 cpu: 250m
-                memory: 500Mi
+                memory: 1536Mi
             envFrom:
               - secretRef:
                   name: api-development-secrets
@@ -87,6 +87,7 @@ jobs:
         run: |-
           helm upgrade \
             --wait \
+            --timeout 15m \
             --install \
             --namespace website \
             --values values.yaml \

.github/workflows/deploy-prod.yaml

@@ -8,9 +8,7 @@ on:
 jobs:
   deploy-prod:
     runs-on: ubuntu-latest
-    if: |
-      ${{ github.event_name == 'workflow_dispatch' }}
-      || ${{ github.event.workflow_run.conclusion == 'success' }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
     environment:
       name: production
       url: https://backend.basedosdados.org

.github/workflows/deploy-staging.yaml

@@ -8,9 +8,7 @@ on:
 jobs:
   deploy-staging:
     runs-on: ubuntu-latest
-    if: |
-      ${{ github.event_name == 'workflow_dispatch' }}
-      || ${{ github.event.workflow_run.conclusion == 'success' }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
     environment:
       name: staging
       url: https://staging.backend.basedosdados.org

.github/workflows/release-dev.yaml

@@ -2,8 +2,7 @@
 name: Release Image (Development)
 on:
   push:
-    branches:
-      - dev
+    branches: [dev]
 jobs:
   release-docker:
     name: Release Image
@@ -13,6 +12,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           ref: dev
+          submodules: recursive
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v2
         with:
@@ -30,6 +30,5 @@ jobs:
           labels: |
             org.opencontainers.image.source=${{ github.event.repository.html_url }}
             org.opencontainers.image.revision=${{ github.sha }}
-          build-args: |
+          build-args: |-
             BUILDKIT_INLINE_CACHE=1
-

.github/workflows/release-prod.yaml

@@ -2,8 +2,7 @@
 name: Release Image (Production)
 on:
   push:
-    branches:
-      - main
+    branches: [main]
 jobs:
   release-docker:
     name: Release Image
@@ -13,6 +12,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           ref: main
+          submodules: recursive
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v2
         with:

.github/workflows/release-staging.yaml

@@ -2,8 +2,7 @@
 name: Release Image (Staging)
 on:
   push:
-    branches:
-      - staging
+    branches: [staging]
 jobs:
   release-docker:
     name: Release Image
@@ -13,6 +12,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           ref: staging
+          submodules: recursive
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v2
         with:

.gitignore

@@ -256,3 +256,6 @@ fixtures*
 
 # Version manager
 .tool-versions
+
+# fetch_metabase script
+metabase_data

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "chatbot"]
+	path = chatbot
+	url = git@github.com:basedosdados/chatbot.git

.pre-commit-config.yaml

@@ -36,4 +36,5 @@ repos:
         name: yamlfix
         types: [yaml]
         language: system
-        entry: yamlfix --exclude "charts/**/*" .
+        entry: yamlfix
+        exclude: ^charts/

Dockerfile

@@ -1,38 +1,49 @@
 ARG PYTHON_VERSION=3.11-slim
 
-FROM python:${PYTHON_VERSION}
+FROM python:$PYTHON_VERSION
 
-# Install virtualenv and create a virtual environment
-RUN pip install --no-cache-dir -U virtualenv>=20.13.1 && virtualenv /env --python=python3.11
-ENV PATH /env/bin:$PATH
+# Define where Poetry virtual environments will be stored
+ARG POETRY_VIRTUALENVS_PATH=/opt/pypoetry/virtualenvs
 
-# Install pip requirements
-WORKDIR /app
-COPY . .
-RUN /env/bin/pip install --no-cache-dir . && rm nginx.conf
+# Ensure that the python output is sent straight to terminal (e.g. your container log)
+# without being first buffered and that you can see the output of your application (e.g. django logs)
+# in real time. Equivalent to python -u: https://docs.python.org/3/using/cmdline.html#cmdoption-u
+ENV PYTHONUNBUFFERED=1
+
+# Prevent Python from writing .pyc files to disc
+# https://docs.python.org/3/using/cmdline.html#envvar-PYTHONDONTWRITEBYTECODE
+ENV PYTHONDONTWRITEBYTECODE=1
 
 # Install make, nginx and copy configuration
 RUN apt-get update \
     && apt-get install -y --no-install-recommends curl make nginx \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* \
     && rm /etc/nginx/sites-enabled/default
+RUN apt-get update && apt-get install -y postgresql postgresql-contrib
 COPY nginx.conf /etc/nginx/nginx.conf
 
-# Prevents Python from writing .pyc files to disc
-# https://docs.python.org/3/using/cmdline.html#envvar-PYTHONDONTWRITEBYTECODE
-ENV PYTHONDONTWRITEBYTECODE 1
+# Install Poetry and add it to PATH so its commands can be executed
+# from anywhere, without specifying the full path to its executable.
+RUN curl -sSL https://install.python-poetry.org | python3 - --version 2.1.3
+ENV PATH="/root/.local/bin:$PATH"
 
-# Ensures that the python output is sent straight to terminal (e.g. your container log)
-# without being first buffered and that you can see the output of your application (e.g. django logs)
-# in real time. Equivalent to python -u: https://docs.python.org/3/using/cmdline.html#cmdoption-u
-ENV PYTHONUNBUFFERED 1
+# Create the folder where Poetry virtual environments will be stored and make it
+# accessible to all users. This is needed by the 'www-data' user during server startup
+RUN mkdir -p $POETRY_VIRTUALENVS_PATH && chmod 755 $POETRY_VIRTUALENVS_PATH
+ENV POETRY_VIRTUALENVS_PATH=$POETRY_VIRTUALENVS_PATH
+
+# Copy and install project
+WORKDIR /app
+COPY . .
+RUN test -d ./chatbot || (echo "ERROR: Git submodule 'chatbot' not found. Please run 'git submodule update --init --recursive'. See backend/README.md for more information." && exit 1)
+RUN poetry install --only main && rm nginx.conf
 
-# Copy app, generate static and set permissions
-RUN /env/bin/python manage.py collectstatic --no-input --settings=backend.settings.base && \
+# Generate static and set permissions
+RUN poetry run python manage.py collectstatic --no-input --settings=backend.settings.base && \
     chown -R www-data:www-data /app
 
 # Expose and run app
 EXPOSE 80
 STOPSIGNAL SIGKILL
-CMD ["/app/start-server.sh"]
+CMD ["poetry", "run", "/app/start-server.sh"]

README.md

@@ -17,6 +17,11 @@
   git clone https://github.com/basedosdados/backend.git
   ```
 
+- Baixar submodulos
+  ```
+  git submodule update --recursive --init
+  ```
+
 - Abrí-lo no seu editor de texto
 
 - No seu ambiente de desenvolvimento, instalar [poetry](https://python-poetry.org/) para gerenciamento de dependências

backend/apps/account/admin.py

@@ -10,7 +10,7 @@
 from django.utils.translation import gettext_lazy
 from faker import Faker
 
-from backend.apps.account.models import Account, BDGroup, BDRole, Team, Role, Career, Subscription
+from backend.apps.account.models import Account, BDGroup, BDRole, Career, Role, Subscription, Team
 from backend.apps.account.tasks import sync_subscription_task
 
 
@@ -204,6 +204,7 @@ class AccountAdmin(BaseAccountAdmin):
         "created_at",
         "is_admin",
         "is_subscriber",
+        "has_chatbot_access",
     )
     list_filter = (
         SuperUserListFilter,
@@ -264,6 +265,7 @@ class AccountAdmin(BaseAccountAdmin):
                     "is_active",
                     "is_admin",
                     "is_superuser",
+                    "has_chatbot_access",
                     "staff_groups",
                 )
             },

backend/apps/account/migrations/0001_initial.py

@@ -84,9 +84,7 @@ class Migration(migrations.Migration):
                 ),
                 (
                     "twitter",
-                    models.CharField(
-                        blank=True, max_length=255, null=True, verbose_name="Twitter"
-                    ),
+                    models.CharField(blank=True, max_length=255, null=True, verbose_name="Twitter"),
                 ),
                 (
                     "linkedin",

backend/apps/account/migrations/0019_role_team_alter_career_end_at_alter_career_level_and_more.py

@@ -1,7 +1,8 @@
+# -*- coding: utf-8 -*-
 # Generated by Django 4.2.18 on 2025-02-04 04:02
 
-from django.db import migrations, models
 import django.db.models.deletion
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):

backend/apps/account/migrations/0020_career_role_new.py

@@ -1,7 +1,8 @@
+# -*- coding: utf-8 -*-
 # Generated by Django 4.2.18 on 2025-02-04 04:04
 
-from django.db import migrations, models
 import django.db.models.deletion
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):

backend/apps/account/migrations/0021_rename_role_career_role_old_and_more.py

@@ -1,23 +1,23 @@
+# -*- coding: utf-8 -*-
 # Generated by Django 4.2.18 on 2025-02-05 00:43
 
-from django.db import migrations, models
+from django.db import migrations
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
-        ('account', '0020_career_role_new'),
+        ("account", "0020_career_role_new"),
     ]
 
     operations = [
         migrations.RenameField(
-            model_name='career',
-            old_name='role',
-            new_name='role_old',
+            model_name="career",
+            old_name="role",
+            new_name="role_old",
         ),
         migrations.RenameField(
-            model_name='career',
-            old_name='team',
-            new_name='team_old',
+            model_name="career",
+            old_name="team",
+            new_name="team_old",
         ),
     ]

backend/apps/account/migrations/0022_rename_role_new_career_role_and_more.py

@@ -4,20 +4,19 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
-        ('account', '0021_rename_role_career_role_old_and_more'),
+        ("account", "0021_rename_role_career_role_old_and_more"),
     ]
 
     operations = [
         migrations.RenameField(
-            model_name='career',
-            old_name='role_new',
-            new_name='role',
+            model_name="career",
+            old_name="role_new",
+            new_name="role",
         ),
         migrations.RenameField(
-            model_name='career',
-            old_name='team_new',
-            new_name='team',
+            model_name="career",
+            old_name="team_new",
+            new_name="team",
         ),
     ]

backend/apps/account/migrations/0023_alter_career_role_old_alter_career_team_old.py

@@ -4,20 +4,19 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
-        ('account', '0022_rename_role_new_career_role_and_more'),
+        ("account", "0022_rename_role_new_career_role_and_more"),
     ]
 
     operations = [
         migrations.AlterField(
-            model_name='career',
-            name='role_old',
-            field=models.CharField(blank=True, max_length=40, verbose_name='Role (old)'),
+            model_name="career",
+            name="role_old",
+            field=models.CharField(blank=True, max_length=40, verbose_name="Role (old)"),
         ),
         migrations.AlterField(
-            model_name='career',
-            name='team_old',
-            field=models.CharField(blank=True, max_length=40, verbose_name='Team (old)'),
+            model_name="career",
+            name="team_old",
+            field=models.CharField(blank=True, max_length=40, verbose_name="Team (old)"),
         ),
     ]