diff --git a/.env b/.env index 8ec94291..51d74b4a 100644 --- a/.env +++ b/.env @@ -4,7 +4,7 @@ VALKEY_HOST=localhost VALKEY_PORT=6379 GH_TOKEN=token REPO_ORG=https://github.com/canonical -SQLALCHEMY_DATABASE_URI=postgresql://postgres:postgres@localhost:5432/postgres +DATABASE_URL=postgresql://postgres:postgres@localhost:5432/content TASK_DELAY=30 DIRECTORY_API_TOKEN=token JIRA_EMAIL=email@example.com @@ -12,3 +12,8 @@ JIRA_TOKEN=token JIRA_URL=https://warthogs.atlassian.net JIRA_LABELS=sites_BAU JIRA_COPY_UPDATES_EPIC=KAN-1 +GOOGLE_CREDENTIALS=googlecreds +GOOGLE_DRIVE_FOLDER_ID=googlecreds +COPYD0C_TEMPLATE_ID=googlecreds +GOOGLE_PRIVATE_KEY=base64encodedprivatekey +GOOGLE_PRIVATE_KEY_ID=privatekeyid diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index b3ac3ac0..02c1a43a 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -116,10 +116,23 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt - name: Run service + env: + GOOGLE_PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }} + GOOGLE_PRIVATE_KEY_ID: ${{ secrets.PRIVATE_KEY_ID }} + DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres + SECRET_KEY: secret_key + VALKEY_HOST: localhost + VALKEY_PORT: 6379 + GH_TOKEN: token + REPO_ORG: https://github.com/canonical + JIRA_EMAIL: example@canonical.com + JIRA_TOKEN: jiratoken + JIRA_URL: https://example.atlassian.net + JIRA_LABELS: somelabel + JIRA_COPY_UPDATES_EPIC: WD-9999999 + GOOGLE_DRIVE_FOLDER_ID: folderid + COPYDOC_TEMPLATE_ID: templateid run: | - set -a - source .env - set +a talisker.gunicorn webapp.app:app --daemon --bind 0.0.0.0:8104 curl --head --fail --retry-delay 1 --retry 30 --retry-connrefused http://localhost:8104 @@ -150,6 +163,9 @@ jobs: run: DOCKER_BUILDKIT=1 docker build --tag websites-content-system . - name: Run image + env: + GOOGLE_PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }} + GOOGLE_PRIVATE_KEY_ID: ${{ secrets.PRIVATE_KEY_ID }} run: | docker run \ -p 8104:8104 \ @@ -158,12 +174,16 @@ jobs: -e VALKEY_PORT=6379 \ -e GH_TOKEN=token \ -e REPO_ORG=https://github.com/canonical \ - -e SQLALCHEMY_DATABASE_URI=postgresql://postgres:postgres@localhost:5432/postgres \ + -e DATABASE_URL=postgresql://postgres:postgres@localhost:5432/postgres \ -e JIRA_EMAIL=example@canonical.com \ -e JIRA_TOKEN=jiratoken \ -e JIRA_URL=https://example.atlassian.net \ -e JIRA_LABELS=somelabel \ -e JIRA_COPY_UPDATES_EPIC=WD-9999999 \ + -e GOOGLE_DRIVE_FOLDER_ID=folderid \ + -e COPYDOC_TEMPLATE_ID=templateid \ + -e GOOGLE_PRIVATE_KEY="$GOOGLE_PRIVATE_KEY" \ + -e GOOGLE_PRIVATE_KEY_ID="$GOOGLE_PRIVATE_KEY_ID" \ --network host \ websites-content-system & sleep 1 curl --head --fail --retry-delay 1 --retry 30 --retry-connrefused http://localhost diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 58807e88..3c440ecd 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -31,7 +31,16 @@ jobs: VALKEY_PORT: 6379 GH_TOKEN: token REPO_ORG: https://github.com/canonical - SQLALCHEMY_DATABASE_URI: postgresql://postgres:postgres@localhost:5432/postgres + DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres + GOOGLE_PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }} + GOOGLE_PRIVATE_KEY_ID: ${{ secrets.PRIVATE_KEY_ID }} + JIRA_EMAIL: example@canonical.com + JIRA_TOKEN: jiratoken + JIRA_URL: https://example.atlassian.net + JIRA_LABELS: somelabel + JIRA_COPY_UPDATES_EPIC: WD-9999999 + GOOGLE_DRIVE_FOLDER_ID: folderid + COPYDOC_TEMPLATE_ID: templateid steps: - uses: actions/checkout@v4 - name: Set up Python 3.12 diff --git a/README.md b/README.md index 9ddb4b91..ba4cd67b 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,8 @@ Backend service for the CMS template parser Before starting, update the environment variables if needed. The default values will work for docker, save the `GH_TOKEN` which must be manually set. You can create a token [here](https://github.com/settings/tokens), by following [these](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens) instructions. Make sure to select the `repo` scope for the token. +You will also require a credentials file for google drive. Please store it as credentials.json in the `credentials` directory. + ```env PORT=8104 FLASK_DEBUG=true @@ -16,6 +18,19 @@ DEVEL=True VALKEY_HOST=0.0.0.0 VALKEY_PORT=6379 GH_TOKEN=ghp_somepersonaltoken +REPO_ORG=https://github.com/canonical +DATABASE_URL=postgresql://postgres:postgres@localhost:5432/content +TASK_DELAY=30 +DIRECTORY_API_TOKEN=token +JIRA_EMAIL=example@canonical.com +JIRA_TOKEN=jiratoken +JIRA_URL=https://warthogs.atlassian.net +JIRA_LABELS=sites_BAU +JIRA_COPY_UPDATES_EPIC=WD-12643 +GOOGLE_DRIVE_FOLDER_ID=1EIFOGJ8DIWpsYIfWk7Yos3YijZIkbJDk +COPYD0C_TEMPLATE_ID=125auRsLQukYH-tKN1oEKaksmpCXd_DTGiswvmbeS2iA +GOOGLE_PRIVATE_KEY=base64encodedprivatekey +GOOGLE_PRIVATE_KEY_ID=privatekeyid ``` ### Using docker diff --git a/konf/site.yaml b/konf/site.yaml index c8644386..2661c95f 100644 --- a/konf/site.yaml +++ b/konf/site.yaml @@ -18,11 +18,6 @@ env: key: github name: webteam-valkey - - name: SECRET_KEY - secretKeyRef: - key: websites-content-system - name: secret-keys - - name: DIRECTORY_API_TOKEN secretKeyRef: key: token @@ -31,12 +26,12 @@ env: - name: JIRA_EMAIL secretKeyRef: key: jira-email - name: cms-jira + name: cs-canonical-com - name: JIRA_TOKEN secretKeyRef: key: jira-token - name: cms-jira + name: cs-canonical-com - name: JIRA_URL value: "https://warthogs.atlassian.net" @@ -47,12 +42,37 @@ env: - name: JIRA_COPY_UPDATES_EPIC value: "WD-12643" + - name: GOOGLE_DRIVE_FOLDER_ID + secretKeyRef: + key: google-drive-folder-id + name: cs-canonical-com + + - name: COPYDOC_TEMPLATE_ID + secretKeyRef: + key: copydoc-template-id + name: cs-canonical-com + + - name: GOOGLE_PRIVATE_KEY + secretKeyRef: + key: google-private-key + name: cs-canonical-com + + - name: GOOGLE_PRIVATE_KEY_ID + secretKeyRef: + key: google-private-key-id + name: cs-canonical-com + # Overrides for production production: replicas: 1 nginxConfigurationSnippet: | more_set_headers "X-Robots-Tag: noindex"; more_set_headers "Link: ; rel=preconnect; crossorigin, ; rel=preconnect"; + env: + - name: DATABASE_URL + secretKeyRef: + key: websites-content-system + name: database-urls # Overrides for staging staging: @@ -60,3 +80,70 @@ staging: nginxConfigurationSnippet: | more_set_headers "X-Robots-Tag: noindex"; more_set_headers "Link: ; rel=preconnect; crossorigin, ; rel=preconnect"; + env: + - name: DATABASE_URL + secretKeyRef: + key: websites-content-system + name: database-urls + +# Overrides for demos +demo: + env: + - name: VALKEY_HOST + secretKeyRef: + key: host + name: webteam-valkey + + - name: VALKEY_PORT + secretKeyRef: + key: port + name: webteam-valkey + + - name: GH_TOKEN + secretKeyRef: + key: github + name: webteam-valkey + + - name: DIRECTORY_API_TOKEN + secretKeyRef: + key: token + name: directory-api + + - name: JIRA_EMAIL + secretKeyRef: + key: jira-email + name: cs-canonical-com + + - name: JIRA_TOKEN + secretKeyRef: + key: jira-token + name: cs-canonical-com + + - name: JIRA_URL + value: "https://warthogs.atlassian.net" + + - name: JIRA_LABELS + value: "sites_BAU" + + - name: JIRA_COPY_UPDATES_EPIC + value: "WD-12643" + + - name: GOOGLE_DRIVE_FOLDER_ID + secretKeyRef: + key: google-drive-folder-id + name: cs-canonical-com + + - name: COPYDOC_TEMPLATE_ID + secretKeyRef: + key: copydoc-template-id + name: cs-canonical-com + + - name: GOOGLE_PRIVATE_KEY + secretKeyRef: + key: google-private-key + name: cs-canonical-com + + - name: GOOGLE_PRIVATE_KEY_ID + secretKeyRef: + key: google-private-key-id + name: cs-canonical-com \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 8b560e25..c25be6fc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ click==8.1.7 django-openid-auth==0.17 Flask==2.3.3 Flask-OpenID==1.3.1 +google-api-python-client==2.78.0 itsdangerous==2.2.0 Jinja2==3.1.4 MarkupSafe==2.1.5 diff --git a/webapp/__init__.py b/webapp/__init__.py index 420b84bd..ff7ccd50 100644 --- a/webapp/__init__.py +++ b/webapp/__init__.py @@ -2,6 +2,7 @@ from webapp.cache import init_cache from webapp.context import base_context +from webapp.gdrive import init_gdrive from webapp.jira import init_jira from webapp.models import init_db from webapp.sso import init_sso @@ -30,4 +31,7 @@ def create_app(): # Initialize JIRA init_jira(app) + # Initialize gdrive + init_gdrive(app) + return app diff --git a/webapp/app.py b/webapp/app.py index 1122caa3..93133ad9 100644 --- a/webapp/app.py +++ b/webapp/app.py @@ -135,11 +135,12 @@ def request_changes(body: ChangesRequestModel): # Make a request to JIRA to create a task try: - create_jira_task(app, body.model_dump()) + task = create_jira_task(app, body.model_dump()) + task_url = f"https://docs.google.com/document/d/{task['id']}" except Exception as e: return jsonify(str(e)), 500 - return jsonify("Task created successfully"), 201 + return jsonify({"message": f"Task created successfully\n{task_url}"}), 201 @app.route("/get-jira-tasks/", methods=["GET"]) diff --git a/webapp/gdrive.py b/webapp/gdrive.py new file mode 100644 index 00000000..c4ac77cd --- /dev/null +++ b/webapp/gdrive.py @@ -0,0 +1,162 @@ +import difflib + +from google.oauth2 import service_account +from googleapiclient.discovery import build +from googleapiclient.errors import HttpError + + +class GoogleDriveClient: + # If modifying these scopes, delete the file token.json. + SCOPES = ["https://www.googleapis.com/auth/drive"] + + def __init__( + self, credentials, drive_folder_id=None, copydoc_template_id=None + ): + self.credentials = self._get_credentials(credentials) + self.service = self._build_service() + self.GOOGLE_DRIVE_FOLDER_ID = drive_folder_id + self.COPYD0C_TEMPLATE_ID = copydoc_template_id + + def _get_credentials(self, credentials): + """Load credentials from an object.""" + + return service_account.Credentials.from_service_account_info( + credentials, + scopes=self.SCOPES, + ) + + def _build_service(self): + return build("drive", "v3", credentials=self.credentials) + + def _item_exists( + self, + folder_name, + parent=None, + mime_type="'application/vnd.google-apps.folder'", + ): + """ + Check whether an item exists in Google Drive. + """ + query = ( + f"name = '{folder_name}' and " + f"mimeType = {mime_type} and " + "trashed = false" + ) + if parent: + query += f" and '{parent}' in parents" + try: + results = ( + self.service.files() + .list( + q=query, + spaces="drive", + fields="files(id, name)", + pageSize=10, + ) + .execute() + ) + except HttpError as error: + raise ValueError(f"An error occurred: Query:{query} Error:{error}") + + if data := results.get("files"): + # Get the closest match to the folder name, if there are several + item_names = [item["name"] for item in data] + result = difflib.get_close_matches(folder_name, item_names)[0] + # Return the file id + result_id = next( + item["id"] for item in data if item["name"] == result + ) + return result_id + + def create_folder(self, name, parent): + """ + Create a folder in the Google Drive. + """ + try: + folder_metadata = { + "name": name, + "mimeType": "application/vnd.google-apps.folder", + "parents": [parent], + } + folder = ( + self.service.files() + .create(body=folder_metadata, fields="id") + .execute() + ) + return folder.get("id") + except HttpError as error: + raise ValueError( + f"An error occurred when creating a new folder: {error}" + ) + + def build_webpage_folder(self, webpage): + """ + Create a folder hierarchy in Google Drive for a webpage. + """ + folders = [f"/{f}" for f in webpage.url.split("/")[:-1] if f != ""] + # Check if the project folder exists, or create one + if not ( + parent := self._item_exists( + webpage.project.name, parent=self.GOOGLE_DRIVE_FOLDER_ID + ) + ): + parent = self.create_folder( + webpage.project.name, self.GOOGLE_DRIVE_FOLDER_ID + ) + + # Create subfolders + for folder in folders: + if not (folder_id := self._item_exists(folder, parent)): + folder_id = self.create_folder(folder, parent) + parent = folder_id + + # Return the last parent folder + return parent + + def copy_file(self, fileID, name, parents): + """ + Create a copydoc from a template. The document is created in the folder + for the webpage project. + """ + try: + copy_metadata = { + "name": name, + "parents": [parents], + "mimeType": "application/vnd.google-apps.file", + } + copy = ( + self.service.files() + .copy( + fileId=fileID, + body=copy_metadata, + ) + .execute() + ) + return copy + except HttpError as error: + raise ValueError( + f"An error occurred when copying copydoc template: {error}" + ) + + def create_copydoc_from_template(self, webpage): + """ + Create a copydoc from a template. The document is created in the folder + for the webpage project. + """ + # Create the folder hierarchy for the webpage + webpage_folder = self.build_webpage_folder(webpage) + + # Clone the template document to the new folder + return self.copy_file( + fileID=self.COPYD0C_TEMPLATE_ID, + name=webpage.url, + parents=webpage_folder, + ) + + +def init_gdrive(app): + app.config["gdrive"] = GoogleDriveClient( + credentials=app.config["GOOGLE_CREDENTIALS"], + drive_folder_id=app.config["GOOGLE_DRIVE_FOLDER_ID"], + copydoc_template_id=app.config["COPYD0C_TEMPLATE_ID"], + ) diff --git a/webapp/helper.py b/webapp/helper.py index 5abddce9..cf503a7f 100644 --- a/webapp/helper.py +++ b/webapp/helper.py @@ -1,4 +1,4 @@ -from webapp.models import JiraTask, User, db, get_or_create +from webapp.models import JiraTask, User, Webpage, db, get_or_create def get_or_create_user_id(user): @@ -43,3 +43,9 @@ def create_jira_task(app, task): webpage_id=task["webpage_id"], user_id=task["reporter_id"], ) + + # Create a new copydoc if the request is a new webpage + if task["type"] == jira.NEW_WEBPAGE: + webpage = Webpage.query.filter_by(id=task["webpage_id"]).first() + client = app.config["gdrive"] + return client.create_copydoc_from_template(webpage) diff --git a/webapp/jira.py b/webapp/jira.py index 3d8475bb..b7408181 100644 --- a/webapp/jira.py +++ b/webapp/jira.py @@ -62,6 +62,7 @@ def __request__( raise Exception( "Failed to make a request to Jira. Status code:" + f" {url} {method} {data} {params}" f" {response.status_code}. Response: {response.text}" ) @@ -76,7 +77,7 @@ def get_reporter_jira_id(self, user_id): str: The Jira ID of the user who reported the issue. """ # Try to get the user from the database - user = User.query.filter_by(id=user_id).first() + user = db.session.query(User).filter_by(id=user_id).first() if not user: raise ValueError(f"User with ID {user_id} not found") # If the user already has a Jira account ID, return it @@ -211,17 +212,19 @@ def create_issue( request_type == self.NEW_WEBPAGE or request_type == self.PAGE_REFRESH ): - parent = None # Create epic epic = self.create_task( summary=summary, issue_type=self.EPIC, description=description, - parent=parent, + parent=None, reporter_jira_id=reporter_jira_id, due_date=due_date, ) + if not epic: + raise Exception("Failed to create epic") + # Create subtasks for this epic for subtask_name in ["UX", "Visual", "Dev"]: self.create_task( @@ -234,14 +237,11 @@ def create_issue( ) return epic - elif request_type == self.COPY_UPDATE: - parent = self.copy_updates_epic - return self.create_task( summary=summary, issue_type=self.SUBTASK, description=description, - parent=parent, + parent=self.copy_updates_epic, reporter_jira_id=reporter_jira_id, due_date=due_date, ) diff --git a/webapp/settings.py b/webapp/settings.py index 61b6562f..f5d07e03 100644 --- a/webapp/settings.py +++ b/webapp/settings.py @@ -1,3 +1,4 @@ +import base64 import os from os import environ @@ -7,11 +8,26 @@ GH_TOKEN = environ.get("GH_TOKEN", "") SECRET_KEY = environ.get("SECRET_KEY") BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -SQLALCHEMY_DATABASE_URI = environ.get( - "SQLALCHEMY_DATABASE_URI", "sqlite:///project.db" -) +SQLALCHEMY_DATABASE_URI = environ.get("DATABASE_URL", "sqlite:///project.db") JIRA_EMAIL = environ.get("JIRA_EMAIL") JIRA_TOKEN = environ.get("JIRA_TOKEN") JIRA_URL = environ.get("JIRA_URL") JIRA_LABELS = environ.get("JIRA_LABELS") -JIRA_COPY_UPDATES_EPIC = environ.get("JIRA_COPY_UPDATES_EPIC") \ No newline at end of file +JIRA_COPY_UPDATES_EPIC = environ.get("JIRA_COPY_UPDATES_EPIC") +GOOGLE_DRIVE_FOLDER_ID = environ.get("GOOGLE_DRIVE_FOLDER_ID") +COPYD0C_TEMPLATE_ID = environ.get("COPYD0C_TEMPLATE_ID") +GOOGLE_CREDENTIALS = { + "type": "service_account", + "project_id": "web-engineering-436014", + "private_key_id": environ.get("GOOGLE_PRIVATE_KEY_ID"), + "private_key": base64.b64decode(environ.get("GOOGLE_PRIVATE_KEY")).replace( + b"\\n", b"\n" + ), + "client_email": "websites-copy-docs-627@web-engineering-436014.iam.gserviceaccount.com", # noqa: E501 + "client_id": "116847960229506342511", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/websites-copy-docs-627%40web-engineering-436014.iam.gserviceaccount.com", # noqa: E501 + "universe_domain": "googleapis.com", +}