From 24bd073aaf80528e51894557f4df2976dcc4f899 Mon Sep 17 00:00:00 2001 From: Olwe Samuel Date: Mon, 30 Sep 2024 11:44:54 +0300 Subject: [PATCH 01/26] Create new copydoc file if webpage is new --- webapp/gdrive.py | 113 +++++++++++++++++++++++++++++++++++++++++++++++ webapp/helper.py | 9 +++- 2 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 webapp/gdrive.py diff --git a/webapp/gdrive.py b/webapp/gdrive.py new file mode 100644 index 00000000..1adaac51 --- /dev/null +++ b/webapp/gdrive.py @@ -0,0 +1,113 @@ +import base64 +import os + +from google.oauth2 import service_account +from googleapiclient.discovery import build +from googleapiclient.errors import HttpError + + +class GoogleDriveClient: + # If modifying these scopes, delete the file token.json. + SCOPES = ["https://www.googleapis.com/auth/drive.metadata.readonly"] + SERVICE_ACCOUNT_FILE = "webapp/local-creds.json" + # GOOGLE_DRIVE_FOLDER_ID = ( + # "0B4s80tIYQW4BMjNiMGFmNzQtNDkxZC00YmQ0LWJiZWUtNTk2YThlY2MzZmJh" + # ) + # COPYD0C_TEMPLATE_ID = "1EPA_Ea8ShIvyftAc9oVxZYUIMHfAPFF6S5x6FOvLkwM" + GOOGLE_DRIVE_FOLDER_ID = "1EIFOGJ8DIWpsYIfWk7Yos3YijZIkbJDk" + COPYD0C_TEMPLATE_ID = "125auRsLQukYH-tKN1oEKaksmpCXd_DTGiswvmbeS2iA" + + def __init__(self): + self.credentials = self._get_credentials() + self.service = self._build_service() + + def _get_credentials(self): + """Load credentials from a base64 encoded environment variable.""" + credentials_text = os.getenv("GOOGLE_SERVICE_ACCOUNT") + with open("service_account.json", "w") as f: + f.write(base64.decode(credentials_text)) + + return service_account.Credentials.from_service_account_file( + self.SERVICE_ACCOUNT_FILE, + scopes=self.SCOPES, + ) + + def _build_service(self): + return build("drive", "v3", credentials=self.credentials) + + def create_copydoc_from_template(self, webpage): + """ + Create a copydoc from a template. The document is created in the + """ + # Create a folder if it does not exist + self.create_folder(webpage.project.name) + try: + copy_metadata = { + "name": webpage.url, + "parents": [self.GOOGLE_DRIVE_FOLDER_ID], + } + copy = ( + self.service.files() + .copy( + fileId=self.COPYD0C_TEMPLATE_ID, + body=copy_metadata, + ) + .execute() + ) + return copy + except HttpError as error: + print(f"An error occurred: {error}") + return None + + def create_folder(self, name): + """ + Create a folder in the Google Drive. + """ + try: + folder_metadata = { + "name": name, + "mimeType": "application/vnd.google-apps.folder", + "parents": [self.GOOGLE_DRIVE_FOLDER_ID], + } + folder = ( + self.service.files() + .create(body=folder_metadata, fields="id") + .execute() + ) + return folder + except HttpError as error: + print(f"An error occurred: {error}") + return None + + def find_folder(service, folder_name): + """ + Check if a folder with the given name exists in Google Drive, to prevent + creating duplicate folders. + """ + query = f"name = '{folder_name}' and mimeType = 'application/vnd.google-apps.folder' and trashed = false" + results = ( + service.files() + .list( + q=query, spaces="drive", fields="files(id, name)", pageSize=10 + ) + .execute() + ) + + items = results.get("files", []) + return items + + def list_files(self, page_size=10): + try: + results = ( + self.service.files() + .list( + pageSize=page_size, + fields="nextPageToken, files(id, name)", + ) + .execute() + ) + items = results.get("files", []) + return items + except HttpError as error: + print(f"An error occurred: {error}") + return None diff --git a/webapp/helper.py b/webapp/helper.py index 5abddce9..279517d3 100644 --- a/webapp/helper.py +++ b/webapp/helper.py @@ -1,4 +1,5 @@ -from webapp.models import JiraTask, User, db, get_or_create +from webapp.gdrive import GoogleDriveClient +from webapp.models import JiraTask, User, Webpage, db, get_or_create def get_or_create_user_id(user): @@ -43,3 +44,9 @@ def create_jira_task(app, task): webpage_id=task["webpage_id"], user_id=task["reporter_id"], ) + + # Create a new copydoc if the request is a new webpage + if task["type"] == jira.NEW_WEBPAGE: + webpage = Webpage.query.filter_by(id=task["webpage_id"]).first() + client = GoogleDriveClient() + client.create_copydoc_from_template(webpage) From 1cdd670224f7952b0c76c6e3efe5ca0b6c17d7f1 Mon Sep 17 00:00:00 2001 From: Olwe Samuel Date: Mon, 30 Sep 2024 11:55:26 +0300 Subject: [PATCH 02/26] conditionally create a new folder --- webapp/gdrive.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/webapp/gdrive.py b/webapp/gdrive.py index 1adaac51..62cab20e 100644 --- a/webapp/gdrive.py +++ b/webapp/gdrive.py @@ -40,7 +40,8 @@ def create_copydoc_from_template(self, webpage): Create a copydoc from a template. The document is created in the """ # Create a folder if it does not exist - self.create_folder(webpage.project.name) + if not self.find_folder(webpage.project.name): + self.create_folder(webpage.project.name) try: copy_metadata = { "name": webpage.url, From d5a58aa37715eb7fa33a225136f1700d46805011 Mon Sep 17 00:00:00 2001 From: Olwe Samuel Date: Mon, 30 Sep 2024 12:13:41 +0300 Subject: [PATCH 03/26] Fixed linter issues --- webapp/gdrive.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/webapp/gdrive.py b/webapp/gdrive.py index 62cab20e..b38b7867 100644 --- a/webapp/gdrive.py +++ b/webapp/gdrive.py @@ -82,10 +82,13 @@ def create_folder(self, name): def find_folder(service, folder_name): """ - Check if a folder with the given name exists in Google Drive, to prevent - creating duplicate folders. + Check if a folder with the given name exists in Google Drive, to + prevent creating duplicate folders. """ - query = f"name = '{folder_name}' and mimeType = 'application/vnd.google-apps.folder' and trashed = false" + query = ( + f"name = '{folder_name}' and mimeType = " + "'application/vnd.google-apps.folder' and trashed = false" + ) results = ( service.files() .list( From dc95eeade630275ac7cd19760f0450b291d5a303 Mon Sep 17 00:00:00 2001 From: Olwe Samuel Date: Mon, 30 Sep 2024 12:18:24 +0300 Subject: [PATCH 04/26] Updated requirements --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 8b560e25..c25be6fc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ click==8.1.7 django-openid-auth==0.17 Flask==2.3.3 Flask-OpenID==1.3.1 +google-api-python-client==2.78.0 itsdangerous==2.2.0 Jinja2==3.1.4 MarkupSafe==2.1.5 From f09f131338d0d1868105d0040a65ca66f1ee9235 Mon Sep 17 00:00:00 2001 From: Olwe Samuel Date: Mon, 30 Sep 2024 12:27:46 +0300 Subject: [PATCH 05/26] Put copydoc in subfolder --- webapp/gdrive.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/webapp/gdrive.py b/webapp/gdrive.py index b38b7867..d1ead075 100644 --- a/webapp/gdrive.py +++ b/webapp/gdrive.py @@ -39,13 +39,14 @@ def create_copydoc_from_template(self, webpage): """ Create a copydoc from a template. The document is created in the """ + parents = [self.GOOGLE_DRIVE_FOLDER_ID] # Create a folder if it does not exist if not self.find_folder(webpage.project.name): - self.create_folder(webpage.project.name) + parents = [self.create_folder(webpage.project.name)] try: copy_metadata = { "name": webpage.url, - "parents": [self.GOOGLE_DRIVE_FOLDER_ID], + "parents": parents, } copy = ( self.service.files() From 5488028f6ea53132a192f054b09095a83e3b50aa Mon Sep 17 00:00:00 2001 From: Olwe Samuel Date: Mon, 30 Sep 2024 12:29:23 +0300 Subject: [PATCH 06/26] Use hardcoded values for the google drive folder and copydoc template --- webapp/gdrive.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/webapp/gdrive.py b/webapp/gdrive.py index d1ead075..1abb9077 100644 --- a/webapp/gdrive.py +++ b/webapp/gdrive.py @@ -10,12 +10,10 @@ class GoogleDriveClient: # If modifying these scopes, delete the file token.json. SCOPES = ["https://www.googleapis.com/auth/drive.metadata.readonly"] SERVICE_ACCOUNT_FILE = "webapp/local-creds.json" - # GOOGLE_DRIVE_FOLDER_ID = ( - # "0B4s80tIYQW4BMjNiMGFmNzQtNDkxZC00YmQ0LWJiZWUtNTk2YThlY2MzZmJh" - # ) - # COPYD0C_TEMPLATE_ID = "1EPA_Ea8ShIvyftAc9oVxZYUIMHfAPFF6S5x6FOvLkwM" - GOOGLE_DRIVE_FOLDER_ID = "1EIFOGJ8DIWpsYIfWk7Yos3YijZIkbJDk" - COPYD0C_TEMPLATE_ID = "125auRsLQukYH-tKN1oEKaksmpCXd_DTGiswvmbeS2iA" + GOOGLE_DRIVE_FOLDER_ID = ( + "0B4s80tIYQW4BMjNiMGFmNzQtNDkxZC00YmQ0LWJiZWUtNTk2YThlY2MzZmJh" + ) + COPYD0C_TEMPLATE_ID = "1EPA_Ea8ShIvyftAc9oVxZYUIMHfAPFF6S5x6FOvLkwM" def __init__(self): self.credentials = self._get_credentials() From db773b6a9e7bfd7241c58fedd89ce969a7abc0de Mon Sep 17 00:00:00 2001 From: Olwe Samuel Date: Mon, 30 Sep 2024 15:45:04 +0300 Subject: [PATCH 07/26] Store credentials in a temp file --- webapp/gdrive.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/webapp/gdrive.py b/webapp/gdrive.py index 1abb9077..3ed64be3 100644 --- a/webapp/gdrive.py +++ b/webapp/gdrive.py @@ -1,5 +1,6 @@ import base64 import os +import tempfile from google.oauth2 import service_account from googleapiclient.discovery import build @@ -8,8 +9,10 @@ class GoogleDriveClient: # If modifying these scopes, delete the file token.json. - SCOPES = ["https://www.googleapis.com/auth/drive.metadata.readonly"] - SERVICE_ACCOUNT_FILE = "webapp/local-creds.json" + SCOPES = [ + "https://www.googleapis.com/auth/drive.metadata.readonly", + "https://www.googleapis.com/auth/drive.file", + ] GOOGLE_DRIVE_FOLDER_ID = ( "0B4s80tIYQW4BMjNiMGFmNzQtNDkxZC00YmQ0LWJiZWUtNTk2YThlY2MzZmJh" ) @@ -20,22 +23,26 @@ def __init__(self): self.service = self._build_service() def _get_credentials(self): - """Load credentials from a base64 encoded environment variable.""" + """ + Load credentials from a base64 encoded environment variable. + """ credentials_text = os.getenv("GOOGLE_SERVICE_ACCOUNT") - with open("service_account.json", "w") as f: + with tempfile.NamedTemporaryFile(delete_on_close=False) as f: f.write(base64.decode(credentials_text)) + f.close() - return service_account.Credentials.from_service_account_file( - self.SERVICE_ACCOUNT_FILE, - scopes=self.SCOPES, - ) + return service_account.Credentials.from_service_account_file( + f.name, + scopes=self.SCOPES, + ) def _build_service(self): return build("drive", "v3", credentials=self.credentials) def create_copydoc_from_template(self, webpage): """ - Create a copydoc from a template. The document is created in the + Create a copydoc from a template. The document is created in the folder + for the webpage project. """ parents = [self.GOOGLE_DRIVE_FOLDER_ID] # Create a folder if it does not exist From afaee116870b239294dd68b5f51559ccbfbc0e09 Mon Sep 17 00:00:00 2001 From: Olwe Samuel Date: Tue, 1 Oct 2024 11:47:35 +0300 Subject: [PATCH 08/26] Moved google creds to env variable --- .env | 3 +++ konf/site.yaml | 15 +++++++++++++++ webapp/gdrive.py | 6 ++---- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/.env b/.env index 8ec94291..7633e3a6 100644 --- a/.env +++ b/.env @@ -12,3 +12,6 @@ JIRA_TOKEN=token JIRA_URL=https://warthogs.atlassian.net JIRA_LABELS=sites_BAU JIRA_COPY_UPDATES_EPIC=KAN-1 +GOOGLE_CREDENTIALS=googlecreds +GOOGLE_DRIVE_FOLDER_ID=googlecreds +COPYD0C_TEMPLATE_ID=googlecreds diff --git a/konf/site.yaml b/konf/site.yaml index c8644386..6d9d602d 100644 --- a/konf/site.yaml +++ b/konf/site.yaml @@ -46,6 +46,21 @@ env: - name: JIRA_COPY_UPDATES_EPIC value: "WD-12643" + + - name: GOOGLE_SERVICE_ACCOUNT + secretKeyRef: + key: google-service-account + name: cms-jira + + - name: GOOGLE_DRIVE_FOLDER_ID + secretKeyRef: + key: google-drive-folder-id + name: cms-jira + + - name: COPYDOC_TEMPLATE_ID + secretKeyRef: + key: copydoc-template-id + name: cms-jira # Overrides for production production: diff --git a/webapp/gdrive.py b/webapp/gdrive.py index 3ed64be3..14a04315 100644 --- a/webapp/gdrive.py +++ b/webapp/gdrive.py @@ -13,14 +13,12 @@ class GoogleDriveClient: "https://www.googleapis.com/auth/drive.metadata.readonly", "https://www.googleapis.com/auth/drive.file", ] - GOOGLE_DRIVE_FOLDER_ID = ( - "0B4s80tIYQW4BMjNiMGFmNzQtNDkxZC00YmQ0LWJiZWUtNTk2YThlY2MzZmJh" - ) - COPYD0C_TEMPLATE_ID = "1EPA_Ea8ShIvyftAc9oVxZYUIMHfAPFF6S5x6FOvLkwM" def __init__(self): self.credentials = self._get_credentials() self.service = self._build_service() + self.GOOGLE_DRIVE_FOLDER_ID = os.getenv("GOOGLE_DRIVE_FOLDER_ID") + self.COPYD0C_TEMPLATE_ID = os.getenv("COPYD0C_TEMPLATE_ID") def _get_credentials(self): """ From c8246a6cb55f0e304c44451e957a72887489c4f3 Mon Sep 17 00:00:00 2001 From: Olwe Samuel Date: Tue, 1 Oct 2024 11:56:47 +0300 Subject: [PATCH 09/26] Use demo database URL --- webapp/settings.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/webapp/settings.py b/webapp/settings.py index 61b6562f..9f67957c 100644 --- a/webapp/settings.py +++ b/webapp/settings.py @@ -7,9 +7,7 @@ GH_TOKEN = environ.get("GH_TOKEN", "") SECRET_KEY = environ.get("SECRET_KEY") BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -SQLALCHEMY_DATABASE_URI = environ.get( - "SQLALCHEMY_DATABASE_URI", "sqlite:///project.db" -) +SQLALCHEMY_DATABASE_URI = environ.get("DATABASE_URL", "sqlite:///project.db") JIRA_EMAIL = environ.get("JIRA_EMAIL") JIRA_TOKEN = environ.get("JIRA_TOKEN") JIRA_URL = environ.get("JIRA_URL") From 4311ff9e4e938e08adca5b87fe1fbeb84e712ac2 Mon Sep 17 00:00:00 2001 From: Olwe Samuel Date: Tue, 1 Oct 2024 12:27:27 +0300 Subject: [PATCH 10/26] Reformat site.yaml --- konf/site.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/konf/site.yaml b/konf/site.yaml index 6d9d602d..237005d6 100644 --- a/konf/site.yaml +++ b/konf/site.yaml @@ -46,17 +46,17 @@ env: - name: JIRA_COPY_UPDATES_EPIC value: "WD-12643" - + - name: GOOGLE_SERVICE_ACCOUNT secretKeyRef: key: google-service-account name: cms-jira - + - name: GOOGLE_DRIVE_FOLDER_ID secretKeyRef: key: google-drive-folder-id name: cms-jira - + - name: COPYDOC_TEMPLATE_ID secretKeyRef: key: copydoc-template-id From faf111b807219f8b4d699a9f619475a63274ed7f Mon Sep 17 00:00:00 2001 From: Olwe Samuel Date: Tue, 1 Oct 2024 13:03:31 +0300 Subject: [PATCH 11/26] Point production to dedicated db --- konf/site.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/konf/site.yaml b/konf/site.yaml index 237005d6..52baca52 100644 --- a/konf/site.yaml +++ b/konf/site.yaml @@ -68,6 +68,11 @@ production: nginxConfigurationSnippet: | more_set_headers "X-Robots-Tag: noindex"; more_set_headers "Link: ; rel=preconnect; crossorigin, ; rel=preconnect"; + env: + - name: DATABASE_URL + secretKeyRef: + key: websites-content-system + name: database-urls # Overrides for staging staging: @@ -75,3 +80,8 @@ staging: nginxConfigurationSnippet: | more_set_headers "X-Robots-Tag: noindex"; more_set_headers "Link: ; rel=preconnect; crossorigin, ; rel=preconnect"; + env: + - name: DATABASE_URL + secretKeyRef: + key: websites-content-system + name: database-urls From ee921a34b74ffdbd298d1a848bad04c20b810a74 Mon Sep 17 00:00:00 2001 From: Olwe Samuel Date: Wed, 2 Oct 2024 10:28:51 +0300 Subject: [PATCH 12/26] Removed secret key definition --- konf/site.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/konf/site.yaml b/konf/site.yaml index 52baca52..cfa88307 100644 --- a/konf/site.yaml +++ b/konf/site.yaml @@ -18,11 +18,6 @@ env: key: github name: webteam-valkey - - name: SECRET_KEY - secretKeyRef: - key: websites-content-system - name: secret-keys - - name: DIRECTORY_API_TOKEN secretKeyRef: key: token From 02682bb5ade17962b729929030c93bc11c4bff10 Mon Sep 17 00:00:00 2001 From: Olwe Samuel Date: Wed, 2 Oct 2024 10:47:37 +0300 Subject: [PATCH 13/26] Updated demo secrets --- konf/site.yaml | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/konf/site.yaml b/konf/site.yaml index cfa88307..0a30491f 100644 --- a/konf/site.yaml +++ b/konf/site.yaml @@ -80,3 +80,60 @@ staging: secretKeyRef: key: websites-content-system name: database-urls + +# Overrides for demos +demo: + env: + - name: VALKEY_HOST + secretKeyRef: + key: host + name: webteam-valkey + + - name: VALKEY_PORT + secretKeyRef: + key: port + name: webteam-valkey + + - name: GH_TOKEN + secretKeyRef: + key: github + name: webteam-valkey + + - name: DIRECTORY_API_TOKEN + secretKeyRef: + key: token + name: directory-api + + - name: JIRA_EMAIL + secretKeyRef: + key: jira-email + name: cms-jira + + - name: JIRA_TOKEN + secretKeyRef: + key: jira-token + name: cms-jira + + - name: JIRA_URL + value: "https://warthogs.atlassian.net" + + - name: JIRA_LABELS + value: "sites_BAU" + + - name: JIRA_COPY_UPDATES_EPIC + value: "WD-12643" + + - name: GOOGLE_SERVICE_ACCOUNT + secretKeyRef: + key: google-service-account + name: cms-jira + + - name: GOOGLE_DRIVE_FOLDER_ID + secretKeyRef: + key: google-drive-folder-id + name: cms-jira + + - name: COPYDOC_TEMPLATE_ID + secretKeyRef: + key: copydoc-template-id + name: cms-jira \ No newline at end of file From 33a37bea7f9ceb6b5a171d50d65e78c32068203e Mon Sep 17 00:00:00 2001 From: Olwe Samuel Date: Wed, 2 Oct 2024 10:52:03 +0300 Subject: [PATCH 14/26] Moved drive env variables to settings --- webapp/gdrive.py | 14 +++++++------- webapp/helper.py | 6 +++++- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/webapp/gdrive.py b/webapp/gdrive.py index 14a04315..d6c57b96 100644 --- a/webapp/gdrive.py +++ b/webapp/gdrive.py @@ -1,5 +1,4 @@ import base64 -import os import tempfile from google.oauth2 import service_account @@ -14,17 +13,18 @@ class GoogleDriveClient: "https://www.googleapis.com/auth/drive.file", ] - def __init__(self): - self.credentials = self._get_credentials() + def __init__( + self, credentials=None, drive_folder_id=None, copydoc_template_id=None + ): + self.credentials = self._get_credentials(credentials_text=credentials) self.service = self._build_service() - self.GOOGLE_DRIVE_FOLDER_ID = os.getenv("GOOGLE_DRIVE_FOLDER_ID") - self.COPYD0C_TEMPLATE_ID = os.getenv("COPYD0C_TEMPLATE_ID") + self.GOOGLE_DRIVE_FOLDER_ID = drive_folder_id + self.COPYD0C_TEMPLATE_ID = copydoc_template_id - def _get_credentials(self): + def _get_credentials(self, credentials_text=None): """ Load credentials from a base64 encoded environment variable. """ - credentials_text = os.getenv("GOOGLE_SERVICE_ACCOUNT") with tempfile.NamedTemporaryFile(delete_on_close=False) as f: f.write(base64.decode(credentials_text)) f.close() diff --git a/webapp/helper.py b/webapp/helper.py index 279517d3..c4ee78ae 100644 --- a/webapp/helper.py +++ b/webapp/helper.py @@ -48,5 +48,9 @@ def create_jira_task(app, task): # Create a new copydoc if the request is a new webpage if task["type"] == jira.NEW_WEBPAGE: webpage = Webpage.query.filter_by(id=task["webpage_id"]).first() - client = GoogleDriveClient() + client = GoogleDriveClient( + credentials=app.config["GOOGLE_SERVICE_ACCOUNT"], + drive_folder_id=app.config["GOOGLE_DRIVE_FOLDER_ID"], + copydoc_template_id=app.config["COPYD0C_TEMPLATE_ID"], + ) client.create_copydoc_from_template(webpage) From 3f22c42ac3a9f932b2000700457397094abb3300 Mon Sep 17 00:00:00 2001 From: Olwe Samuel Date: Wed, 2 Oct 2024 11:37:45 +0300 Subject: [PATCH 15/26] Removed redundant code --- webapp/gdrive.py | 16 ---------------- webapp/settings.py | 5 ++++- 2 files changed, 4 insertions(+), 17 deletions(-) diff --git a/webapp/gdrive.py b/webapp/gdrive.py index d6c57b96..dc55b565 100644 --- a/webapp/gdrive.py +++ b/webapp/gdrive.py @@ -103,19 +103,3 @@ def find_folder(service, folder_name): items = results.get("files", []) return items - - def list_files(self, page_size=10): - try: - results = ( - self.service.files() - .list( - pageSize=page_size, - fields="nextPageToken, files(id, name)", - ) - .execute() - ) - items = results.get("files", []) - return items - except HttpError as error: - print(f"An error occurred: {error}") - return None diff --git a/webapp/settings.py b/webapp/settings.py index 9f67957c..38fa24cb 100644 --- a/webapp/settings.py +++ b/webapp/settings.py @@ -12,4 +12,7 @@ JIRA_TOKEN = environ.get("JIRA_TOKEN") JIRA_URL = environ.get("JIRA_URL") JIRA_LABELS = environ.get("JIRA_LABELS") -JIRA_COPY_UPDATES_EPIC = environ.get("JIRA_COPY_UPDATES_EPIC") \ No newline at end of file +JIRA_COPY_UPDATES_EPIC = environ.get("JIRA_COPY_UPDATES_EPIC") +GOOGLE_DRIVE_FOLDER_ID = environ.get("GOOGLE_DRIVE_FOLDER_ID") +COPYD0C_TEMPLATE_ID = environ.get("COPYD0C_TEMPLATE_ID") +GOOGLE_SERVICE_ACCOUNT = environ.get("GOOGLE_SERVICE_ACCOUNT") \ No newline at end of file From 1ee286da574dcd5cf32dcba8ee662bea2db7b8c1 Mon Sep 17 00:00:00 2001 From: Olwe Samuel Date: Mon, 7 Oct 2024 11:21:07 +0300 Subject: [PATCH 16/26] Updated google drive module --- README.md | 12 ++++ webapp/__init__.py | 4 ++ webapp/gdrive.py | 155 ++++++++++++++++++++++++++++++++------------- webapp/helper.py | 7 +- webapp/jira.py | 14 ++-- webapp/models.py | 2 + 6 files changed, 136 insertions(+), 58 deletions(-) diff --git a/README.md b/README.md index 9ddb4b91..12631077 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,18 @@ DEVEL=True VALKEY_HOST=0.0.0.0 VALKEY_PORT=6379 GH_TOKEN=ghp_somepersonaltoken +REPO_ORG=https://github.com/canonical +DATABASE_URL=postgresql://postgres:postgres@localhost:5432/content +TASK_DELAY=30 +DIRECTORY_API_TOKEN=token +JIRA_EMAIL=example@canonical.com +JIRA_TOKEN=jiratoken +JIRA_URL=https://warthogs.atlassian.net +JIRA_LABELS=sites_BAU +JIRA_COPY_UPDATES_EPIC=WD-12643 +GOOGLE_SERVICE_ACCOUNT=base64encodedjsonfile +GOOGLE_DRIVE_FOLDER_ID=1EIFOGJ8DIWpsYIfWk7Yos3YijZIkbJDk +COPYD0C_TEMPLATE_ID=125auRsLQukYH-tKN1oEKaksmpCXd_DTGiswvmbeS2iA ``` ### Using docker diff --git a/webapp/__init__.py b/webapp/__init__.py index 420b84bd..ff7ccd50 100644 --- a/webapp/__init__.py +++ b/webapp/__init__.py @@ -2,6 +2,7 @@ from webapp.cache import init_cache from webapp.context import base_context +from webapp.gdrive import init_gdrive from webapp.jira import init_jira from webapp.models import init_db from webapp.sso import init_sso @@ -30,4 +31,7 @@ def create_app(): # Initialize JIRA init_jira(app) + # Initialize gdrive + init_gdrive(app) + return app diff --git a/webapp/gdrive.py b/webapp/gdrive.py index dc55b565..eac1bcd9 100644 --- a/webapp/gdrive.py +++ b/webapp/gdrive.py @@ -1,4 +1,5 @@ import base64 +import difflib import tempfile from google.oauth2 import service_account @@ -8,10 +9,7 @@ class GoogleDriveClient: # If modifying these scopes, delete the file token.json. - SCOPES = [ - "https://www.googleapis.com/auth/drive.metadata.readonly", - "https://www.googleapis.com/auth/drive.file", - ] + SCOPES = ["https://www.googleapis.com/auth/drive"] def __init__( self, credentials=None, drive_folder_id=None, copydoc_template_id=None @@ -25,46 +23,59 @@ def _get_credentials(self, credentials_text=None): """ Load credentials from a base64 encoded environment variable. """ - with tempfile.NamedTemporaryFile(delete_on_close=False) as f: - f.write(base64.decode(credentials_text)) - f.close() + with tempfile.NamedTemporaryFile() as f: + data = base64.b64decode(credentials_text).replace(b"\\n", b"\n") + f.write(data) return service_account.Credentials.from_service_account_file( - f.name, + "webapp/credentials.json", scopes=self.SCOPES, ) def _build_service(self): return build("drive", "v3", credentials=self.credentials) - def create_copydoc_from_template(self, webpage): + def _item_exists( + self, + folder_name, + parent=None, + mime_type="'application/vnd.google-apps.folder'", + ): """ - Create a copydoc from a template. The document is created in the folder - for the webpage project. + Check whether an item exists in Google Drive. """ - parents = [self.GOOGLE_DRIVE_FOLDER_ID] - # Create a folder if it does not exist - if not self.find_folder(webpage.project.name): - parents = [self.create_folder(webpage.project.name)] + query = ( + f"name = '{folder_name}' and " + f"mimeType = {mime_type} and " + "trashed = false" + ) + if parent: + query += f" and '{parent}' in parents" try: - copy_metadata = { - "name": webpage.url, - "parents": parents, - } - copy = ( + results = ( self.service.files() - .copy( - fileId=self.COPYD0C_TEMPLATE_ID, - body=copy_metadata, + .list( + q=query, + spaces="drive", + fields="files(id, name)", + pageSize=10, ) .execute() ) - return copy except HttpError as error: - print(f"An error occurred: {error}") - return None + raise ValueError(f"An error occurred: Query:{query} Error:{error}") - def create_folder(self, name): + if data := results.get("files"): + # Get the closest match to the folder name, if there are several + item_names = [item["name"] for item in data] + result = difflib.get_close_matches(folder_name, item_names)[0] + # Return the file id + result_id = next( + item["id"] for item in data if item["name"] == result + ) + return result_id + + def create_folder(self, name, parent): """ Create a folder in the Google Drive. """ @@ -72,34 +83,88 @@ def create_folder(self, name): folder_metadata = { "name": name, "mimeType": "application/vnd.google-apps.folder", - "parents": [self.GOOGLE_DRIVE_FOLDER_ID], + "parents": [parent], } folder = ( self.service.files() .create(body=folder_metadata, fields="id") .execute() ) - return folder + return folder.get("id") except HttpError as error: - print(f"An error occurred: {error}") - return None + raise ValueError( + f"An error occurred when creating a new folder: {error}" + ) - def find_folder(service, folder_name): + def build_webpage_folder(self, webpage): """ - Check if a folder with the given name exists in Google Drive, to - prevent creating duplicate folders. + Create a folder hierarchy in Google Drive for a webpage. """ - query = ( - f"name = '{folder_name}' and mimeType = " - "'application/vnd.google-apps.folder' and trashed = false" - ) - results = ( - service.files() - .list( - q=query, spaces="drive", fields="files(id, name)", pageSize=10 + folders = webpage.url.split("/")[:-1] + # Check if the project folder exists, or create one + if not ( + parent := self._item_exists( + webpage.project.name, parent=self.GOOGLE_DRIVE_FOLDER_ID + ) + ): + parent = self.create_folder( + webpage.project.name, self.GOOGLE_DRIVE_FOLDER_ID ) - .execute() + + # Create subfolders + for folder in folders: + if folder != "": + folder_id = self.create_folder(folder, parent) + parent = folder_id + + # Return the last parent folder + return parent + + def copy_file(self, fileID, name, parents): + """ + Create a copydoc from a template. The document is created in the folder + for the webpage project. + """ + try: + copy_metadata = { + "name": name, + "parents": [parents], + "mimeType": "application/vnd.google-apps.file", + } + copy = ( + self.service.files() + .copy( + fileId=fileID, + body=copy_metadata, + ) + .execute() + ) + return copy + except HttpError as error: + raise ValueError( + f"An error occurred when copying copydoc template: {error}" + ) + + def create_copydoc_from_template(self, webpage): + """ + Create a copydoc from a template. The document is created in the folder + for the webpage project. + """ + # Create the folder hierarchy for the webpage + webpage_folder = self.build_webpage_folder(webpage) + + # Clone the template document to the new folder + doc_name = f'{webpage.url} - "{webpage.title}"' + self.copy_file( + fileID=self.COPYD0C_TEMPLATE_ID, + name=doc_name, + parents=webpage_folder, ) - items = results.get("files", []) - return items + +def init_gdrive(app): + app.config["gdrive"] = GoogleDriveClient( + credentials=app.config["GOOGLE_SERVICE_ACCOUNT"], + drive_folder_id=app.config["GOOGLE_DRIVE_FOLDER_ID"], + copydoc_template_id=app.config["COPYD0C_TEMPLATE_ID"], + ) diff --git a/webapp/helper.py b/webapp/helper.py index c4ee78ae..f36a4db4 100644 --- a/webapp/helper.py +++ b/webapp/helper.py @@ -1,4 +1,3 @@ -from webapp.gdrive import GoogleDriveClient from webapp.models import JiraTask, User, Webpage, db, get_or_create @@ -48,9 +47,5 @@ def create_jira_task(app, task): # Create a new copydoc if the request is a new webpage if task["type"] == jira.NEW_WEBPAGE: webpage = Webpage.query.filter_by(id=task["webpage_id"]).first() - client = GoogleDriveClient( - credentials=app.config["GOOGLE_SERVICE_ACCOUNT"], - drive_folder_id=app.config["GOOGLE_DRIVE_FOLDER_ID"], - copydoc_template_id=app.config["COPYD0C_TEMPLATE_ID"], - ) + client = app.config["gdrive"] client.create_copydoc_from_template(webpage) diff --git a/webapp/jira.py b/webapp/jira.py index 3d8475bb..b7408181 100644 --- a/webapp/jira.py +++ b/webapp/jira.py @@ -62,6 +62,7 @@ def __request__( raise Exception( "Failed to make a request to Jira. Status code:" + f" {url} {method} {data} {params}" f" {response.status_code}. Response: {response.text}" ) @@ -76,7 +77,7 @@ def get_reporter_jira_id(self, user_id): str: The Jira ID of the user who reported the issue. """ # Try to get the user from the database - user = User.query.filter_by(id=user_id).first() + user = db.session.query(User).filter_by(id=user_id).first() if not user: raise ValueError(f"User with ID {user_id} not found") # If the user already has a Jira account ID, return it @@ -211,17 +212,19 @@ def create_issue( request_type == self.NEW_WEBPAGE or request_type == self.PAGE_REFRESH ): - parent = None # Create epic epic = self.create_task( summary=summary, issue_type=self.EPIC, description=description, - parent=parent, + parent=None, reporter_jira_id=reporter_jira_id, due_date=due_date, ) + if not epic: + raise Exception("Failed to create epic") + # Create subtasks for this epic for subtask_name in ["UX", "Visual", "Dev"]: self.create_task( @@ -234,14 +237,11 @@ def create_issue( ) return epic - elif request_type == self.COPY_UPDATE: - parent = self.copy_updates_epic - return self.create_task( summary=summary, issue_type=self.SUBTASK, description=description, - parent=parent, + parent=self.copy_updates_epic, reporter_jira_id=reporter_jira_id, due_date=due_date, ) diff --git a/webapp/models.py b/webapp/models.py index e44ab5f6..c05cfd92 100644 --- a/webapp/models.py +++ b/webapp/models.py @@ -113,6 +113,7 @@ class Reviewer(db.Model, DateTimeMixin): user = relationship("User", back_populates="reviewers") webpages = relationship("Webpage", back_populates="reviewers") + class JIRATaskStatus: TRIAGED = "TRIAGED" UNTRIAGED = "UNTRIAGED" @@ -122,6 +123,7 @@ class JIRATaskStatus: DONE = "DONE" REJECTED = "REJECTED" + class JiraTask(db.Model, DateTimeMixin): __tablename__ = "jira_tasks" From a0047b271a665c74b079025021e7dd5e38a21ad0 Mon Sep 17 00:00:00 2001 From: Olwe Samuel Date: Mon, 7 Oct 2024 11:31:11 +0300 Subject: [PATCH 17/26] Use physical creds --- README.md | 3 ++- webapp/gdrive.py | 27 ++++++++++----------------- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 12631077..26a6c6d7 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,8 @@ Backend service for the CMS template parser Before starting, update the environment variables if needed. The default values will work for docker, save the `GH_TOKEN` which must be manually set. You can create a token [here](https://github.com/settings/tokens), by following [these](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens) instructions. Make sure to select the `repo` scope for the token. +You will also require a credentials file for google drive. Please store it as credentials.json in the parent directory. + ```env PORT=8104 FLASK_DEBUG=true @@ -25,7 +27,6 @@ JIRA_TOKEN=jiratoken JIRA_URL=https://warthogs.atlassian.net JIRA_LABELS=sites_BAU JIRA_COPY_UPDATES_EPIC=WD-12643 -GOOGLE_SERVICE_ACCOUNT=base64encodedjsonfile GOOGLE_DRIVE_FOLDER_ID=1EIFOGJ8DIWpsYIfWk7Yos3YijZIkbJDk COPYD0C_TEMPLATE_ID=125auRsLQukYH-tKN1oEKaksmpCXd_DTGiswvmbeS2iA ``` diff --git a/webapp/gdrive.py b/webapp/gdrive.py index eac1bcd9..44e0a99c 100644 --- a/webapp/gdrive.py +++ b/webapp/gdrive.py @@ -1,36 +1,30 @@ -import base64 import difflib -import tempfile from google.oauth2 import service_account from googleapiclient.discovery import build from googleapiclient.errors import HttpError +from webapp.settings import BASE_DIR + class GoogleDriveClient: # If modifying these scopes, delete the file token.json. SCOPES = ["https://www.googleapis.com/auth/drive"] - def __init__( - self, credentials=None, drive_folder_id=None, copydoc_template_id=None - ): - self.credentials = self._get_credentials(credentials_text=credentials) + def __init__(self, drive_folder_id=None, copydoc_template_id=None): + self.credentials = self._get_credentials() self.service = self._build_service() self.GOOGLE_DRIVE_FOLDER_ID = drive_folder_id self.COPYD0C_TEMPLATE_ID = copydoc_template_id - def _get_credentials(self, credentials_text=None): - """ - Load credentials from a base64 encoded environment variable. + def _get_credentials(self): """ - with tempfile.NamedTemporaryFile() as f: - data = base64.b64decode(credentials_text).replace(b"\\n", b"\n") - f.write(data) + Load credentials from a file.""" - return service_account.Credentials.from_service_account_file( - "webapp/credentials.json", - scopes=self.SCOPES, - ) + return service_account.Credentials.from_service_account_file( + f"{BASE_DIR}/credentials.json", + scopes=self.SCOPES, + ) def _build_service(self): return build("drive", "v3", credentials=self.credentials) @@ -164,7 +158,6 @@ def create_copydoc_from_template(self, webpage): def init_gdrive(app): app.config["gdrive"] = GoogleDriveClient( - credentials=app.config["GOOGLE_SERVICE_ACCOUNT"], drive_folder_id=app.config["GOOGLE_DRIVE_FOLDER_ID"], copydoc_template_id=app.config["COPYD0C_TEMPLATE_ID"], ) From 13eb0f5209bc2290d664d2a5985c4be032f9bb18 Mon Sep 17 00:00:00 2001 From: Olwe Samuel Date: Mon, 7 Oct 2024 12:48:49 +0300 Subject: [PATCH 18/26] Removed service account env variable --- README.md | 2 +- credentials/__init__.py | 0 konf/site.yaml | 10 ---------- webapp/gdrive.py | 2 +- webapp/settings.py | 3 +-- 5 files changed, 3 insertions(+), 14 deletions(-) create mode 100644 credentials/__init__.py diff --git a/README.md b/README.md index 26a6c6d7..5c2b84ba 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Backend service for the CMS template parser Before starting, update the environment variables if needed. The default values will work for docker, save the `GH_TOKEN` which must be manually set. You can create a token [here](https://github.com/settings/tokens), by following [these](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens) instructions. Make sure to select the `repo` scope for the token. -You will also require a credentials file for google drive. Please store it as credentials.json in the parent directory. +You will also require a credentials file for google drive. Please store it as credentials.json in the `credentials` directory. ```env PORT=8104 diff --git a/credentials/__init__.py b/credentials/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/konf/site.yaml b/konf/site.yaml index 0a30491f..eae34e86 100644 --- a/konf/site.yaml +++ b/konf/site.yaml @@ -42,11 +42,6 @@ env: - name: JIRA_COPY_UPDATES_EPIC value: "WD-12643" - - name: GOOGLE_SERVICE_ACCOUNT - secretKeyRef: - key: google-service-account - name: cms-jira - - name: GOOGLE_DRIVE_FOLDER_ID secretKeyRef: key: google-drive-folder-id @@ -123,11 +118,6 @@ demo: - name: JIRA_COPY_UPDATES_EPIC value: "WD-12643" - - name: GOOGLE_SERVICE_ACCOUNT - secretKeyRef: - key: google-service-account - name: cms-jira - - name: GOOGLE_DRIVE_FOLDER_ID secretKeyRef: key: google-drive-folder-id diff --git a/webapp/gdrive.py b/webapp/gdrive.py index 44e0a99c..97b31194 100644 --- a/webapp/gdrive.py +++ b/webapp/gdrive.py @@ -22,7 +22,7 @@ def _get_credentials(self): Load credentials from a file.""" return service_account.Credentials.from_service_account_file( - f"{BASE_DIR}/credentials.json", + f"{BASE_DIR}/credentials/credentials.json", scopes=self.SCOPES, ) diff --git a/webapp/settings.py b/webapp/settings.py index 38fa24cb..db8c7c55 100644 --- a/webapp/settings.py +++ b/webapp/settings.py @@ -14,5 +14,4 @@ JIRA_LABELS = environ.get("JIRA_LABELS") JIRA_COPY_UPDATES_EPIC = environ.get("JIRA_COPY_UPDATES_EPIC") GOOGLE_DRIVE_FOLDER_ID = environ.get("GOOGLE_DRIVE_FOLDER_ID") -COPYD0C_TEMPLATE_ID = environ.get("COPYD0C_TEMPLATE_ID") -GOOGLE_SERVICE_ACCOUNT = environ.get("GOOGLE_SERVICE_ACCOUNT") \ No newline at end of file +COPYD0C_TEMPLATE_ID = environ.get("COPYD0C_TEMPLATE_ID") \ No newline at end of file From c1ce39356665310175c35724bb9d0330b68c7ddc Mon Sep 17 00:00:00 2001 From: Olwe Samuel Date: Tue, 8 Oct 2024 10:40:36 +0300 Subject: [PATCH 19/26] Pass only private key as env --- .env | 2 ++ .github/workflows/CI.yml | 10 ++++++++++ README.md | 2 ++ credentials/__init__.py | 0 konf/site.yaml | 26 ++++++++++++++++++-------- webapp/gdrive.py | 18 +++++++++--------- webapp/settings.py | 18 +++++++++++++++++- 7 files changed, 58 insertions(+), 18 deletions(-) delete mode 100644 credentials/__init__.py diff --git a/.env b/.env index 7633e3a6..5f35206d 100644 --- a/.env +++ b/.env @@ -15,3 +15,5 @@ JIRA_COPY_UPDATES_EPIC=KAN-1 GOOGLE_CREDENTIALS=googlecreds GOOGLE_DRIVE_FOLDER_ID=googlecreds COPYD0C_TEMPLATE_ID=googlecreds +PRIVATE_KEY=base64encodedprivatekey +PRIVATE_KEY_ID=privatekeyid diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index b3ac3ac0..7a1b4de5 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -116,6 +116,9 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt - name: Run service + env: + PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }} + PRIVATE_KEY_ID: ${{ secrets.PRIVATE_KEY_ID }} run: | set -a source .env @@ -150,6 +153,9 @@ jobs: run: DOCKER_BUILDKIT=1 docker build --tag websites-content-system . - name: Run image + env: + PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }} + PRIVATE_KEY_ID: ${{ secrets.PRIVATE_KEY_ID }} run: | docker run \ -p 8104:8104 \ @@ -164,6 +170,10 @@ jobs: -e JIRA_URL=https://example.atlassian.net \ -e JIRA_LABELS=somelabel \ -e JIRA_COPY_UPDATES_EPIC=WD-9999999 \ + -e GOOGLE_DRIVE_FOLDER_ID=folderid \ + -e COPYDOC_TEMPLATE_ID=templateid \ + -e PRIVATE_KEY="$PRIVATE_KEY" \ + -e PRIVATE_KEY_ID="$PRIVATE_KEY_ID" \ --network host \ websites-content-system & sleep 1 curl --head --fail --retry-delay 1 --retry 30 --retry-connrefused http://localhost diff --git a/README.md b/README.md index 5c2b84ba..89e22f94 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,8 @@ JIRA_LABELS=sites_BAU JIRA_COPY_UPDATES_EPIC=WD-12643 GOOGLE_DRIVE_FOLDER_ID=1EIFOGJ8DIWpsYIfWk7Yos3YijZIkbJDk COPYD0C_TEMPLATE_ID=125auRsLQukYH-tKN1oEKaksmpCXd_DTGiswvmbeS2iA +PRIVATE_KEY=base64encodedprivatekey +PRIVATE_KEY_ID=privatekeyid ``` ### Using docker diff --git a/credentials/__init__.py b/credentials/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/konf/site.yaml b/konf/site.yaml index eae34e86..c26d04ca 100644 --- a/konf/site.yaml +++ b/konf/site.yaml @@ -26,12 +26,12 @@ env: - name: JIRA_EMAIL secretKeyRef: key: jira-email - name: cms-jira + name: cs-canonical-com - name: JIRA_TOKEN secretKeyRef: key: jira-token - name: cms-jira + name: cs-canonical-com - name: JIRA_URL value: "https://warthogs.atlassian.net" @@ -45,12 +45,22 @@ env: - name: GOOGLE_DRIVE_FOLDER_ID secretKeyRef: key: google-drive-folder-id - name: cms-jira + name: cs-canonical-com - name: COPYDOC_TEMPLATE_ID secretKeyRef: key: copydoc-template-id - name: cms-jira + name: cs-canonical-com + + - name: PRIVATE_KEY + secretKeyRef: + key: private-key + name: cs-canonical-com + + - name: PRIVATE_KEY_ID + secretKeyRef: + key: private-key-id + name: cs-canonical-com # Overrides for production production: @@ -102,12 +112,12 @@ demo: - name: JIRA_EMAIL secretKeyRef: key: jira-email - name: cms-jira + name: cs-canonical-com - name: JIRA_TOKEN secretKeyRef: key: jira-token - name: cms-jira + name: cs-canonical-com - name: JIRA_URL value: "https://warthogs.atlassian.net" @@ -121,9 +131,9 @@ demo: - name: GOOGLE_DRIVE_FOLDER_ID secretKeyRef: key: google-drive-folder-id - name: cms-jira + name: cs-canonical-com - name: COPYDOC_TEMPLATE_ID secretKeyRef: key: copydoc-template-id - name: cms-jira \ No newline at end of file + name: cs-canonical-com \ No newline at end of file diff --git a/webapp/gdrive.py b/webapp/gdrive.py index 97b31194..0cba859d 100644 --- a/webapp/gdrive.py +++ b/webapp/gdrive.py @@ -4,25 +4,24 @@ from googleapiclient.discovery import build from googleapiclient.errors import HttpError -from webapp.settings import BASE_DIR - class GoogleDriveClient: # If modifying these scopes, delete the file token.json. SCOPES = ["https://www.googleapis.com/auth/drive"] - def __init__(self, drive_folder_id=None, copydoc_template_id=None): - self.credentials = self._get_credentials() + def __init__( + self, credentials, drive_folder_id=None, copydoc_template_id=None + ): + self.credentials = self._get_credentials(credentials) self.service = self._build_service() self.GOOGLE_DRIVE_FOLDER_ID = drive_folder_id self.COPYD0C_TEMPLATE_ID = copydoc_template_id - def _get_credentials(self): - """ - Load credentials from a file.""" + def _get_credentials(self, credentials): + """Load credentials from an object.""" - return service_account.Credentials.from_service_account_file( - f"{BASE_DIR}/credentials/credentials.json", + return service_account.Credentials.from_service_account_info( + credentials, scopes=self.SCOPES, ) @@ -158,6 +157,7 @@ def create_copydoc_from_template(self, webpage): def init_gdrive(app): app.config["gdrive"] = GoogleDriveClient( + credentials=app.config["GOOGLE_CREDENTIALS"], drive_folder_id=app.config["GOOGLE_DRIVE_FOLDER_ID"], copydoc_template_id=app.config["COPYD0C_TEMPLATE_ID"], ) diff --git a/webapp/settings.py b/webapp/settings.py index db8c7c55..161b11b7 100644 --- a/webapp/settings.py +++ b/webapp/settings.py @@ -1,3 +1,4 @@ +import base64 import os from os import environ @@ -14,4 +15,19 @@ JIRA_LABELS = environ.get("JIRA_LABELS") JIRA_COPY_UPDATES_EPIC = environ.get("JIRA_COPY_UPDATES_EPIC") GOOGLE_DRIVE_FOLDER_ID = environ.get("GOOGLE_DRIVE_FOLDER_ID") -COPYD0C_TEMPLATE_ID = environ.get("COPYD0C_TEMPLATE_ID") \ No newline at end of file +COPYD0C_TEMPLATE_ID = environ.get("COPYD0C_TEMPLATE_ID") +GOOGLE_CREDENTIALS = { + "type": "service_account", + "project_id": "web-engineering-436014", + "private_key_id": environ.get("PRIVATE_KEY_ID"), + "private_key": base64.b64decode(environ.get("PRIVATE_KEY")).replace( + b"\\n", b"\n" + ), + "client_email": "websites-copy-docs-627@web-engineering-436014.iam.gserviceaccount.com", # noqa: E501 + "client_id": "116847960229506342511", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/websites-copy-docs-627%40web-engineering-436014.iam.gserviceaccount.com", # noqa: E501 + "universe_domain": "googleapis.com", +} From 471bd3673d5e6c029335b7db8f3f17671fb68bf3 Mon Sep 17 00:00:00 2001 From: Olwe Samuel Date: Tue, 8 Oct 2024 10:47:45 +0300 Subject: [PATCH 20/26] declared env variables explicitly for python ci --- .env | 2 +- .github/workflows/CI.yml | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/.env b/.env index 5f35206d..2703b490 100644 --- a/.env +++ b/.env @@ -4,7 +4,7 @@ VALKEY_HOST=localhost VALKEY_PORT=6379 GH_TOKEN=token REPO_ORG=https://github.com/canonical -SQLALCHEMY_DATABASE_URI=postgresql://postgres:postgres@localhost:5432/postgres +DATABASE_URL=postgresql://postgres:postgres@localhost:5432/content TASK_DELAY=30 DIRECTORY_API_TOKEN=token JIRA_EMAIL=email@example.com diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 7a1b4de5..146af479 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -119,10 +119,20 @@ jobs: env: PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }} PRIVATE_KEY_ID: ${{ secrets.PRIVATE_KEY_ID }} + DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres + SECRET_KEY: secret_key + VALKEY_HOST: localhost + VALKEY_PORT: 6379 + GH_TOKEN: token + REPO_ORG: https://github.com/canonical + JIRA_EMAIL: example@canonical.com + JIRA_TOKEN: jiratoken + JIRA_URL: https://example.atlassian.net + JIRA_LABELS: somelabel + JIRA_COPY_UPDATES_EPIC: WD-9999999 + GOOGLE_DRIVE_FOLDER_ID: folderid + COPYDOC_TEMPLATE_ID: templateid run: | - set -a - source .env - set +a talisker.gunicorn webapp.app:app --daemon --bind 0.0.0.0:8104 curl --head --fail --retry-delay 1 --retry 30 --retry-connrefused http://localhost:8104 @@ -164,7 +174,7 @@ jobs: -e VALKEY_PORT=6379 \ -e GH_TOKEN=token \ -e REPO_ORG=https://github.com/canonical \ - -e SQLALCHEMY_DATABASE_URI=postgresql://postgres:postgres@localhost:5432/postgres \ + -e DATABASE_URL=postgresql://postgres:postgres@localhost:5432/postgres \ -e JIRA_EMAIL=example@canonical.com \ -e JIRA_TOKEN=jiratoken \ -e JIRA_URL=https://example.atlassian.net \ From c401f38b7c24fbb6391cca8e2ce6a3a9859c7b6a Mon Sep 17 00:00:00 2001 From: Olwe Samuel Date: Tue, 8 Oct 2024 11:20:46 +0300 Subject: [PATCH 21/26] dont duplicate subfolders --- webapp/gdrive.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/webapp/gdrive.py b/webapp/gdrive.py index 0cba859d..5606a881 100644 --- a/webapp/gdrive.py +++ b/webapp/gdrive.py @@ -93,7 +93,7 @@ def build_webpage_folder(self, webpage): """ Create a folder hierarchy in Google Drive for a webpage. """ - folders = webpage.url.split("/")[:-1] + folders = [f for f in webpage.url.split("/")[:-1] if f != ""] # Check if the project folder exists, or create one if not ( parent := self._item_exists( @@ -106,9 +106,9 @@ def build_webpage_folder(self, webpage): # Create subfolders for folder in folders: - if folder != "": + if not (folder_id := self._item_exists(folder, parent)): folder_id = self.create_folder(folder, parent) - parent = folder_id + parent = folder_id # Return the last parent folder return parent From ce84b457bdba1c5bf9beb3687061a6960c298b7d Mon Sep 17 00:00:00 2001 From: Olwe Samuel Date: Tue, 8 Oct 2024 11:35:55 +0300 Subject: [PATCH 22/26] Added private key to demo --- konf/site.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/konf/site.yaml b/konf/site.yaml index c26d04ca..5a292a02 100644 --- a/konf/site.yaml +++ b/konf/site.yaml @@ -136,4 +136,14 @@ demo: - name: COPYDOC_TEMPLATE_ID secretKeyRef: key: copydoc-template-id + name: cs-canonical-com + + - name: PRIVATE_KEY + secretKeyRef: + key: private-key + name: cs-canonical-com + + - name: PRIVATE_KEY_ID + secretKeyRef: + key: private-key-id name: cs-canonical-com \ No newline at end of file From 39db5be0fd367a878f99535e1c0bbbed7b045f07 Mon Sep 17 00:00:00 2001 From: Olwe Samuel Date: Tue, 8 Oct 2024 16:13:35 +0300 Subject: [PATCH 23/26] Returned file url after copydoc creation --- webapp/app.py | 5 +++-- webapp/gdrive.py | 7 +++---- webapp/helper.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/webapp/app.py b/webapp/app.py index 47ee01c5..9dae0f16 100644 --- a/webapp/app.py +++ b/webapp/app.py @@ -135,8 +135,9 @@ def request_changes(body: ChangesRequestModel): # Make a request to JIRA to create a task try: - create_jira_task(app, body.model_dump()) + task = create_jira_task(app, body.model_dump()) + task_url = f"https://docs.google.com/document/d/{task['id']}" except Exception as e: return jsonify(str(e)), 500 - return jsonify("Task created successfully"), 201 + return jsonify({"message": f"Task created successfully\n{task_url}"}), 201 diff --git a/webapp/gdrive.py b/webapp/gdrive.py index 5606a881..c4ac77cd 100644 --- a/webapp/gdrive.py +++ b/webapp/gdrive.py @@ -93,7 +93,7 @@ def build_webpage_folder(self, webpage): """ Create a folder hierarchy in Google Drive for a webpage. """ - folders = [f for f in webpage.url.split("/")[:-1] if f != ""] + folders = [f"/{f}" for f in webpage.url.split("/")[:-1] if f != ""] # Check if the project folder exists, or create one if not ( parent := self._item_exists( @@ -147,10 +147,9 @@ def create_copydoc_from_template(self, webpage): webpage_folder = self.build_webpage_folder(webpage) # Clone the template document to the new folder - doc_name = f'{webpage.url} - "{webpage.title}"' - self.copy_file( + return self.copy_file( fileID=self.COPYD0C_TEMPLATE_ID, - name=doc_name, + name=webpage.url, parents=webpage_folder, ) diff --git a/webapp/helper.py b/webapp/helper.py index f36a4db4..cf503a7f 100644 --- a/webapp/helper.py +++ b/webapp/helper.py @@ -48,4 +48,4 @@ def create_jira_task(app, task): if task["type"] == jira.NEW_WEBPAGE: webpage = Webpage.query.filter_by(id=task["webpage_id"]).first() client = app.config["gdrive"] - client.create_copydoc_from_template(webpage) + return client.create_copydoc_from_template(webpage) From 99410c0affa23582ab138f1aad89303543c75f08 Mon Sep 17 00:00:00 2001 From: Olwe Samuel Date: Tue, 8 Oct 2024 16:19:54 +0300 Subject: [PATCH 24/26] Use explicit naming for private keys --- .env | 4 ++-- .github/workflows/CI.yml | 12 ++++++------ README.md | 4 ++-- konf/site.yaml | 16 ++++++++-------- webapp/settings.py | 4 ++-- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.env b/.env index 2703b490..51d74b4a 100644 --- a/.env +++ b/.env @@ -15,5 +15,5 @@ JIRA_COPY_UPDATES_EPIC=KAN-1 GOOGLE_CREDENTIALS=googlecreds GOOGLE_DRIVE_FOLDER_ID=googlecreds COPYD0C_TEMPLATE_ID=googlecreds -PRIVATE_KEY=base64encodedprivatekey -PRIVATE_KEY_ID=privatekeyid +GOOGLE_PRIVATE_KEY=base64encodedprivatekey +GOOGLE_PRIVATE_KEY_ID=privatekeyid diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 146af479..c4da1466 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -117,8 +117,8 @@ jobs: pip install -r requirements.txt - name: Run service env: - PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }} - PRIVATE_KEY_ID: ${{ secrets.PRIVATE_KEY_ID }} + GOOGLE_PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }} + GOOGLE_PRIVATE_KEY_ID: ${{ secrets.PRIVATE_KEY_ID }} DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres SECRET_KEY: secret_key VALKEY_HOST: localhost @@ -164,8 +164,8 @@ jobs: - name: Run image env: - PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }} - PRIVATE_KEY_ID: ${{ secrets.PRIVATE_KEY_ID }} + GOOGLE_PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }} + GOOGLE_PRIVATE_KEY_ID: ${{ secrets.PRIVATE_KEY_ID }} run: | docker run \ -p 8104:8104 \ @@ -182,8 +182,8 @@ jobs: -e JIRA_COPY_UPDATES_EPIC=WD-9999999 \ -e GOOGLE_DRIVE_FOLDER_ID=folderid \ -e COPYDOC_TEMPLATE_ID=templateid \ - -e PRIVATE_KEY="$PRIVATE_KEY" \ - -e PRIVATE_KEY_ID="$PRIVATE_KEY_ID" \ + -e GOOGLE_PRIVATE_KEY="$PRIVATE_KEY" \ + -e GOOGLE_PRIVATE_KEY_ID="$PRIVATE_KEY_ID" \ --network host \ websites-content-system & sleep 1 curl --head --fail --retry-delay 1 --retry 30 --retry-connrefused http://localhost diff --git a/README.md b/README.md index 89e22f94..ba4cd67b 100644 --- a/README.md +++ b/README.md @@ -29,8 +29,8 @@ JIRA_LABELS=sites_BAU JIRA_COPY_UPDATES_EPIC=WD-12643 GOOGLE_DRIVE_FOLDER_ID=1EIFOGJ8DIWpsYIfWk7Yos3YijZIkbJDk COPYD0C_TEMPLATE_ID=125auRsLQukYH-tKN1oEKaksmpCXd_DTGiswvmbeS2iA -PRIVATE_KEY=base64encodedprivatekey -PRIVATE_KEY_ID=privatekeyid +GOOGLE_PRIVATE_KEY=base64encodedprivatekey +GOOGLE_PRIVATE_KEY_ID=privatekeyid ``` ### Using docker diff --git a/konf/site.yaml b/konf/site.yaml index 5a292a02..2661c95f 100644 --- a/konf/site.yaml +++ b/konf/site.yaml @@ -52,14 +52,14 @@ env: key: copydoc-template-id name: cs-canonical-com - - name: PRIVATE_KEY + - name: GOOGLE_PRIVATE_KEY secretKeyRef: - key: private-key + key: google-private-key name: cs-canonical-com - - name: PRIVATE_KEY_ID + - name: GOOGLE_PRIVATE_KEY_ID secretKeyRef: - key: private-key-id + key: google-private-key-id name: cs-canonical-com # Overrides for production @@ -138,12 +138,12 @@ demo: key: copydoc-template-id name: cs-canonical-com - - name: PRIVATE_KEY + - name: GOOGLE_PRIVATE_KEY secretKeyRef: - key: private-key + key: google-private-key name: cs-canonical-com - - name: PRIVATE_KEY_ID + - name: GOOGLE_PRIVATE_KEY_ID secretKeyRef: - key: private-key-id + key: google-private-key-id name: cs-canonical-com \ No newline at end of file diff --git a/webapp/settings.py b/webapp/settings.py index 161b11b7..f5d07e03 100644 --- a/webapp/settings.py +++ b/webapp/settings.py @@ -19,8 +19,8 @@ GOOGLE_CREDENTIALS = { "type": "service_account", "project_id": "web-engineering-436014", - "private_key_id": environ.get("PRIVATE_KEY_ID"), - "private_key": base64.b64decode(environ.get("PRIVATE_KEY")).replace( + "private_key_id": environ.get("GOOGLE_PRIVATE_KEY_ID"), + "private_key": base64.b64decode(environ.get("GOOGLE_PRIVATE_KEY")).replace( b"\\n", b"\n" ), "client_email": "websites-copy-docs-627@web-engineering-436014.iam.gserviceaccount.com", # noqa: E501 From 3447d2627846750f447cea6bde3a25fed866d945 Mon Sep 17 00:00:00 2001 From: Olwe Samuel Date: Tue, 8 Oct 2024 16:37:13 +0300 Subject: [PATCH 25/26] Update CI.yml --- .github/workflows/CI.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index c4da1466..02c1a43a 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -182,8 +182,8 @@ jobs: -e JIRA_COPY_UPDATES_EPIC=WD-9999999 \ -e GOOGLE_DRIVE_FOLDER_ID=folderid \ -e COPYDOC_TEMPLATE_ID=templateid \ - -e GOOGLE_PRIVATE_KEY="$PRIVATE_KEY" \ - -e GOOGLE_PRIVATE_KEY_ID="$PRIVATE_KEY_ID" \ + -e GOOGLE_PRIVATE_KEY="$GOOGLE_PRIVATE_KEY" \ + -e GOOGLE_PRIVATE_KEY_ID="$GOOGLE_PRIVATE_KEY_ID" \ --network host \ websites-content-system & sleep 1 curl --head --fail --retry-delay 1 --retry 30 --retry-connrefused http://localhost From 1642cbee8a86c3a1ac67c71bb2ae68e0d71cd72f Mon Sep 17 00:00:00 2001 From: Olwe Samuel Date: Tue, 8 Oct 2024 17:03:21 +0300 Subject: [PATCH 26/26] Added env variables to playwright --- .github/workflows/playwright.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 58807e88..3c440ecd 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -31,7 +31,16 @@ jobs: VALKEY_PORT: 6379 GH_TOKEN: token REPO_ORG: https://github.com/canonical - SQLALCHEMY_DATABASE_URI: postgresql://postgres:postgres@localhost:5432/postgres + DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres + GOOGLE_PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }} + GOOGLE_PRIVATE_KEY_ID: ${{ secrets.PRIVATE_KEY_ID }} + JIRA_EMAIL: example@canonical.com + JIRA_TOKEN: jiratoken + JIRA_URL: https://example.atlassian.net + JIRA_LABELS: somelabel + JIRA_COPY_UPDATES_EPIC: WD-9999999 + GOOGLE_DRIVE_FOLDER_ID: folderid + COPYDOC_TEMPLATE_ID: templateid steps: - uses: actions/checkout@v4 - name: Set up Python 3.12