Merge pull request #84 from odissei-data/development

Removed twente, trimbos and hsn from the deployment list.

Author: FjodorvRijsselberg

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,13 @@
+Fixes Jira issue ODSP-
+
+# Description of changes
+
+# How to test
+
+# Related PRs
+
+(Add links)
+
+*
+
+# Notify

Dockerfile.server

@@ -1,4 +1,4 @@
-FROM python:3.11-slim
+FROM python:3.13.7-slim
 
 ENV PYTHONPATH="${PYTHONPATH}:/app/scripts/" \
     PYTHONDONTWRITEBYTECODE=1

Dockerfile.worker

@@ -1,4 +1,4 @@
-FROM python:3.11-slim
+FROM python:3.13.7-slim
 
 ENV PYTHONPATH="${PYTHONPATH}:/app/scripts/" \
     PYTHONDONTWRITEBYTECODE=1

Makefile

@@ -5,7 +5,9 @@ PROJECT_NAME = prefect_docker
 PROJECT_SRV = ${PROJECT_NAME}
 TARGET_URL ?=
 TARGET_KEY ?=
+TARGET_BUCKET ?=
 DO_HARVEST ?= True
+FULL_HARVEST ?= False
 
 .PHONY = help
 .DEFAULT:
@@ -61,6 +63,6 @@ submodules: ## Sets up the submodules and checks out their main branch.
 	git submodule foreach git checkout main
 	git submodule foreach git pull origin main
 ingest: ## Runs the ingest workflow for a specified data provider. The url and key of the target can be optionally added. eg: make ingest data_provider=CBS TARGET_URL=https://portal.example.odissei.nl TARGET_KEY=abcde123-11aa-22bb-3c4d-098765432abc
-	@docker exec -it ${PROJECT_CONTAINER_NAME} python run_ingestion.py --data_provider=$(data_provider) --target_url=$(TARGET_URL) --target_key=$(TARGET_KEY) --do_harvest=$(DO_HARVEST)
+	@docker exec -it ${PROJECT_CONTAINER_NAME} python run_ingestion.py --data_provider=$(data_provider) --target_url=$(TARGET_URL) --target_key=$(TARGET_KEY) --target_bucket=$(TARGET_BUCKET) --do_harvest=$(DO_HARVEST) --full_harvest=$(FULL_HARVEST)
 deploy: ## Deploys all ingestion workflows to the prefect server.
 	@docker exec -it prefect python deployment/deploy_ingestion_pipelines.py

boto_utils/boto_bucket_downloader.py

@@ -0,0 +1,46 @@
+import argparse
+import boto3
+
+parser = argparse.ArgumentParser(description="Download an S3 bucket")
+parser.add_argument('--url', type=str, dest='boto_url', help='URL of the server')
+parser.add_argument('--username', type=str, dest='access_key', help='AWS access key')
+parser.add_argument('--password', type=str, dest='secret_key', help='AWS secret key')
+parser.add_argument('--bucket', type=str, dest='bucket_name', help='Name of the bucket')
+parser.add_argument('--directory', type=str, dest='download_directory', default=".", help='Directory to download files to')
+args = parser.parse_args()
+
+s3_client = boto3.client(
+    's3',
+    endpoint_url=args.boto_url,
+    aws_access_key_id=args.access_key,
+    aws_secret_access_key=args.secret_key
+)
+
+download_directory = args.download_directory
+
+try: 
+    paginator = s3_client.get_paginator("list_objects_v2")
+    pages = paginator.paginate(
+        Bucket=args.bucket_name,
+    )
+    object_count = 0
+    page_count = 0
+    for page in pages:
+        page_count += 1
+        if 'Contents' in page:
+            object_count += len(page['Contents'])
+            print(f"Number of objects in page '{page_count}' of bucket '{args.bucket_name}': {len(page['Contents'])}")
+            print(f"Start Downloading {len(page['Contents'])} objects from bucket '{args.bucket_name}' in page {page_count}:")
+            for obj in page['Contents']:
+                s3_client.download_file(Bucket=args.bucket_name, Key=obj['Key'], Filename=download_directory+'/'+obj['Key'])
+                print(".", end="", flush=True) # most primitive progress indicator
+            print()  # for newline after progress dots 
+            print(f"Downloading page {page_count} done.")
+            print(f"Total number of objects downloaded so far from '{args.bucket_name}': {object_count}")
+        else:
+            print(f"No objects found in '{args.bucket_name}'")
+    if object_count > 0:
+        print(f"Downloading done. Total number of objects downloaded from '{args.bucket_name}': {object_count}")
+    # Note that downloading objects is done per page,
+    # but the list of objects could be extracted first and then we could download them all in one loop.
+    print(f"Error downloading objects from bucket '{args.bucket_name}': {e}")

docker-compose.yml

@@ -37,10 +37,7 @@ services:
     container_name: db
     volumes:
       - prefectdb:/var/lib/postgresql/data
-    environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_PASSWORD=lolgres
-      - POSTGRES_DB=prefect
+    env_file: .env
     networks:
       - prefect
 

notifications.md

@@ -0,0 +1,25 @@
+Notifications
+=============
+
+When workflows fail notifications should be sent (to a Slack channel) in order for the person(s) 
+responsible for the service to check the logs and fix the problems. 
+
+
+# Configure Slack
+
+There is a Slack workplace created for the messages about the Odissei Ingest (workflows) with the name `odissei-ingest`.
+In this workspace there is a `prefect-notifications` channel that will be used for sending 
+the notifications from the prefect workflow on failures. 
+Using the Slack Api page (https://api.slack.com/apps/) a ‘Webhook App' application was created . 
+With this application there is a 'Webhooks Features' option that allows to create new webhooks. The URL for this newly created webhook is then used for the configuration of Prefect, described in the next section. 
+
+The 'sample’ URL looks like this: 
+```
+curl -X POST -H 'Content-type: application/json' --data '{"text":"Hello, World!"}' https://hooks.slack.com/services/<unique string here>
+```
+You can test this on the commandline, the message should appear in that Slack channel. 
+
+# Configure Prefect
+
+Create a ‘Slack Webhook Block’ in the Prefect UI. That webhook URL from Slack is pasted into the the URL field of that block. The Prefect UI will give sample code to be placed inside your flow code in order to notify. This has been used in the code at the point where the 'bucket' with the failure information (failed dataset PIDs) is created. Noet that instead of using the hardcodede name of the block, you need to assign that name to the `PREFECT_SLACK_WEBHOOK_BLOCK` setting. 
+The simplest way to test if it works is to force an error by using wrong API key. Then check the logs, the Slack channel and the 'bucket'.