Refactor pdc.

Author: Rup-Narayan-Rajbanshi

apps/etl/etl_tasks/pdc.py

@@ -13,13 +13,14 @@
 
 
 @shared_task
-def extract_and_transform_pdc_data():
+def ext_and_transform_pdc_latest_data():
+    # TODO Fix according to latest extraction logic that accepts Metadata
     data_url = f"{etl_config.PDC_SENTRY_BASE_URL}/hp_srv/services/hazards/t/json/get_active_hazards"
     PDCExtraction.task.s(data_url).apply_async()
 
 
 @shared_task(queue="extraction")
-def extract_and_transform_historical_pdc_data():
+def ext_and_transform_pdc_historical_data():
     pdc_start_date = datetime.strptime(str(etl_config.PDC_START_DATE), "%Y-%m-%d")
     pdc_interval_years = etl_config.PDC_EXTRACTION_INTERVAL_YEARS
 

apps/etl/management/commands/extract_pdc_data.py

@@ -2,7 +2,7 @@
 
 from django.core.management.base import BaseCommand
 
-from apps.etl.etl_tasks.pdc import extract_and_transform_historical_pdc_data
+from apps.etl.etl_tasks.pdc import ext_and_transform_pdc_historical_data
 
 logger = logging.getLogger(__name__)
 
@@ -11,4 +11,4 @@ class Command(BaseCommand):
     help = "Import data from pdc api"
 
     def handle(self, *args, **options):
-        extract_and_transform_historical_pdc_data.delay()
+        ext_and_transform_pdc_historical_data.delay()

apps/etl/tasks.py

@@ -2,21 +2,6 @@
 from django.core.management import call_command
 
 
-@shared_task
-def extract_pdc_data():
-    call_command("extract_pdc_data")
-
-
-@shared_task
-def extract_gidd_data():
-    call_command("extract_gidd_data")
-
-
-@shared_task
-def extract_usgs_data():
-    call_command("extract_usgs_data")
-
-
 @shared_task
 def load_data():
     call_command("load_data_to_stac")

main/cronjobs.py

@@ -50,15 +50,15 @@ class CronJob(typing.NamedTuple):
         schedule=crontab(minute=0, hour=17),
     ),
     "import_gidd_data": CronJob(
-        task="apps.etl.tasks.extract_gidd_data",
+        task="apps.etl.etl_tasks.gidd.ext_and_transform_gidd_latest_data",
         schedule=crontab(minute=0, hour=18),
     ),
     "import_usgs_data": CronJob(
         task="apps.etl.etl_tasks.usgs.ext_and_transform_usgs_latest_data",
         schedule=crontab(minute=0, hour=19),
     ),
     "import_pdc_data": CronJob(
-        task="apps.etl.tasks.extract_pdc_data",
+        task="apps.etl.etl_tasks.pdc.ext_and_transform_pdc_latest_data",
         schedule=crontab(minute=30, hour=20),
     ),
     "load_data_to_stac": CronJob(