basedosdados
diff --git a/‎.github/workflows/data-check.yml
Lines changed: 8 additions & 10 deletions b/‎.github/workflows/data-check.yml
Lines changed: 8 additions & 10 deletions
diff --git a/‎.github/workflows/data-check/checks.yaml
Lines changed: 14 additions & 6 deletions b/‎.github/workflows/data-check/checks.yaml
Lines changed: 14 additions & 6 deletions
diff --git a/‎.github/workflows/data-check/test_data.py
Lines changed: 2 additions & 2 deletions b/‎.github/workflows/data-check/test_data.py
Lines changed: 2 additions & 2 deletions
diff --git a/‎.github/workflows/env-setup/env_setup.py
Lines changed: 4 additions & 0 deletions b/‎.github/workflows/env-setup/env_setup.py
Lines changed: 4 additions & 0 deletions
diff --git a/‎.github/workflows/etl-caged.yml
Lines changed: 2 additions & 0 deletions b/‎.github/workflows/etl-caged.yml
Lines changed: 2 additions & 0 deletions
diff --git a/‎.github/workflows/metadata-validate.yml
Lines changed: 57 additions & 0 deletions b/‎.github/workflows/metadata-validate.yml
Lines changed: 57 additions & 0 deletions
diff --git a/‎.github/workflows/metadata-validate/metadata_validate.py
Lines changed: 112 additions & 0 deletions b/‎.github/workflows/metadata-validate/metadata_validate.py
Lines changed: 112 additions & 0 deletions
diff --git a/‎.github/workflows/python-ci.yml
Lines changed: 16 additions & 10 deletions b/‎.github/workflows/python-ci.yml
Lines changed: 16 additions & 10 deletions
@@ -4,7 +4,6 @@ on:
   pull_request_target:
     types:
       - labeled
-
 jobs:
   guard:
     runs-on: ubuntu-latest
@@ -25,7 +24,6 @@ jobs:
           steps.changes.outputs.workflows == 'true' &&
           github.event.pull_request.head.repo.full_name != github.repository
         run: exit 1
-
   get-changes:
     needs: guard
     runs-on: ubuntu-latest
@@ -57,22 +55,22 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: '3.9.x'
+          python-version: "3.9.x"
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install basedosdados==1.5.2 pyarrow pytest toml
-      - name: Set up basedosdados environment
-        run: |
-          cd .github/workflows/env-setup
-          python env_setup.py
+          pip install basedosdados==1.6.0 pyarrow pytest toml
+      - name: Set up base dos dados environment
         shell: bash
-        env:
-          BUCKET_NAME: basedosdados-dev
+        env:     
+          BUCKET_NAME: basedosdados-dev 
           PROJECT_NAME_PROD: basedosdados-dev
           PROJECT_NAME_STAGING: basedosdados-dev
           GCP_BD_PROD: ${{ secrets.GCP_BD_DEV_PROD }}
           GCP_BD_STAGING: ${{ secrets.GCP_BD_DEV_STAGING }}
+          CKAN_URL: "https://staging.basedosdados.org"
+          CKAN_API_KEY: ${{ secrets.CKAN_STAGING }}
+        run: python .github/workflows/env-setup/env_setup.py
       - name: Test data and fill report
         run: pytest -v .github/workflows/data-check
         shell: bash
 
@@ -1,3 +1,11 @@
+# test_select_all_works:
+#   id: {{ dataset_id }}/{{ table_id }}/1
+#   name: Check if select all  works
+#   query: |
+#     SELECT EXISTS (
+#         SELECT * 
+#         FROM `{{ project_id }}.{{ dataset_id }}.{{ table_id }}`
+#     ) AS sucess
 
 test_table_exists:
   id: {{ dataset_id }}/{{ table_id }}/0
@@ -26,19 +34,19 @@ test_table_has_no_null_column:
     SELECT col_name, nulls_count / total_count null_percent
     FROM n_nulls, n_total
 
-test_primary_key_has_unique_values:
+test_identifying_column_has_unique_values:
   id: {{ dataset_id }}/{{ table_id }}/3
-  name: Check if primary key has unique values
+  name: Check if identifying column has unique values
   query: |
-    {% if primary_keys is defined and primary_keys is not none -%}
+    {% if identifying_columns is defined and identifying_columns is not none -%}
     SELECT 
       COUNT(
         DISTINCT CONCAT(
-          {% for primary_key in primary_keys -%}
-            IFNULL(SAFE_CAST({{ primary_key }} AS STRING), " "), "&",
+          {% for identifying_column in identifying_columns -%}
+            IFNULL(SAFE_CAST({{ identifying_column }} AS STRING), " "), "&",
           {% endfor -%}
           "EOF"
         )
       ) / COUNT(*) unique_percentage 
     FROM `{{ project_id }}.{{ dataset_id }}.{{ table_id }}` t
-    {% endif %}
+    {% endif %}
@@ -83,8 +83,8 @@ def test_table_has_no_null_column(configs):
     assert result.empty or (result.null_percent.max() < 1)
 
 
-def test_primary_key_has_unique_values(configs):
-    config = configs["test_primary_key_has_unique_values"]
+def test_identifying_column_has_unique_values(configs):
+    config = configs["test_identifying_column_has_unique_values"]
     result = fetch_data(config)
 
     result = result.unique_percentage.values[0]
 
@@ -77,6 +77,10 @@ def env_setup():
                 ),
             },
         },
+        "ckan": {
+            "url": os.getenv("CKAN_URL"),
+            "api_key": decoding_base64(os.environ.get("CKAN_API_KEY")).replace("\n", ""),
+        },
     }
 
     # load the secret of prod and staging data
 
@@ -58,6 +58,8 @@ jobs:
           PROJECT_NAME_STAGING: basedosdados-staging
           GCP_BD_PROD: ${{ secrets.GCP_BD_DEV_PROD }}
           GCP_BD_STAGING: ${{ secrets.GCP_BD_DEV_STAGING }}
+          CKAN_URL: "https://staging.basedosdados.org"
+          CKAN_API_KEY: ${{ secrets.CKAN_STAGING }}
       - name: Run ETL-CAGED
         run: python bases/br_me_caged/code/caged_novo/caged_novo.py
         shell: bash
@@ -0,0 +1,57 @@
+name: metadata-validate
+
+on:
+  push:
+    paths:
+      - 'bases/**'
+jobs:
+  get-changes:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check file changes
+        id: file_changes
+        uses: trilom/file-changes-action@v1.2.4
+      - name: Copy file changes
+        run: cp $HOME/files.json files.json
+      - name: Upload file changes
+        uses: actions/upload-artifact@v2
+        with:
+          name: push-changes
+          path: files.json
+
+  metadata-validate:
+    needs: get-changes
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v2
+      - name: Download changes
+        uses: actions/download-artifact@v2
+        with:
+          name: push-changes
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.9.x"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install basedosdados==1.6.0 toml
+      - name: Set up base dos dados environment
+        run: python .github/workflows/env-setup/env_setup.py
+        shell: bash
+        env:
+          BUCKET_NAME: basedosdados
+          PROJECT_NAME_PROD: basedosdados
+          PROJECT_NAME_STAGING: basedosdados-staging
+          GCP_BD_PROD: ${{ secrets.GCP_TABLE_APPROVE_PROD }}
+          GCP_BD_STAGING: ${{ secrets.GCP_TABLE_APPROVE_STAGING }}
+          CKAN_URL: "https://staging.basedosdados.org"
+          CKAN_API_KEY: ${{ secrets.CKAN_STAGING }}
+      - name: Metadata validate
+        run: python -u .github/workflows/metadata-validate/metadata_validate.py
+        shell: bash
+        env:
+          PROJECT_ID: ${{ secrets.GCP_MAIN_PROJECT_ID }}
+          BUCKET_NAME_BACKUP: basedosdados-backup
+          BUCKET_NAME_DESTINATION: basedosdados
@@ -0,0 +1,112 @@
+import json
+import os
+import traceback
+from pathlib import Path
+from pprint import pprint
+
+import basedosdados as bd
+import yaml
+from basedosdados import Dataset, Storage
+from basedosdados.upload.base import Base
+from basedosdados.upload.metadata import Metadata
+
+
+def tprint(title=""):
+    if not len(title):
+        print(
+            "\n",
+            "#" * 80,
+            "\n",
+        )
+    else:
+        size = 38 - int(len(title) / 2)
+        print("\n\n\n", "#" * size, title, "#" * size, "\n")
+
+
+def load_configs(dataset_id, table_id):
+    # get the config file in .basedosdados/config.toml
+    configs_path = Base()._load_config()
+
+    # get the path to metadata_path, where the folder bases with metadata information
+    metadata_path = configs_path["metadata_path"]
+
+    # get the path to table_config.yaml
+    table_path = f"{metadata_path}/{dataset_id}/{table_id}"
+
+    return (
+        # load the table_config.yaml
+        yaml.load(open(f"{table_path}/table_config.yaml", "r"), Loader=yaml.FullLoader),
+        # return the path to .basedosdados configs
+        configs_path,
+    )
+
+
+def get_table_dataset_id():
+    # load the change files in PR || diff between PR and master
+    changes = Path("files.json").open("r")
+    changes = json.load(changes)
+
+    # create a dict to save the dataset and source_bucket related to each table_id
+    dataset_table_ids = {}
+
+    # create a list to save the table folder path, for each table changed in the commit
+    table_folders = []
+    for change_file in changes:
+        # get the directory path for a table with changes
+        file_dir = Path(change_file).parent
+
+        # append the table directory if it was not already appended
+        if file_dir not in table_folders:
+            table_folders.append(file_dir)
+
+    # construct the iterable for the table_config paths
+    table_config_paths = [Path(root / "table_config.yaml") for root in table_folders]
+
+    # iterate through each config path
+    for filepath in table_config_paths:
+
+        # check if the table_config.yaml exists in the changed folder
+        if filepath.is_file():
+
+            # load the found table_config.yaml
+            table_config = yaml.load(open(filepath, "r"), Loader=yaml.SafeLoader)
+
+            # add the dataset and source_bucket for each table_id
+            dataset_table_ids[table_config["table_id"]] = {
+                "dataset_id": table_config["dataset_id"],
+                "source_bucket_name": table_config["source_bucket_name"],
+            }
+
+    return dataset_table_ids
+
+
+def metadata_validate():
+    # find the dataset and tables of the PR
+    dataset_table_ids = get_table_dataset_id()
+
+    # print dataset tables info
+    tprint("TABLES FOUND")
+    pprint(dataset_table_ids)
+    tprint()
+
+    # iterate over each table in dataset of the PR
+    for table_id in dataset_table_ids.keys():
+        dataset_id = dataset_table_ids[table_id]["dataset_id"]
+        source_bucket_name = dataset_table_ids[table_id]["source_bucket_name"]
+
+        try:
+            # push the table to bigquery
+            md = Metadata(dataset_id=dataset_id, table_id=table_id)
+
+            md.validate()
+            tprint(f"SUCESS VALIDATE {dataset_id}.{table_id}")
+            tprint()
+
+        except Exception as error:
+            tprint(f"ERROR ON {dataset_id}.{table_id}")
+            traceback.print_exc()
+            tprint()
+
+
+if __name__ == "__main__":
+    metadata_validate()
@@ -1,13 +1,15 @@
 name: python-ci
+
 on:
   push:
-    branches:
-      - master
-  pull_request:
+    # branches:
+    #   - master
+    #   - python-1.6.0
     paths:
       - .github/**
       - python-package/**
-  workflow_dispatch:
+    
+
 jobs:
   guard:
     runs-on: ubuntu-latest
@@ -39,12 +41,12 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           python -m pip install isort black pylint
-      - name: Check library sort
-        run: python -m isort --check-only --profile black .github python-package
-      - name: Check code format
-        run: python -m black --check .github python-package
-      - name: Check lint
-        run: python -m pylint --exit-zero .github/**/*.py python-package
+      # - name: Check library sort
+      #   run: python -m isort --check-only --profile black .github python-package
+      # - name: Check code format
+      #   run: python -m black --check .github python-package
+      # - name: Check lint
+      #   run: python -m pylint --exit-zero .github/**/*.py python-package
   build-linux:
     needs: lint
     runs-on: ubuntu-latest
@@ -74,6 +76,8 @@ jobs:
         PROJECT_NAME_STAGING: basedosdados-dev
         GCP_BD_PROD: ${{ secrets.GCP_BD_DEV_PROD }}
         GCP_BD_STAGING: ${{ secrets.GCP_BD_DEV_STAGING }}
+        CKAN_URL: "https://staging.basedosdados.org"
+        CKAN_API_KEY: ${{ secrets.CKAN_STAGING }}
       shell: bash
     - name: Test
       if: github.event_name == 'pull_request'
@@ -121,6 +125,8 @@ jobs:
         PROJECT_NAME_STAGING: basedosdados-dev
         GCP_BD_PROD: ${{ secrets.GCP_BD_DEV_PROD }}
         GCP_BD_STAGING: ${{ secrets.GCP_BD_DEV_STAGING }}
+        CKAN_URL: "https://staging.basedosdados.org"
+        CKAN_API_KEY: ${{ secrets.CKAN_STAGING }}
     - name: Test
       if: github.event_name == 'pull_request'
       run: |