Add include_unofficial option (#72)

Author: mcarans

documentation/index.md

@@ -70,9 +70,10 @@ The usage of the country mappings functionality is best illustrated by some exam
 
     from hdx.location.country import Country
 
-    Country.countriesdata(use_live=False, country_name_overrides={"PSE": "oPt"})
-    # Set up using non live data from repo rather and override default country name
-    # (Leaving out this step will use live data and no overrides)
+    Country.countriesdata(include_unofficial=True, use_live=False, country_name_overrides={"PSE": "oPt"})
+    # Set up including unofficial alpha2 (eg. AN) and alpha3 codes (eg. XKX)
+    # Use non live data from repo and override default country name
+    # (Leaving out this step will exclude unofficial alpha codes, use live data and no overrides)
     Country.get_country_name_from_iso3("jpn", use_live=False)  # returns "Japan"
     Country.get_country_name_from_iso3("vEn", formal=True)  # returns "the Bolivarian Republic of Venezuela"
     # uselive=False forces the use of internal files instead of accessing the live feeds.

pyproject.toml

@@ -34,7 +34,7 @@ classifiers = [
 requires-python = ">=3.8"
 
 dependencies = [
-    "hdx-python-utilities>=3.8.6",
+    "hdx-python-utilities>=3.8.7",
     "libhxl>=5.2.2",
     "tenacity",
 ]

requirements.txt

@@ -30,8 +30,6 @@ colorama==0.4.6
     # via mkdocs-material
 coverage==7.8.0
     # via pytest-cov
-cydifflib==1.2.0
-    # via hdx-python-utilities
 distlib==0.3.9
     # via virtualenv
 et-xmlfile==2.0.0
@@ -42,15 +40,15 @@ frictionless==5.18.1
     # via hdx-python-utilities
 ghp-import==2.1.0
     # via mkdocs
-hdx-python-utilities==3.8.6
+hdx-python-utilities==3.8.7
     # via hdx-python-country (pyproject.toml)
 humanize==4.12.3
     # via frictionless
 identify==2.6.10
     # via pre-commit
 idna==3.10
     # via requests
-ijson==3.3.0
+ijson==3.4.0
     # via hdx-python-utilities
 iniconfig==2.1.0
     # via pytest
@@ -95,15 +93,15 @@ mergedeep==1.3.4
     # via
     #   mkdocs
     #   mkdocs-get-deps
-mkapi==4.3.2
+mkapi==4.4.0
     # via hdx-python-country (pyproject.toml)
 mkdocs==1.6.1
     # via
     #   mkapi
     #   mkdocs-material
 mkdocs-get-deps==0.2.0
     # via mkdocs
-mkdocs-material==9.6.12
+mkdocs-material==9.6.14
     # via mkapi
 mkdocs-material-extensions==1.3.1
     # via mkdocs-material
@@ -121,11 +119,11 @@ pathspec==0.12.1
     # via mkdocs
 petl==1.7.16
     # via frictionless
-platformdirs==4.3.7
+platformdirs==4.3.8
     # via
     #   mkdocs-get-deps
     #   virtualenv
-pluggy==1.5.0
+pluggy==1.6.0
     # via pytest
 ply==3.11
     # via
@@ -170,7 +168,7 @@ pyyaml==6.0.2
     #   pymdown-extensions
     #   pyyaml-env-tag
     #   tableschema-to-template
-pyyaml-env-tag==0.1
+pyyaml-env-tag==1.1
     # via mkdocs
 ratelimit==2.2.1
     # via hdx-python-utilities
@@ -190,7 +188,7 @@ rfc3986==2.0.0
     # via frictionless
 rich==14.0.0
     # via typer
-rpds-py==0.24.0
+rpds-py==0.25.0
     # via
     #   jsonschema
     #   referencing
@@ -214,7 +212,7 @@ tenacity==9.1.2
     # via hdx-python-country (pyproject.toml)
 text-unidecode==1.3
     # via python-slugify
-typer==0.15.3
+typer==0.15.4
     # via frictionless
 typing-extensions==4.13.2
     # via
@@ -236,7 +234,7 @@ urllib3==2.4.0
     #   requests
 validators==0.35.0
     # via frictionless
-virtualenv==20.31.1
+virtualenv==20.31.2
     # via pre-commit
 watchdog==6.0.0
     # via mkdocs

src/hdx/location/Countries & Territories Taxonomy MVP - C&T Taxonomy with HXL Tags.csv

@@ -8,7 +8,7 @@
 from typing import Dict, List, Optional, Tuple, Union
 
 import hxl
-from hxl import InputOptions
+from hxl import Dataset, InputOptions
 
 from hdx.utilities.path import script_dir_plus_file
 from hdx.utilities.text import get_words_in_sentence
@@ -69,6 +69,8 @@ class Country:
         "KINGDOM",
         "PROTECTORATE",
     ]
+    _include_unofficial_default = False
+    _include_unofficial = _include_unofficial_default
     _use_live_default = True
     _use_live = _use_live_default
     _countriesdata = None
@@ -102,6 +104,8 @@ def _add_countriesdata(cls, iso3: str, hxlcountry: hxl.Row) -> Dict:
         if countryname is not None:
             country["#country+name+override"] = countryname
         iso2 = hxlcountry.get("#country+code+v_iso2")
+        if not iso2 and cls._include_unofficial:
+            iso2 = hxlcountry.get("#country+code+alpha2")
         if iso2:
             cls._countriesdata["iso2iso3"][iso2] = iso3
             # different types so keys won't clash
@@ -159,12 +163,12 @@ def add_country_to_set(colname, idval, iso3):
         return country
 
     @classmethod
-    def set_countriesdata(cls, countries: str) -> None:
+    def set_countriesdata(cls, countries: Dataset) -> None:
         """
-        Set up countries data from data in form provided by UNStats and World Bank
+        Set up countries data from OCHA countries and territories dataset
 
         Args:
-            countries (str): Countries data in HTML format provided by UNStats
+            countries (Dataset): Countries data from countries and territories dataset
 
         Returns:
             None
@@ -186,7 +190,12 @@ def set_countriesdata(cls, countries: str) -> None:
         for country in countries:
             iso3 = country.get("#country+code+v_iso3")
             if not iso3:
-                continue
+                if cls._include_unofficial:
+                    iso3 = country.get("#country+code+alpha3")
+                    if not iso3:
+                        continue
+                else:
+                    continue
             iso3 = iso3.upper()
             countrydict = cls._add_countriesdata(iso3, country)
             cls._countriesdata["countries"][iso3] = countrydict
@@ -202,30 +211,37 @@ def sort_list(colname):
     @classmethod
     def countriesdata(
         cls,
-        use_live: bool = None,
-        country_name_overrides: Dict = None,
-        country_name_mappings: Dict = None,
+        include_unofficial: Optional[bool] = None,
+        use_live: Optional[bool] = None,
+        country_name_overrides: Optional[Dict] = None,
+        country_name_mappings: Optional[Dict] = None,
     ) -> List[Dict[str, Dict]]:
         """
-        Read countries data from OCHA countries feed (falling back to file)
+        Read countries data from OCHA countries feed (falling back to file).
+        include_unofficial, use_live, country_name_overrides and country_name_mappings
+        are taken from internal defaults if they are None, otherwise the internal
+        defaults are overridden.
 
         Args:
-            use_live (bool): Try to get use latest data from web rather than file in package. Defaults to True.
-            country_name_overrides (Dict): Dictionary of mappings from iso3 to country name
-            country_name_mappings (Dict): Dictionary of mappings from country name to iso3
+            include_unofficial (Optional[bool]): Include unofficial country alpha codes. Defaults to False.
+            use_live (Optional[bool]): Try to get latest data from web rather than file in package. Defaults to True.
+            country_name_overrides (Optional[Dict]): Dictionary of mappings from iso3 to country name
+            country_name_mappings (Optional[Dict]): Dictionary of mappings from country name to iso3
 
         Returns:
             List[Dict[str,Dict]]: Countries dictionaries
         """
-        if use_live is None:
-            use_live = cls._use_live
+        if include_unofficial is not None:
+            cls._include_unofficial = include_unofficial
+        if use_live is not None:
+            cls._use_live = use_live
         if cls._countriesdata is None:
             countries = None
             if country_name_overrides is not None:
                 cls.set_country_name_overrides(country_name_overrides)
             if country_name_mappings is not None:
                 cls.set_country_name_mappings(country_name_mappings)
-            if use_live:
+            if cls._use_live:
                 try:
                     countries = hxl.data(cls._ochaurl, InputOptions(encoding="utf-8"))
                 except OSError:
@@ -240,6 +256,24 @@ def countriesdata(
             cls.set_countriesdata(countries)
         return cls._countriesdata
 
+    @classmethod
+    def set_include_unofficial_default(
+        cls, include_unofficial: Optional[bool] = None
+    ) -> None:
+        """
+        Set the default for include_unofficial which defines if unofficial alpha2 and
+        alpha3 codes such as AN and XKX will be included.
+
+        Args:
+            include_unofficial (bool): Default value to use for include_unofficial. Defaults to internal value (False).
+
+        Returns:
+            None
+        """
+        if include_unofficial is None:
+            include_unofficial = cls._include_unofficial
+        cls._include_unofficial = include_unofficial
+
     @classmethod
     def set_use_live_default(cls, use_live: Optional[bool] = None) -> None:
         """