Merge pull request #149 from statisticsnorway/issues_march_2025

Klass XML - Round up - versions and Paths

Author: aecorn

README.md

@@ -90,6 +90,43 @@ logger = logging.getLogger(__name__)
 logger.info("This is an info message")
 ```
 
+
+
+### Export XMLs that can be imported into the KLASS UI
+```python
+from fagfunksjoner import make_klass_xml_codelist
+
+
+make_klass_xml_codelist(path="kjoenn.xml",
+    codes=["1", "2"],
+    names_bokmaal=["Mann", "Kvinne"])
+```
+
+
+### Round data UP
+
+```python
+import pandas as pd
+
+from fagfunksjoner import round_up
+
+
+print(round(2.5, 0), round_up(2.5, 0))
+
+round_up(pd.Series([1.5, 2.5, 3.5]), 0)  # Datatype blir Int64 når man runder til 0 desimaler
+round_up(pd.Series([1.15, 2.15, 3.15]), 1)  # Datatype blir Float64 når man runder til mer enn 0 desimaler
+
+df = pd.DataFrame(
+    {"col1": [1.5, 2.5, 1.2345, 1.2355],
+    "col2": [3.5, 4.5, 5.6789, 6.7891]}
+    ).astype({"col1": "Float64", "col2": "Float64"})
+rounded = round_up(df, decimal_places=0, col_names="col1")  # Avrunder kun col1, den endrer datatype til Int64
+
+rounded2 = round_up(df, col_names={"col1": 1, "col2": 2})  # Avrunder col1 til 1 desimal, col2 til 2 desimaler
+
+```
+
+
 ### Aggregation / Categories
 
 Aggregate on all exclusive combinations of codes in certain columns (maybe before sending to statbank? Like proc means?)
@@ -159,7 +196,7 @@ all_combos_agg_inclusive(
     grand_total=True)
 ```
 
-
+### "Formats" like in SAS
 
 Perform mapping using SsbFormat. Behaves like a dictionary. Has functionality for mapping ranges and 'other'-category and detecting different types of NaN-values. Does not handle non-exclusive / overlapping categories, please only use for exclusive categories.
 
@@ -203,6 +240,7 @@ some_frmt = get_format(path+'format_name.json')
 ```
 
 ### Opening archive-files based on Datadok-api in prodsone
+
 We have "flat files", which are not comma seperated. These need metadata to correctly open. In SAS we do this with "lastescript". But there is an API to old Datadok in prodsone, so these functions let you just specify a path, and attempt to open the flat files directly into pandas, with the metadata also available.
 
 ```python
@@ -220,6 +258,9 @@ archive_object.datatypes  # The datatypes the archivdata ended up having?
 archive_object.widths  # Width of each column in the flat file
 
 ```
+
+
+
 ### Operation to Oracle database
 
 Remember that any credidential values to the database should not be stored

poetry.lock

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ssb-fagfunksjoner"
-version = "1.0.9"
+version = "1.1.0"
 description = "Fellesfunksjoner for ssb i Python"
 authors = ["SSB-pythonistas <ssb-pythonistas@ssb.no>"]
 license = "MIT"
@@ -40,8 +40,8 @@ types-beautifulsoup4 = ">=4.12.0.20240511"
 types-colorama = ">=0.4.15.20240205"
 types-openpyxl = ">=3.1.5.20240719"
 pyarrow-stubs = ">=10.0.1.9"
-nox = "^2025.2.9"
-nox-poetry = "^1.1.0"
+nox = ">=2025.2.9"
+nox-poetry = ">=1.1.0"
 
 [tool.poetry.group.dev.dependencies]
 pygments = ">=2.10.0"

pyproject.toml

@@ -46,16 +46,22 @@ def _try_getting_pyproject_toml(e: Exception | None = None) -> str:
     open_path_datadok,
     open_path_metapath_datadok,
 )
+from fagfunksjoner.data.klass_xml import make_klass_xml_codelist
 from fagfunksjoner.data.pandas_combinations import (
     all_combos_agg,
     all_combos_agg_inclusive,
 )
 from fagfunksjoner.data.pandas_dtypes import auto_dtype
+from fagfunksjoner.data.round_ssb import round_up
 from fagfunksjoner.data.view_dataframe import view_dataframe
 from fagfunksjoner.formats.formats import SsbFormat
 from fagfunksjoner.log.statlogger import StatLogger
 from fagfunksjoner.paths.project_root import ProjectRoot
-from fagfunksjoner.paths.versions import get_latest_fileversions, next_version_path
+from fagfunksjoner.paths.versions import (
+    get_latest_fileversions,
+    latest_version_path,
+    next_version_path,
+)
 from fagfunksjoner.prodsone.check_env import check_env, linux_shortcuts
 from fagfunksjoner.prodsone.saspy_ssb import saspy_df_from_path, saspy_session
 
@@ -69,10 +75,13 @@ def _try_getting_pyproject_toml(e: Exception | None = None) -> str:
     "auto_dtype",
     "check_env",
     "get_latest_fileversions",
+    "latest_version_path",
     "linux_shortcuts",
+    "make_klass_xml_codelist",
     "next_version_path",
     "open_path_datadok",
     "open_path_metapath_datadok",
+    "round_up",
     "saspy_df_from_path",
     "saspy_session",
     "view_dataframe",

src/fagfunksjoner/__init__.py

@@ -0,0 +1,100 @@
+"""This module contains functions to create a xml file that can be loaded in the KLASS UI.
+
+It passes data trhough a pandas DataFrame from a list of codes and names, to an XML from the pandas dataframe.
+"""
+
+import pandas as pd
+
+
+def make_klass_df_codelist(
+    codes: list[str | int],
+    names_bokmaal: list[str] | None = None,
+    names_nynorsk: list[str] | None = None,
+    names_engelsk: list[str] | None = None,
+) -> pd.DataFrame:
+    """Make a pandas Dataframe from lists of codes and names.
+
+    Args:
+        codes: List of codes.
+        names_bokmaal: List of names in Bokmål.
+        names_nynorsk: List of names in Nynorsk.
+        names_engelsk: List of names in English.
+
+    Returns:
+        pd.DataFrame: Dataframe with columns for codes and names.
+
+    Raises:
+        ValueError: If names_bokmaal and names_nynorsk are both None, or if the length of
+            codes and names do not match.
+    """
+    if names_bokmaal is None and names_nynorsk is None:
+        raise ValueError("Must have content in names_bokmaal or names_nynorsk")
+    for name in [names_bokmaal, names_nynorsk, names_engelsk]:
+        if name and len(codes) != len(name):
+            raise ValueError(
+                "Length of the entered names must match the length of codes."
+            )
+
+    cols = [
+        "kode",
+        "forelder",
+        "navn_bokmål",
+        "navn_nynorsk",
+        "navn_engelsk",
+        "kortnavn_bokmål",
+        "kortnavn_nynorsk",
+        "kortnavn_engelsk",
+        "noter_bokmål",
+        "noter_nynorsk",
+        "noter_engelsk",
+        "gyldig_fra",
+        "gyldig_til",
+    ]
+
+    data = {col: [None] * len(codes) for col in cols} | {
+        "kode": codes,
+        "navn_bokmål": names_bokmaal,
+        "navn_nynorsk": names_nynorsk,
+        "navn_engelsk": names_engelsk,
+    }
+
+    return pd.DataFrame({name: data for name, data in data.items()})
+
+
+def make_klass_xml_codelist(
+    path: str,
+    codes: list[str | int],
+    names_bokmaal: list[str] | None = None,
+    names_nynorsk: list[str] | None = None,
+    names_engelsk: list[str] | None = None,
+) -> pd.DataFrame:
+    """Make a klass xml file and pandas Dataframe from a list of codes and names.
+
+    This XML can be loaded into the old KLASS UI under version -> import to the top right.
+
+    Args:
+        path (str): Path to save the xml file.
+        codes (list[str|int]): List of codes.
+        names_bokmaal (list[str] | None): List of names in Bokmål.
+        names_nynorsk (list[str] | None): List of names in Nynorsk.
+        names_engelsk (list[str] | None): List of names in English.
+
+    Returns:
+        pd.DataFrame: Dataframe with columns for codes and names.
+    """
+    df = make_klass_df_codelist(
+        codes=codes,
+        names_bokmaal=names_bokmaal,
+        names_nynorsk=names_nynorsk,
+        names_engelsk=names_engelsk,
+    )
+    df.to_xml(
+        path,
+        root_name="versjon",
+        row_name="element",
+        namespaces={
+            "ns1": "https://klass.ssb.no/version",
+        },
+        prefix="ns1",
+    )
+    return df

src/fagfunksjoner/data/klass_xml.py

@@ -0,0 +1,135 @@
+"""Reproduce the functionality of the default round function from Excel or SAS, rounding data up to a given number of decimal places.
+
+Instead of Python's default of rounding to even.
+"""
+
+from decimal import ROUND_HALF_UP, Decimal, localcontext
+from typing import TYPE_CHECKING, Any, overload
+
+import pandas as pd
+
+
+# Alias for type checking
+if TYPE_CHECKING:
+    pd_Series = pd.Series[Any]
+else:
+    pd_Series = (
+        object  # Fallback to avoid runtime issues where pd.Series is not subscriptable
+    )
+
+
+# Overloads, output type is dependent on input type
+@overload
+def round_up(data: pd.DataFrame, decimal_places: int) -> pd.DataFrame: ...
+@overload
+def round_up(data: pd_Series, decimal_places: int) -> pd_Series: ...
+
+
+# Mypy does not like getting specific with Literal[0], thats too bad
+@overload
+def round_up(data: int | float, decimal_places: int) -> int | float: ...
+@overload
+def round_up(
+    data: pd._libs.missing.NAType, decimal_places: int
+) -> pd._libs.missing.NAType: ...
+
+
+def round_up(
+    data: pd.DataFrame | pd_Series | float | pd._libs.missing.NAType,
+    decimal_places: int = 0,
+    col_names: str | list[str] | dict[str, int] = "",
+) -> pd.DataFrame | pd_Series | int | float | pd._libs.missing.NAType:
+    """Round up a number, to a given number of decimal places. Avoids Pythons default of rounding to even.
+
+    Args:
+        data: The data to round up, can be a float, Series, or DataFrame.
+        decimal_places: The number of decimal places to round up to. Ignored if you send a dictionary into col_names with column names and decimal places.
+        col_names: The column names to round up. If a dictionary is provided, it should map column names to the number of decimal places for each column.
+            If a list is provided, it should contain the names of the columns to round up. If a string is provided, it should be the name of a single column to round up.
+
+    Returns:
+         pd.DataFrame | pd.Series | int | float: The rounded up number as an int, float, Series, or DataFrame.
+
+    Raises:
+        TypeError: If data is not a DataFrame, Series, int, float, or NAType.
+    """
+    if isinstance(data, pd.DataFrame):
+        if isinstance(col_names, dict):
+            # Assuming col_names is a dictionary with column names as keys and decimal places as values
+            for col, dec in col_names.items():
+                data = _apply_rounding_to_df_col(data, col, dec)
+        elif isinstance(col_names, list):
+            # Assuming col_names is a list of column names
+            for col in col_names:
+                data = _apply_rounding_to_df_col(data, col, decimal_places)
+        elif isinstance(col_names, str):
+            # Assuming col_names is a single column name
+            data = _apply_rounding_to_df_col(data, col_names, decimal_places)
+    elif isinstance(data, pd.Series):
+        # If data is a Series, round it directly
+        data = _set_dtype_from_decimal_places(
+            data.apply(_round, decimals=decimal_places), decimal_places
+        )
+    elif isinstance(data, int | float | pd._libs.missing.NAType):
+        data = _round(data, decimals=decimal_places)
+    else:
+        raise TypeError(
+            "data must be a DataFrame, Series, int, float, or NAType. "
+            f"Got {type(data)} instead."
+        )
+    return data
+
+
+def _apply_rounding_to_df_col(
+    df: pd.DataFrame, col_name: str, decimal_places: int
+) -> pd.DataFrame:
+    """Apply rounding to a specific column in a DataFrame.
+
+    Args:
+        df: The DataFrame to round.
+        col_name: The name of the column to round.
+        decimal_places: The number of decimal places to round to.
+
+    Returns:
+        pd.DataFrame: The DataFrame with the rounded column.
+    """
+    if col_name in df.columns:
+        df[col_name] = _set_dtype_from_decimal_places(
+            df[col_name].apply(_round, decimals=decimal_places), decimal_places
+        )
+    return df
+
+
+def _set_dtype_from_decimal_places(
+    data: pd_Series,
+    decimal_places: int = 0,
+) -> pd_Series:
+    """Set the dtype of the data based on the number of decimal places.
+
+    Args:
+        data: The column to set the dtype for.
+        decimal_places: The number of decimal places.
+
+    Returns:
+        pd_Series: The data with the updated dtype.
+    """
+    if decimal_places == 0:
+        return data.astype("Int64")
+    else:
+        return data.astype("Float64")
+
+
+def _round(
+    n: float | pd._libs.missing.NAType,
+    decimals: int = 0,
+) -> float | int | pd._libs.missing.NAType:
+    if pd.isna(n):
+        return pd.NA
+    elif n or n == 0:
+        with localcontext() as ctx:
+            ctx.rounding = ROUND_HALF_UP
+            rounded = round(Decimal(n), decimals)
+            if decimals == 0:
+                return int(Decimal(rounded).to_integral_value())
+            return float(rounded)
+    return n

src/fagfunksjoner/data/round_ssb.py

@@ -1,4 +1,5 @@
-# +
+"""Extract user information from the environment."""
+
 import getpass
 import os
 import subprocess
@@ -63,6 +64,3 @@ def verify_ssbmail(user: str | None) -> str | None:
     if "@" not in user and len(user) == 3:
         return user + "@ssb.no"
     return None
-
-
-# -