From 1db95ef39cbf23d893443ef2a9d0f3a5eb36580e Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Wed, 25 Sep 2024 06:47:55 +0000 Subject: [PATCH 01/10] chore(vscode): turn off pyright typechecking --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 31f2bba..208a1ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,7 @@ source = ["compiler_admin"] [tool.pyright] include = ["compiler_admin", "tests"] +typeCheckingMode = "off" [tool.pytest.ini_options] testpaths = ["tests"] From b29df21ab807a1ccee4b145bf4ae937902f50ef6 Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Wed, 25 Sep 2024 06:33:58 +0000 Subject: [PATCH 02/10] feat(toggl): helper config and functions for the Toggl API --- .env.sample | 4 +++ compiler_admin/services/toggl.py | 56 ++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/.env.sample b/.env.sample index de2b222..4db21a7 100644 --- a/.env.sample +++ b/.env.sample @@ -4,3 +4,7 @@ HARVEST_DATA=data/harvest-sample.csv TOGGL_DATA=data/toggl-sample.csv TOGGL_PROJECT_INFO=data/toggl-project-info-sample.json TOGGL_USER_INFO=data/toggl-user-info-sample.json + +TOGGL_API_TOKEN=token +TOGGL_CLIENT_ID=client +TOGGL_WORKSPACE_ID=workspace diff --git a/compiler_admin/services/toggl.py b/compiler_admin/services/toggl.py index 9b39061..98d8447 100644 --- a/compiler_admin/services/toggl.py +++ b/compiler_admin/services/toggl.py @@ -1,12 +1,19 @@ +from base64 import b64encode import os import sys from typing import TextIO import pandas as pd +from compiler_admin import __version__ from compiler_admin.services.google import user_info as google_user_info import compiler_admin.services.files as files +# Toggl API config +API_BASE_URL = "https://api.track.toggl.com" +API_REPORTS_BASE_URL = "reports/api/v3" +API_WORKSPACE = "workspace/{}" + # cache of previously seen project information, keyed on Toggl project name PROJECT_INFO = {} @@ -36,6 +43,50 @@ def _get_info(obj: dict, key: str, env_key: str): return obj.get(key) +def _toggl_api_authorization_header(): + """Gets an `Authorization: Basic xyz` header using the Toggl API token. + + See https://engineering.toggl.com/docs/authentication. + """ + token = _toggl_api_token() + creds = f"{token}:api_token" + creds64 = b64encode(bytes(creds, "utf-8")).decode("utf-8") + return {"Authorization": "Basic {}".format(creds64)} + + +def _toggl_api_headers(): + """Gets a dict of headers for Toggl API requests. + + See https://engineering.toggl.com/docs/. + """ + headers = {"Content-Type": "application/json"} + headers.update({"User-Agent": "compilerla/compiler-admin:{}".format(__version__)}) + headers.update(_toggl_api_authorization_header()) + return headers + + +def _toggl_api_report_url(endpoint: str): + """Get a fully formed URL for the Toggl Reports API v3 endpoint. + + See https://engineering.toggl.com/docs/reports_start. + """ + workspace_id = _toggl_workspace() + return "/".join((API_BASE_URL, API_REPORTS_BASE_URL, API_WORKSPACE.format(workspace_id), endpoint)) + + +def _toggl_api_token(): + """Gets the value of the TOGGL_API_TOKEN env var.""" + return os.environ.get("TOGGL_API_TOKEN") + + +def _toggl_client_id(): + """Gets the value of the TOGGL_CLIENT_ID env var.""" + client_id = os.environ.get("TOGGL_CLIENT_ID") + if client_id: + return int(client_id) + return None + + def _toggl_project_info(project: str): """Return the cached project for the given project key.""" return _get_info(PROJECT_INFO, project, "TOGGL_PROJECT_INFO") @@ -46,6 +97,11 @@ def _toggl_user_info(email: str): return _get_info(USER_INFO, email, "TOGGL_USER_INFO") +def _toggl_workspace(): + """Gets the value of the TOGGL_WORKSPACE_ID env var.""" + return os.environ.get("TOGGL_WORKSPACE_ID") + + def _get_first_name(email: str) -> str: """Get cached first name or derive from email.""" user = _toggl_user_info(email) From 439e4f51a2106445aa4a0f7abf340ecbcc1b77ec Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Wed, 25 Sep 2024 06:46:54 +0000 Subject: [PATCH 03/10] feat(toggl): download a detailed time entry CSV report --- compiler_admin/services/toggl.py | 71 ++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/compiler_admin/services/toggl.py b/compiler_admin/services/toggl.py index 98d8447..7f0f133 100644 --- a/compiler_admin/services/toggl.py +++ b/compiler_admin/services/toggl.py @@ -1,9 +1,12 @@ from base64 import b64encode +from datetime import datetime +import io import os import sys from typing import TextIO import pandas as pd +import requests from compiler_admin import __version__ from compiler_admin.services.google import user_info as google_user_info @@ -183,3 +186,71 @@ def convert_to_harvest( source["Hours"] = (source["Duration"].dt.total_seconds() / 3600).round(2) files.write_csv(output_path, source, columns=output_cols) + + +def download_time_entries( + start_date: datetime, + end_date: datetime, + output_path: str | TextIO = sys.stdout, + output_cols: list[str] | None = INPUT_COLUMNS, + **kwargs, +): + """Download a CSV report from Toggl of detailed time entries for the given date range. + + Args: + start_date (datetime): The beginning of the reporting period. + + end_date (str): The end of the reporting period. + + output_path: The path to a CSV file where Toggl time entries will be written; or a writeable buffer for the same. + + output_cols (list[str]): A list of column names for the output. + + Extra kwargs are passed along in the POST request body. + + By default, requests a report with the following configuration: + * `billable=True` + * `client_ids=[$TOGGL_CLIENT_ID]` + * `rounding=1` (True, but this is an int param) + * `rounding_minutes=15` + + See https://engineering.toggl.com/docs/reports/detailed_reports#post-export-detailed-report. + + Returns: + None. Either prints the resulting CSV data or writes to output_path. + """ + start = start_date.strftime("%Y-%m-%d") + end = end_date.strftime("%Y-%m-%d") + + if ("client_ids" not in kwargs or not kwargs["client_ids"]) and isinstance(_toggl_client_id(), int): + kwargs["client_ids"] = [_toggl_client_id()] + + params = dict( + billable=True, + start_date=start, + end_date=end, + rounding=1, + rounding_minutes=15, + ) + params.update(kwargs) + + headers = _toggl_api_headers() + url = _toggl_api_report_url("search/time_entries.csv") + + response = requests.post(url, json=params, headers=headers, timeout=5) + response.raise_for_status() + + # the raw response has these initial 3 bytes: + # + # b"\xef\xbb\xbfUser,Email,Client..." + # + # \xef\xbb\xb is the Byte Order Mark (BOM) sometimes used in unicode text files + # these 3 bytes indicate a utf-8 encoded text file + # + # See more + # - https://en.wikipedia.org/wiki/Byte_order_mark + # - https://stackoverflow.com/a/50131187 + csv = response.content.decode("utf-8-sig") + + df = pd.read_csv(io.StringIO(csv)) + files.write_csv(output_path, df, columns=output_cols) From 9d73c630f954e56881e0fe11833ff73ec58342d2 Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Wed, 25 Sep 2024 06:51:08 +0000 Subject: [PATCH 04/10] feat(notebooks): download and convert process --- notebooks/download-and-convert.ipynb | 64 ++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 notebooks/download-and-convert.ipynb diff --git a/notebooks/download-and-convert.ipynb b/notebooks/download-and-convert.ipynb new file mode 100644 index 0000000..79777ad --- /dev/null +++ b/notebooks/download-and-convert.ipynb @@ -0,0 +1,64 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from datetime import datetime\n", + "\n", + "from compiler_admin.services.toggl import download_time_entries, convert_to_harvest" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "start = datetime(2024, 9, 1)\n", + "end = datetime(2024, 9, 24)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "download_time_entries(start, end, \"data/time_entries.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "convert_to_harvest(\"data/time_entries.csv\", \"data/time_entries_harvest.csv\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 4aa818ab76f9d38cc19b3e4fed0a09df6d950fde Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Wed, 25 Sep 2024 19:16:51 +0000 Subject: [PATCH 05/10] chore(tests): add pytest-socket to block connections we don't want/need our tests making real calls to the internet --- pyproject.toml | 3 ++- tests/__init__.py | 0 tests/conftest.py | 5 +++++ 3 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 tests/__init__.py diff --git a/pyproject.toml b/pyproject.toml index 208a1ef..775ee54 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,8 @@ dev = [ test = [ "coverage", "pytest", - "pytest-mock" + "pytest-mock", + "pytest-socket" ] [project.scripts] diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py index a24ad3c..7dedd70 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,8 +1,13 @@ import pytest +from pytest_socket import disable_socket from compiler_admin import RESULT_SUCCESS +def pytest_runtest_setup(): + disable_socket() + + @pytest.fixture def mock_module_name(mocker): """Fixture returns a function taking a name, that returns a function taking a module, From be99fbe24fdf1ac277f90ed9d67fb428cab79617 Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Wed, 25 Sep 2024 19:56:27 +0000 Subject: [PATCH 06/10] test(toggl): download_time_entries with a mock report --- tests/services/test_toggl.py | 57 ++++++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/tests/services/test_toggl.py b/tests/services/test_toggl.py index b14621d..e52660c 100644 --- a/tests/services/test_toggl.py +++ b/tests/services/test_toggl.py @@ -1,6 +1,7 @@ +from datetime import datetime, timedelta +from io import BytesIO, StringIO import sys -from datetime import timedelta -from io import StringIO +from pathlib import Path from tempfile import NamedTemporaryFile import pandas as pd @@ -22,6 +23,7 @@ _get_last_name, _str_timedelta, convert_to_harvest, + download_time_entries, ) @@ -57,6 +59,11 @@ def mock_google_user_info(mocker): return mocker.patch(f"{MODULE}.google_user_info") +@pytest.fixture +def mock_requests(mocker): + return mocker.patch(f"{MODULE}.requests") + + def test_harvest_client_name(monkeypatch): assert _harvest_client_name() == "Test_Client" @@ -212,3 +219,49 @@ def test_convert_to_harvest_sample(toggl_file, harvest_file, mock_google_user_in assert set(output_df.columns.to_list()) <= set(sample_output_df.columns.to_list()) assert output_df["Client"].eq("Test Client 123").all() + + +def test_download_time_entries(monkeypatch, toggl_file, mock_requests, mocker): + monkeypatch.setenv("TOGGL_API_TOKEN", "token") + monkeypatch.setenv("TOGGL_CLIENT_ID", "1234") + monkeypatch.setenv("TOGGL_WORKSPACE_ID", "workspace") + + # setup a mock response to a requests.post call + mock_csv_bytes = Path(toggl_file).read_bytes() + mock_post_response = mocker.Mock() + mock_post_response.raise_for_status.return_value = None + # prepend the BOM to the mock content + mock_post_response.content = b"\xef\xbb\xbf" + mock_csv_bytes + # override the requests.post call to return the mock response + mock_requests.post.return_value = mock_post_response + + dt = datetime.now() + + with NamedTemporaryFile("w") as temp: + download_time_entries(dt, dt, temp.name, extra_1=1, extra_2="two") + + called_params = mock_requests.post.call_args.kwargs["json"] + assert isinstance(called_params, dict) + assert called_params["billable"] is True + assert called_params["client_ids"] == [1234] + assert called_params["end_date"] == dt.strftime("%Y-%m-%d") + assert called_params["extra_1"] == 1 + assert called_params["extra_2"] == "two" + assert called_params["rounding"] == 1 + assert called_params["rounding_minutes"] == 15 + assert called_params["start_date"] == dt.strftime("%Y-%m-%d") + + temp.flush() + response_csv_bytes = Path(temp.name).read_bytes() + + # load each CSV into a DataFrame + mock_df = pd.read_csv(BytesIO(mock_csv_bytes)) + response_df = pd.read_csv(BytesIO(response_csv_bytes)) + + # check that the response DataFrame has all columns from the mock DataFrame + assert set(response_df.columns.to_list()).issubset(mock_df.columns.to_list()) + + # check that all column values from response DataFrame are the same + # as corresponding column values from the mock DataFrame + for col in response_df.columns: + assert response_df[col].equals(mock_df[col]) From deced79347bcb7c0d4ed0fcc4fb90f68071f3a32 Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Wed, 25 Sep 2024 20:32:37 +0000 Subject: [PATCH 07/10] feat: helpers to calculate prior (local) month range by default use the America/Los_Angeles time zone override with TZ_NAME env var --- .env.sample | 2 ++ compiler_admin/main.py | 22 ++++++++++++++++++++++ pyproject.toml | 1 + tests/test_main.py | 32 +++++++++++++++++++++++++++++++- 4 files changed, 56 insertions(+), 1 deletion(-) diff --git a/.env.sample b/.env.sample index 4db21a7..9959b54 100644 --- a/.env.sample +++ b/.env.sample @@ -8,3 +8,5 @@ TOGGL_USER_INFO=data/toggl-user-info-sample.json TOGGL_API_TOKEN=token TOGGL_CLIENT_ID=client TOGGL_WORKSPACE_ID=workspace + +TZ_NAME=America/Los_Angeles diff --git a/compiler_admin/main.py b/compiler_admin/main.py index a5d9fa5..f4a398a 100644 --- a/compiler_admin/main.py +++ b/compiler_admin/main.py @@ -1,6 +1,10 @@ from argparse import ArgumentParser, _SubParsersAction +from datetime import datetime, timedelta +import os import sys +from pytz import timezone + from compiler_admin import __version__ as version from compiler_admin.commands.info import info from compiler_admin.commands.init import init @@ -9,6 +13,24 @@ from compiler_admin.commands.user.convert import ACCOUNT_TYPE_OU +TZINFO = timezone(os.environ.get("TZ_NAME", "America/Los_Angeles")) + + +def local_now(): + return datetime.now(tz=TZINFO) + + +def prior_month_end(): + now = local_now() + first = now.replace(day=1) + return first - timedelta(days=1) + + +def prior_month_start(): + end = prior_month_end() + return end.replace(day=1) + + def add_sub_cmd_parser(parser: ArgumentParser, dest="subcommand", help=None): """Helper adds a subparser for the given dest.""" return parser.add_subparsers(dest=dest, help=help) diff --git a/pyproject.toml b/pyproject.toml index 775ee54..7537122 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ requires-python = ">=3.11" dependencies = [ "advanced-gam-for-google-workspace @ git+https://github.com/taers232c/GAMADV-XTD3.git@v7.00.05#subdirectory=src", "pandas==2.2.3", + "tzdata", ] [project.urls] diff --git a/tests/test_main.py b/tests/test_main.py index 4ae00d3..ab1d2ec 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,14 +1,32 @@ from argparse import Namespace +from datetime import datetime import subprocess import sys import pytest import compiler_admin.main -from compiler_admin.main import main, __name__ as MODULE +from compiler_admin.main import main, prior_month_start, prior_month_end, TZINFO, __name__ as MODULE from compiler_admin.services.google import DOMAIN +@pytest.fixture +def mock_local_now(mocker): + dt = datetime(2024, 9, 25, tzinfo=TZINFO) + mocker.patch(f"{MODULE}.local_now", return_value=dt) + return dt + + +@pytest.fixture +def mock_start(mock_local_now): + return datetime(2024, 8, 1, tzinfo=TZINFO) + + +@pytest.fixture +def mock_end(mock_local_now): + return datetime(2024, 8, 31, tzinfo=TZINFO) + + @pytest.fixture def mock_commands_info(mock_commands_info): return mock_commands_info(MODULE) @@ -29,6 +47,18 @@ def mock_commands_user(mock_commands_user): return mock_commands_user(MODULE) +def test_prior_month_start(mock_start): + start = prior_month_start() + + assert start == mock_start + + +def test_prior_month_end(mock_end): + end = prior_month_end() + + assert end == mock_end + + def test_main_info(mock_commands_info): main(argv=["info"]) From 957729e2bd48b16fe30c9a8e0da31974b2994e35 Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Wed, 25 Sep 2024 20:40:49 +0000 Subject: [PATCH 08/10] feat(time): wire up download command compiler-admin time download --start YYYY-MM-DD --end YYYY-MM-DD --output path/to/file.csv small fix on convert test assertion --- README.md | 24 +++++++++++++--- compiler_admin/commands/time/__init__.py | 1 + compiler_admin/commands/time/download.py | 10 +++++++ compiler_admin/main.py | 19 +++++++++++++ tests/commands/time/test_download.py | 21 ++++++++++++++ tests/commands/user/test_convert.py | 2 +- tests/test_main.py | 35 ++++++++++++++++++++++++ 7 files changed, 107 insertions(+), 5 deletions(-) create mode 100644 compiler_admin/commands/time/download.py create mode 100644 tests/commands/time/test_download.py diff --git a/README.md b/README.md index 152d831..13be6f9 100644 --- a/README.md +++ b/README.md @@ -62,14 +62,30 @@ The `time` command provides an interface for working with time entries from Comp ```bash $ compiler-admin time -h -usage: compiler-admin time [-h] {convert} ... +usage: compiler-admin time [-h] {convert,download} ... positional arguments: - {convert} The time command to run. - convert Convert a time report from one format into another. + {convert,download} The time command to run. + convert Convert a time report from one format into another. + download Download a Toggl report in CSV format. options: - -h, --help show this help message and exit + -h, --help show this help message and exit +``` + +### Downloading a Toggl report + +Use this command to download a time report from Toggl in CSV format: + +```bash +$ compiler-admin time download -h +usage: compiler-admin time download [-h] [--start YYYY-MM-DD] [--end YYYY-MM-DD] [--output OUTPUT] + +options: + -h, --help show this help message and exit + --start YYYY-MM-DD The start date of the reporting period. Defaults to the beginning of the prior month. + --end YYYY-MM-DD The end date of the reporting period. Defaults to the end of the prior month. + --output OUTPUT The path to the file where converted data should be written. Defaults to stdout. ``` ### Converting an hours report diff --git a/compiler_admin/commands/time/__init__.py b/compiler_admin/commands/time/__init__.py index b1f9bc7..a56c97d 100644 --- a/compiler_admin/commands/time/__init__.py +++ b/compiler_admin/commands/time/__init__.py @@ -1,6 +1,7 @@ from argparse import Namespace from compiler_admin.commands.time.convert import convert # noqa: F401 +from compiler_admin.commands.time.download import download # noqa: F401 def time(args: Namespace, *extra): diff --git a/compiler_admin/commands/time/download.py b/compiler_admin/commands/time/download.py new file mode 100644 index 0000000..0849c53 --- /dev/null +++ b/compiler_admin/commands/time/download.py @@ -0,0 +1,10 @@ +from argparse import Namespace + +from compiler_admin import RESULT_SUCCESS +from compiler_admin.services.toggl import INPUT_COLUMNS as TOGGL_COLUMNS, download_time_entries + + +def download(args: Namespace, *extras): + download_time_entries(args.start, args.end, args.output, TOGGL_COLUMNS) + + return RESULT_SUCCESS diff --git a/compiler_admin/main.py b/compiler_admin/main.py index f4a398a..b07cacb 100644 --- a/compiler_admin/main.py +++ b/compiler_admin/main.py @@ -76,6 +76,25 @@ def setup_time_command(cmd_parsers: _SubParsersAction): ) time_convert.add_argument("--client", default=None, help="The name of the client to use in converted data.") + time_download = add_sub_cmd(time_subcmds, "download", help="Download a Toggl report in CSV format.") + time_download.add_argument( + "--start", + metavar="YYYY-MM-DD", + default=prior_month_start(), + type=lambda s: TZINFO.localize(datetime.strptime(s, "%Y-%m-%d")), + help="The start date of the reporting period. Defaults to the beginning of the prior month.", + ) + time_download.add_argument( + "--end", + metavar="YYYY-MM-DD", + default=prior_month_end(), + type=lambda s: TZINFO.localize(datetime.strptime(s, "%Y-%m-%d")), + help="The end date of the reporting period. Defaults to the end of the prior month.", + ) + time_download.add_argument( + "--output", default=sys.stdout, help="The path to the file where converted data should be written. Defaults to stdout." + ) + def setup_user_command(cmd_parsers: _SubParsersAction): user_cmd = add_sub_cmd(cmd_parsers, "user", help="Work with users in the Compiler org.") diff --git a/tests/commands/time/test_download.py b/tests/commands/time/test_download.py new file mode 100644 index 0000000..c8abb2a --- /dev/null +++ b/tests/commands/time/test_download.py @@ -0,0 +1,21 @@ +from argparse import Namespace +from datetime import datetime +import pytest + +from compiler_admin import RESULT_SUCCESS +from compiler_admin.commands.time.download import __name__ as MODULE, download, TOGGL_COLUMNS + + +@pytest.fixture +def mock_download_time_entries(mocker): + return mocker.patch(f"{MODULE}.download_time_entries") + + +def test_download_default(mock_download_time_entries): + date = datetime.now() + args = Namespace(start=date, end=date, output="output") + + res = download(args) + + assert res == RESULT_SUCCESS + mock_download_time_entries.assert_called_once_with(args.start, args.end, args.output, TOGGL_COLUMNS) diff --git a/tests/commands/user/test_convert.py b/tests/commands/user/test_convert.py index bac0c2c..8440f6b 100644 --- a/tests/commands/user/test_convert.py +++ b/tests/commands/user/test_convert.py @@ -172,7 +172,7 @@ def test_convert_partner(mock_google_add_user_to_group, mock_google_move_user_ou res = convert(args) assert res == RESULT_SUCCESS - mock_google_add_user_to_group.call_count == 2 + assert mock_google_add_user_to_group.call_count == 2 mock_google_move_user_ou.assert_called_once() diff --git a/tests/test_main.py b/tests/test_main.py index ab1d2ec..d70a214 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -114,6 +114,41 @@ def test_main_time_convert_default(mock_commands_time): ) +@pytest.mark.usefixtures("mock_local_now") +def test_main_time_download_default(mock_commands_time, mock_start, mock_end): + main(argv=["time", "download"]) + + mock_commands_time.assert_called_once() + call_args = mock_commands_time.call_args.args + assert ( + Namespace( + func=mock_commands_time, command="time", subcommand="download", start=mock_start, end=mock_end, output=sys.stdout + ) + in call_args + ) + + +def test_main_time_download_args(mock_commands_time): + main(argv=["time", "download", "--start", "2024-01-01", "--end", "2024-01-31", "--output", "file.csv"]) + + expected_start = TZINFO.localize(datetime(2024, 1, 1)) + expected_end = TZINFO.localize(datetime(2024, 1, 31)) + + mock_commands_time.assert_called_once() + call_args = mock_commands_time.call_args.args + assert ( + Namespace( + func=mock_commands_time, + command="time", + subcommand="download", + start=expected_start, + end=expected_end, + output="file.csv", + ) + in call_args + ) + + def test_main_time_convert_client(mock_commands_time): main(argv=["time", "convert", "--client", "client123"]) From b876c19391d5016e00e39fe154959f8d99fa662a Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Thu, 26 Sep 2024 00:28:10 +0000 Subject: [PATCH 09/10] feat(time): additional filter args for report download --- README.md | 13 ++++-- compiler_admin/commands/time/download.py | 13 +++++- compiler_admin/main.py | 32 ++++++++++++++ tests/commands/time/test_download.py | 37 ++++++++++++++++- tests/test_main.py | 53 +++++++++++++++++++++++- 5 files changed, 139 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 13be6f9..8617400 100644 --- a/README.md +++ b/README.md @@ -80,12 +80,17 @@ Use this command to download a time report from Toggl in CSV format: ```bash $ compiler-admin time download -h usage: compiler-admin time download [-h] [--start YYYY-MM-DD] [--end YYYY-MM-DD] [--output OUTPUT] + [--client CLIENT_ID] [--project PROJECT_ID] [--task TASK_ID] [--user USER_ID] options: - -h, --help show this help message and exit - --start YYYY-MM-DD The start date of the reporting period. Defaults to the beginning of the prior month. - --end YYYY-MM-DD The end date of the reporting period. Defaults to the end of the prior month. - --output OUTPUT The path to the file where converted data should be written. Defaults to stdout. + -h, --help show this help message and exit + --start YYYY-MM-DD The start date of the reporting period. Defaults to the beginning of the prior month. + --end YYYY-MM-DD The end date of the reporting period. Defaults to the end of the prior month. + --output OUTPUT The path to the file where converted data should be written. Defaults to stdout. + --client CLIENT_ID An ID for a Toggl Client to filter for in reports. Can be supplied more than once. + --project PROJECT_ID An ID for a Toggl Project to filter for in reports. Can be supplied more than once. + --task TASK_ID An ID for a Toggl Project Task to filter for in reports. Can be supplied more than once. + --user USER_ID An ID for a Toggl User to filter for in reports. Can be supplied more than once. ``` ### Converting an hours report diff --git a/compiler_admin/commands/time/download.py b/compiler_admin/commands/time/download.py index 0849c53..6d2e468 100644 --- a/compiler_admin/commands/time/download.py +++ b/compiler_admin/commands/time/download.py @@ -5,6 +5,17 @@ def download(args: Namespace, *extras): - download_time_entries(args.start, args.end, args.output, TOGGL_COLUMNS) + params = dict(start_date=args.start, end_date=args.end, output_path=args.output, output_cols=TOGGL_COLUMNS) + + if args.client_ids: + params.update(dict(client_ids=args.client_ids)) + if args.project_ids: + params.update(dict(project_ids=args.project_ids)) + if args.task_ids: + params.update(dict(task_ids=args.task_ids)) + if args.user_ids: + params.update(dict(user_ids=args.user_ids)) + + download_time_entries(**params) return RESULT_SUCCESS diff --git a/compiler_admin/main.py b/compiler_admin/main.py index b07cacb..a18b09c 100644 --- a/compiler_admin/main.py +++ b/compiler_admin/main.py @@ -94,6 +94,38 @@ def setup_time_command(cmd_parsers: _SubParsersAction): time_download.add_argument( "--output", default=sys.stdout, help="The path to the file where converted data should be written. Defaults to stdout." ) + time_download.add_argument( + "--client", + dest="client_ids", + metavar="CLIENT_ID", + action="append", + type=int, + help="An ID for a Toggl Client to filter for in reports. Can be supplied more than once.", + ) + time_download.add_argument( + "--project", + dest="project_ids", + metavar="PROJECT_ID", + action="append", + type=int, + help="An ID for a Toggl Project to filter for in reports. Can be supplied more than once.", + ) + time_download.add_argument( + "--task", + dest="task_ids", + metavar="TASK_ID", + action="append", + type=int, + help="An ID for a Toggl Project Task to filter for in reports. Can be supplied more than once.", + ) + time_download.add_argument( + "--user", + dest="user_ids", + metavar="USER_ID", + action="append", + type=int, + help="An ID for a Toggl User to filter for in reports. Can be supplied more than once.", + ) def setup_user_command(cmd_parsers: _SubParsersAction): diff --git a/tests/commands/time/test_download.py b/tests/commands/time/test_download.py index c8abb2a..a634fdf 100644 --- a/tests/commands/time/test_download.py +++ b/tests/commands/time/test_download.py @@ -13,9 +13,42 @@ def mock_download_time_entries(mocker): def test_download_default(mock_download_time_entries): date = datetime.now() - args = Namespace(start=date, end=date, output="output") + args = Namespace( + start=date, + end=date, + output="output", + client_ids=None, + project_ids=None, + task_ids=None, + user_ids=None, + ) res = download(args) assert res == RESULT_SUCCESS - mock_download_time_entries.assert_called_once_with(args.start, args.end, args.output, TOGGL_COLUMNS) + mock_download_time_entries.assert_called_once_with( + start_date=args.start, + end_date=args.end, + output_path=args.output, + output_cols=TOGGL_COLUMNS, + ) + + +def test_download_ids(mock_download_time_entries): + date = datetime.now() + ids = [1, 2, 3] + args = Namespace(start=date, end=date, output="output", client_ids=ids, project_ids=ids, task_ids=ids, user_ids=ids) + + res = download(args) + + assert res == RESULT_SUCCESS + mock_download_time_entries.assert_called_once_with( + start_date=args.start, + end_date=args.end, + output_path=args.output, + output_cols=TOGGL_COLUMNS, + client_ids=ids, + project_ids=ids, + task_ids=ids, + user_ids=ids, + ) diff --git a/tests/test_main.py b/tests/test_main.py index d70a214..a6a9421 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -122,17 +122,62 @@ def test_main_time_download_default(mock_commands_time, mock_start, mock_end): call_args = mock_commands_time.call_args.args assert ( Namespace( - func=mock_commands_time, command="time", subcommand="download", start=mock_start, end=mock_end, output=sys.stdout + func=mock_commands_time, + command="time", + subcommand="download", + start=mock_start, + end=mock_end, + output=sys.stdout, + client_ids=None, + project_ids=None, + task_ids=None, + user_ids=None, ) in call_args ) def test_main_time_download_args(mock_commands_time): - main(argv=["time", "download", "--start", "2024-01-01", "--end", "2024-01-31", "--output", "file.csv"]) + main( + argv=[ + "time", + "download", + "--start", + "2024-01-01", + "--end", + "2024-01-31", + "--output", + "file.csv", + "--client", + "1", + "--client", + "2", + "--client", + "3", + "--project", + "1", + "--project", + "2", + "--project", + "3", + "--task", + "1", + "--task", + "2", + "--task", + "3", + "--user", + "1", + "--user", + "2", + "--user", + "3", + ] + ) expected_start = TZINFO.localize(datetime(2024, 1, 1)) expected_end = TZINFO.localize(datetime(2024, 1, 31)) + ids = [1, 2, 3] mock_commands_time.assert_called_once() call_args = mock_commands_time.call_args.args @@ -144,6 +189,10 @@ def test_main_time_download_args(mock_commands_time): start=expected_start, end=expected_end, output="file.csv", + client_ids=ids, + project_ids=ids, + task_ids=ids, + user_ids=ids, ) in call_args ) From d6e7b25427431013df0e470fbf6d8db11c7be146 Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Thu, 26 Sep 2024 01:30:13 +0000 Subject: [PATCH 10/10] feat(toggl): dynamic timeout based on report range --- compiler_admin/services/toggl.py | 6 ++- tests/services/test_toggl.py | 71 +++++++++++++++++++++----------- 2 files changed, 52 insertions(+), 25 deletions(-) diff --git a/compiler_admin/services/toggl.py b/compiler_admin/services/toggl.py index 7f0f133..7448586 100644 --- a/compiler_admin/services/toggl.py +++ b/compiler_admin/services/toggl.py @@ -221,6 +221,10 @@ def download_time_entries( """ start = start_date.strftime("%Y-%m-%d") end = end_date.strftime("%Y-%m-%d") + # calculate a timeout based on the size of the reporting period in days + # approximately 5 seconds per month of query size, with a minimum of 5 seconds + range_days = (end_date - start_date).days + timeout = int((max(30, range_days) / 30.0) * 5) if ("client_ids" not in kwargs or not kwargs["client_ids"]) and isinstance(_toggl_client_id(), int): kwargs["client_ids"] = [_toggl_client_id()] @@ -237,7 +241,7 @@ def download_time_entries( headers = _toggl_api_headers() url = _toggl_api_report_url("search/time_entries.csv") - response = requests.post(url, json=params, headers=headers, timeout=5) + response = requests.post(url, json=params, headers=headers, timeout=timeout) response.raise_for_status() # the raw response has these initial 3 bytes: diff --git a/tests/services/test_toggl.py b/tests/services/test_toggl.py index e52660c..3d98efe 100644 --- a/tests/services/test_toggl.py +++ b/tests/services/test_toggl.py @@ -59,11 +59,31 @@ def mock_google_user_info(mocker): return mocker.patch(f"{MODULE}.google_user_info") +@pytest.fixture +def mock_api_env(monkeypatch): + monkeypatch.setenv("TOGGL_API_TOKEN", "token") + monkeypatch.setenv("TOGGL_CLIENT_ID", "1234") + monkeypatch.setenv("TOGGL_WORKSPACE_ID", "workspace") + + @pytest.fixture def mock_requests(mocker): return mocker.patch(f"{MODULE}.requests") +@pytest.fixture +def mock_api_post(mocker, mock_requests, toggl_file): + # setup a mock response to a requests.post call + mock_csv_bytes = Path(toggl_file).read_bytes() + mock_post_response = mocker.Mock() + mock_post_response.raise_for_status.return_value = None + # prepend the BOM to the mock content + mock_post_response.content = b"\xef\xbb\xbf" + mock_csv_bytes + # override the requests.post call to return the mock response + mock_requests.post.return_value = mock_post_response + return mock_requests + + def test_harvest_client_name(monkeypatch): assert _harvest_client_name() == "Test_Client" @@ -221,35 +241,26 @@ def test_convert_to_harvest_sample(toggl_file, harvest_file, mock_google_user_in assert output_df["Client"].eq("Test Client 123").all() -def test_download_time_entries(monkeypatch, toggl_file, mock_requests, mocker): - monkeypatch.setenv("TOGGL_API_TOKEN", "token") - monkeypatch.setenv("TOGGL_CLIENT_ID", "1234") - monkeypatch.setenv("TOGGL_WORKSPACE_ID", "workspace") - - # setup a mock response to a requests.post call - mock_csv_bytes = Path(toggl_file).read_bytes() - mock_post_response = mocker.Mock() - mock_post_response.raise_for_status.return_value = None - # prepend the BOM to the mock content - mock_post_response.content = b"\xef\xbb\xbf" + mock_csv_bytes - # override the requests.post call to return the mock response - mock_requests.post.return_value = mock_post_response - +@pytest.mark.usefixtures("mock_api_env") +def test_download_time_entries(toggl_file, mock_api_post): dt = datetime.now() + mock_csv_bytes = Path(toggl_file).read_bytes() with NamedTemporaryFile("w") as temp: download_time_entries(dt, dt, temp.name, extra_1=1, extra_2="two") - called_params = mock_requests.post.call_args.kwargs["json"] - assert isinstance(called_params, dict) - assert called_params["billable"] is True - assert called_params["client_ids"] == [1234] - assert called_params["end_date"] == dt.strftime("%Y-%m-%d") - assert called_params["extra_1"] == 1 - assert called_params["extra_2"] == "two" - assert called_params["rounding"] == 1 - assert called_params["rounding_minutes"] == 15 - assert called_params["start_date"] == dt.strftime("%Y-%m-%d") + json_params = mock_api_post.post.call_args.kwargs["json"] + assert isinstance(json_params, dict) + assert json_params["billable"] is True + assert json_params["client_ids"] == [1234] + assert json_params["end_date"] == dt.strftime("%Y-%m-%d") + assert json_params["extra_1"] == 1 + assert json_params["extra_2"] == "two" + assert json_params["rounding"] == 1 + assert json_params["rounding_minutes"] == 15 + assert json_params["start_date"] == dt.strftime("%Y-%m-%d") + + assert mock_api_post.post.call_args.kwargs["timeout"] == 5 temp.flush() response_csv_bytes = Path(temp.name).read_bytes() @@ -265,3 +276,15 @@ def test_download_time_entries(monkeypatch, toggl_file, mock_requests, mocker): # as corresponding column values from the mock DataFrame for col in response_df.columns: assert response_df[col].equals(mock_df[col]) + + +@pytest.mark.usefixtures("mock_api_env") +def test_download_time_entries_dynamic_timeout(mock_api_post): + # range of 6 months + # timeout should be 6 * 5 = 30 + start = datetime(2024, 1, 1) + end = datetime(2024, 6, 30) + + download_time_entries(start, end) + + assert mock_api_post.post.call_args.kwargs["timeout"] == 30