diff --git a/.env.sample b/.env.sample index de2b222..9959b54 100644 --- a/.env.sample +++ b/.env.sample @@ -4,3 +4,9 @@ HARVEST_DATA=data/harvest-sample.csv TOGGL_DATA=data/toggl-sample.csv TOGGL_PROJECT_INFO=data/toggl-project-info-sample.json TOGGL_USER_INFO=data/toggl-user-info-sample.json + +TOGGL_API_TOKEN=token +TOGGL_CLIENT_ID=client +TOGGL_WORKSPACE_ID=workspace + +TZ_NAME=America/Los_Angeles diff --git a/README.md b/README.md index 152d831..8617400 100644 --- a/README.md +++ b/README.md @@ -62,14 +62,35 @@ The `time` command provides an interface for working with time entries from Comp ```bash $ compiler-admin time -h -usage: compiler-admin time [-h] {convert} ... +usage: compiler-admin time [-h] {convert,download} ... positional arguments: - {convert} The time command to run. - convert Convert a time report from one format into another. + {convert,download} The time command to run. + convert Convert a time report from one format into another. + download Download a Toggl report in CSV format. options: - -h, --help show this help message and exit + -h, --help show this help message and exit +``` + +### Downloading a Toggl report + +Use this command to download a time report from Toggl in CSV format: + +```bash +$ compiler-admin time download -h +usage: compiler-admin time download [-h] [--start YYYY-MM-DD] [--end YYYY-MM-DD] [--output OUTPUT] + [--client CLIENT_ID] [--project PROJECT_ID] [--task TASK_ID] [--user USER_ID] + +options: + -h, --help show this help message and exit + --start YYYY-MM-DD The start date of the reporting period. Defaults to the beginning of the prior month. + --end YYYY-MM-DD The end date of the reporting period. Defaults to the end of the prior month. + --output OUTPUT The path to the file where converted data should be written. Defaults to stdout. + --client CLIENT_ID An ID for a Toggl Client to filter for in reports. Can be supplied more than once. + --project PROJECT_ID An ID for a Toggl Project to filter for in reports. Can be supplied more than once. + --task TASK_ID An ID for a Toggl Project Task to filter for in reports. Can be supplied more than once. + --user USER_ID An ID for a Toggl User to filter for in reports. Can be supplied more than once. ``` ### Converting an hours report diff --git a/compiler_admin/commands/time/__init__.py b/compiler_admin/commands/time/__init__.py index b1f9bc7..a56c97d 100644 --- a/compiler_admin/commands/time/__init__.py +++ b/compiler_admin/commands/time/__init__.py @@ -1,6 +1,7 @@ from argparse import Namespace from compiler_admin.commands.time.convert import convert # noqa: F401 +from compiler_admin.commands.time.download import download # noqa: F401 def time(args: Namespace, *extra): diff --git a/compiler_admin/commands/time/download.py b/compiler_admin/commands/time/download.py new file mode 100644 index 0000000..6d2e468 --- /dev/null +++ b/compiler_admin/commands/time/download.py @@ -0,0 +1,21 @@ +from argparse import Namespace + +from compiler_admin import RESULT_SUCCESS +from compiler_admin.services.toggl import INPUT_COLUMNS as TOGGL_COLUMNS, download_time_entries + + +def download(args: Namespace, *extras): + params = dict(start_date=args.start, end_date=args.end, output_path=args.output, output_cols=TOGGL_COLUMNS) + + if args.client_ids: + params.update(dict(client_ids=args.client_ids)) + if args.project_ids: + params.update(dict(project_ids=args.project_ids)) + if args.task_ids: + params.update(dict(task_ids=args.task_ids)) + if args.user_ids: + params.update(dict(user_ids=args.user_ids)) + + download_time_entries(**params) + + return RESULT_SUCCESS diff --git a/compiler_admin/main.py b/compiler_admin/main.py index a5d9fa5..a18b09c 100644 --- a/compiler_admin/main.py +++ b/compiler_admin/main.py @@ -1,6 +1,10 @@ from argparse import ArgumentParser, _SubParsersAction +from datetime import datetime, timedelta +import os import sys +from pytz import timezone + from compiler_admin import __version__ as version from compiler_admin.commands.info import info from compiler_admin.commands.init import init @@ -9,6 +13,24 @@ from compiler_admin.commands.user.convert import ACCOUNT_TYPE_OU +TZINFO = timezone(os.environ.get("TZ_NAME", "America/Los_Angeles")) + + +def local_now(): + return datetime.now(tz=TZINFO) + + +def prior_month_end(): + now = local_now() + first = now.replace(day=1) + return first - timedelta(days=1) + + +def prior_month_start(): + end = prior_month_end() + return end.replace(day=1) + + def add_sub_cmd_parser(parser: ArgumentParser, dest="subcommand", help=None): """Helper adds a subparser for the given dest.""" return parser.add_subparsers(dest=dest, help=help) @@ -54,6 +76,57 @@ def setup_time_command(cmd_parsers: _SubParsersAction): ) time_convert.add_argument("--client", default=None, help="The name of the client to use in converted data.") + time_download = add_sub_cmd(time_subcmds, "download", help="Download a Toggl report in CSV format.") + time_download.add_argument( + "--start", + metavar="YYYY-MM-DD", + default=prior_month_start(), + type=lambda s: TZINFO.localize(datetime.strptime(s, "%Y-%m-%d")), + help="The start date of the reporting period. Defaults to the beginning of the prior month.", + ) + time_download.add_argument( + "--end", + metavar="YYYY-MM-DD", + default=prior_month_end(), + type=lambda s: TZINFO.localize(datetime.strptime(s, "%Y-%m-%d")), + help="The end date of the reporting period. Defaults to the end of the prior month.", + ) + time_download.add_argument( + "--output", default=sys.stdout, help="The path to the file where converted data should be written. Defaults to stdout." + ) + time_download.add_argument( + "--client", + dest="client_ids", + metavar="CLIENT_ID", + action="append", + type=int, + help="An ID for a Toggl Client to filter for in reports. Can be supplied more than once.", + ) + time_download.add_argument( + "--project", + dest="project_ids", + metavar="PROJECT_ID", + action="append", + type=int, + help="An ID for a Toggl Project to filter for in reports. Can be supplied more than once.", + ) + time_download.add_argument( + "--task", + dest="task_ids", + metavar="TASK_ID", + action="append", + type=int, + help="An ID for a Toggl Project Task to filter for in reports. Can be supplied more than once.", + ) + time_download.add_argument( + "--user", + dest="user_ids", + metavar="USER_ID", + action="append", + type=int, + help="An ID for a Toggl User to filter for in reports. Can be supplied more than once.", + ) + def setup_user_command(cmd_parsers: _SubParsersAction): user_cmd = add_sub_cmd(cmd_parsers, "user", help="Work with users in the Compiler org.") diff --git a/compiler_admin/services/toggl.py b/compiler_admin/services/toggl.py index 9b39061..7448586 100644 --- a/compiler_admin/services/toggl.py +++ b/compiler_admin/services/toggl.py @@ -1,12 +1,22 @@ +from base64 import b64encode +from datetime import datetime +import io import os import sys from typing import TextIO import pandas as pd +import requests +from compiler_admin import __version__ from compiler_admin.services.google import user_info as google_user_info import compiler_admin.services.files as files +# Toggl API config +API_BASE_URL = "https://api.track.toggl.com" +API_REPORTS_BASE_URL = "reports/api/v3" +API_WORKSPACE = "workspace/{}" + # cache of previously seen project information, keyed on Toggl project name PROJECT_INFO = {} @@ -36,6 +46,50 @@ def _get_info(obj: dict, key: str, env_key: str): return obj.get(key) +def _toggl_api_authorization_header(): + """Gets an `Authorization: Basic xyz` header using the Toggl API token. + + See https://engineering.toggl.com/docs/authentication. + """ + token = _toggl_api_token() + creds = f"{token}:api_token" + creds64 = b64encode(bytes(creds, "utf-8")).decode("utf-8") + return {"Authorization": "Basic {}".format(creds64)} + + +def _toggl_api_headers(): + """Gets a dict of headers for Toggl API requests. + + See https://engineering.toggl.com/docs/. + """ + headers = {"Content-Type": "application/json"} + headers.update({"User-Agent": "compilerla/compiler-admin:{}".format(__version__)}) + headers.update(_toggl_api_authorization_header()) + return headers + + +def _toggl_api_report_url(endpoint: str): + """Get a fully formed URL for the Toggl Reports API v3 endpoint. + + See https://engineering.toggl.com/docs/reports_start. + """ + workspace_id = _toggl_workspace() + return "/".join((API_BASE_URL, API_REPORTS_BASE_URL, API_WORKSPACE.format(workspace_id), endpoint)) + + +def _toggl_api_token(): + """Gets the value of the TOGGL_API_TOKEN env var.""" + return os.environ.get("TOGGL_API_TOKEN") + + +def _toggl_client_id(): + """Gets the value of the TOGGL_CLIENT_ID env var.""" + client_id = os.environ.get("TOGGL_CLIENT_ID") + if client_id: + return int(client_id) + return None + + def _toggl_project_info(project: str): """Return the cached project for the given project key.""" return _get_info(PROJECT_INFO, project, "TOGGL_PROJECT_INFO") @@ -46,6 +100,11 @@ def _toggl_user_info(email: str): return _get_info(USER_INFO, email, "TOGGL_USER_INFO") +def _toggl_workspace(): + """Gets the value of the TOGGL_WORKSPACE_ID env var.""" + return os.environ.get("TOGGL_WORKSPACE_ID") + + def _get_first_name(email: str) -> str: """Get cached first name or derive from email.""" user = _toggl_user_info(email) @@ -127,3 +186,75 @@ def convert_to_harvest( source["Hours"] = (source["Duration"].dt.total_seconds() / 3600).round(2) files.write_csv(output_path, source, columns=output_cols) + + +def download_time_entries( + start_date: datetime, + end_date: datetime, + output_path: str | TextIO = sys.stdout, + output_cols: list[str] | None = INPUT_COLUMNS, + **kwargs, +): + """Download a CSV report from Toggl of detailed time entries for the given date range. + + Args: + start_date (datetime): The beginning of the reporting period. + + end_date (str): The end of the reporting period. + + output_path: The path to a CSV file where Toggl time entries will be written; or a writeable buffer for the same. + + output_cols (list[str]): A list of column names for the output. + + Extra kwargs are passed along in the POST request body. + + By default, requests a report with the following configuration: + * `billable=True` + * `client_ids=[$TOGGL_CLIENT_ID]` + * `rounding=1` (True, but this is an int param) + * `rounding_minutes=15` + + See https://engineering.toggl.com/docs/reports/detailed_reports#post-export-detailed-report. + + Returns: + None. Either prints the resulting CSV data or writes to output_path. + """ + start = start_date.strftime("%Y-%m-%d") + end = end_date.strftime("%Y-%m-%d") + # calculate a timeout based on the size of the reporting period in days + # approximately 5 seconds per month of query size, with a minimum of 5 seconds + range_days = (end_date - start_date).days + timeout = int((max(30, range_days) / 30.0) * 5) + + if ("client_ids" not in kwargs or not kwargs["client_ids"]) and isinstance(_toggl_client_id(), int): + kwargs["client_ids"] = [_toggl_client_id()] + + params = dict( + billable=True, + start_date=start, + end_date=end, + rounding=1, + rounding_minutes=15, + ) + params.update(kwargs) + + headers = _toggl_api_headers() + url = _toggl_api_report_url("search/time_entries.csv") + + response = requests.post(url, json=params, headers=headers, timeout=timeout) + response.raise_for_status() + + # the raw response has these initial 3 bytes: + # + # b"\xef\xbb\xbfUser,Email,Client..." + # + # \xef\xbb\xb is the Byte Order Mark (BOM) sometimes used in unicode text files + # these 3 bytes indicate a utf-8 encoded text file + # + # See more + # - https://en.wikipedia.org/wiki/Byte_order_mark + # - https://stackoverflow.com/a/50131187 + csv = response.content.decode("utf-8-sig") + + df = pd.read_csv(io.StringIO(csv)) + files.write_csv(output_path, df, columns=output_cols) diff --git a/notebooks/download-and-convert.ipynb b/notebooks/download-and-convert.ipynb new file mode 100644 index 0000000..79777ad --- /dev/null +++ b/notebooks/download-and-convert.ipynb @@ -0,0 +1,64 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from datetime import datetime\n", + "\n", + "from compiler_admin.services.toggl import download_time_entries, convert_to_harvest" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "start = datetime(2024, 9, 1)\n", + "end = datetime(2024, 9, 24)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "download_time_entries(start, end, \"data/time_entries.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "convert_to_harvest(\"data/time_entries.csv\", \"data/time_entries_harvest.csv\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/pyproject.toml b/pyproject.toml index 31f2bba..7537122 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ requires-python = ">=3.11" dependencies = [ "advanced-gam-for-google-workspace @ git+https://github.com/taers232c/GAMADV-XTD3.git@v7.00.05#subdirectory=src", "pandas==2.2.3", + "tzdata", ] [project.urls] @@ -31,7 +32,8 @@ dev = [ test = [ "coverage", "pytest", - "pytest-mock" + "pytest-mock", + "pytest-socket" ] [project.scripts] @@ -53,6 +55,7 @@ source = ["compiler_admin"] [tool.pyright] include = ["compiler_admin", "tests"] +typeCheckingMode = "off" [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/commands/time/test_download.py b/tests/commands/time/test_download.py new file mode 100644 index 0000000..a634fdf --- /dev/null +++ b/tests/commands/time/test_download.py @@ -0,0 +1,54 @@ +from argparse import Namespace +from datetime import datetime +import pytest + +from compiler_admin import RESULT_SUCCESS +from compiler_admin.commands.time.download import __name__ as MODULE, download, TOGGL_COLUMNS + + +@pytest.fixture +def mock_download_time_entries(mocker): + return mocker.patch(f"{MODULE}.download_time_entries") + + +def test_download_default(mock_download_time_entries): + date = datetime.now() + args = Namespace( + start=date, + end=date, + output="output", + client_ids=None, + project_ids=None, + task_ids=None, + user_ids=None, + ) + + res = download(args) + + assert res == RESULT_SUCCESS + mock_download_time_entries.assert_called_once_with( + start_date=args.start, + end_date=args.end, + output_path=args.output, + output_cols=TOGGL_COLUMNS, + ) + + +def test_download_ids(mock_download_time_entries): + date = datetime.now() + ids = [1, 2, 3] + args = Namespace(start=date, end=date, output="output", client_ids=ids, project_ids=ids, task_ids=ids, user_ids=ids) + + res = download(args) + + assert res == RESULT_SUCCESS + mock_download_time_entries.assert_called_once_with( + start_date=args.start, + end_date=args.end, + output_path=args.output, + output_cols=TOGGL_COLUMNS, + client_ids=ids, + project_ids=ids, + task_ids=ids, + user_ids=ids, + ) diff --git a/tests/commands/user/test_convert.py b/tests/commands/user/test_convert.py index bac0c2c..8440f6b 100644 --- a/tests/commands/user/test_convert.py +++ b/tests/commands/user/test_convert.py @@ -172,7 +172,7 @@ def test_convert_partner(mock_google_add_user_to_group, mock_google_move_user_ou res = convert(args) assert res == RESULT_SUCCESS - mock_google_add_user_to_group.call_count == 2 + assert mock_google_add_user_to_group.call_count == 2 mock_google_move_user_ou.assert_called_once() diff --git a/tests/conftest.py b/tests/conftest.py index a24ad3c..7dedd70 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,8 +1,13 @@ import pytest +from pytest_socket import disable_socket from compiler_admin import RESULT_SUCCESS +def pytest_runtest_setup(): + disable_socket() + + @pytest.fixture def mock_module_name(mocker): """Fixture returns a function taking a name, that returns a function taking a module, diff --git a/tests/services/test_toggl.py b/tests/services/test_toggl.py index b14621d..3d98efe 100644 --- a/tests/services/test_toggl.py +++ b/tests/services/test_toggl.py @@ -1,6 +1,7 @@ +from datetime import datetime, timedelta +from io import BytesIO, StringIO import sys -from datetime import timedelta -from io import StringIO +from pathlib import Path from tempfile import NamedTemporaryFile import pandas as pd @@ -22,6 +23,7 @@ _get_last_name, _str_timedelta, convert_to_harvest, + download_time_entries, ) @@ -57,6 +59,31 @@ def mock_google_user_info(mocker): return mocker.patch(f"{MODULE}.google_user_info") +@pytest.fixture +def mock_api_env(monkeypatch): + monkeypatch.setenv("TOGGL_API_TOKEN", "token") + monkeypatch.setenv("TOGGL_CLIENT_ID", "1234") + monkeypatch.setenv("TOGGL_WORKSPACE_ID", "workspace") + + +@pytest.fixture +def mock_requests(mocker): + return mocker.patch(f"{MODULE}.requests") + + +@pytest.fixture +def mock_api_post(mocker, mock_requests, toggl_file): + # setup a mock response to a requests.post call + mock_csv_bytes = Path(toggl_file).read_bytes() + mock_post_response = mocker.Mock() + mock_post_response.raise_for_status.return_value = None + # prepend the BOM to the mock content + mock_post_response.content = b"\xef\xbb\xbf" + mock_csv_bytes + # override the requests.post call to return the mock response + mock_requests.post.return_value = mock_post_response + return mock_requests + + def test_harvest_client_name(monkeypatch): assert _harvest_client_name() == "Test_Client" @@ -212,3 +239,52 @@ def test_convert_to_harvest_sample(toggl_file, harvest_file, mock_google_user_in assert set(output_df.columns.to_list()) <= set(sample_output_df.columns.to_list()) assert output_df["Client"].eq("Test Client 123").all() + + +@pytest.mark.usefixtures("mock_api_env") +def test_download_time_entries(toggl_file, mock_api_post): + dt = datetime.now() + mock_csv_bytes = Path(toggl_file).read_bytes() + + with NamedTemporaryFile("w") as temp: + download_time_entries(dt, dt, temp.name, extra_1=1, extra_2="two") + + json_params = mock_api_post.post.call_args.kwargs["json"] + assert isinstance(json_params, dict) + assert json_params["billable"] is True + assert json_params["client_ids"] == [1234] + assert json_params["end_date"] == dt.strftime("%Y-%m-%d") + assert json_params["extra_1"] == 1 + assert json_params["extra_2"] == "two" + assert json_params["rounding"] == 1 + assert json_params["rounding_minutes"] == 15 + assert json_params["start_date"] == dt.strftime("%Y-%m-%d") + + assert mock_api_post.post.call_args.kwargs["timeout"] == 5 + + temp.flush() + response_csv_bytes = Path(temp.name).read_bytes() + + # load each CSV into a DataFrame + mock_df = pd.read_csv(BytesIO(mock_csv_bytes)) + response_df = pd.read_csv(BytesIO(response_csv_bytes)) + + # check that the response DataFrame has all columns from the mock DataFrame + assert set(response_df.columns.to_list()).issubset(mock_df.columns.to_list()) + + # check that all column values from response DataFrame are the same + # as corresponding column values from the mock DataFrame + for col in response_df.columns: + assert response_df[col].equals(mock_df[col]) + + +@pytest.mark.usefixtures("mock_api_env") +def test_download_time_entries_dynamic_timeout(mock_api_post): + # range of 6 months + # timeout should be 6 * 5 = 30 + start = datetime(2024, 1, 1) + end = datetime(2024, 6, 30) + + download_time_entries(start, end) + + assert mock_api_post.post.call_args.kwargs["timeout"] == 30 diff --git a/tests/test_main.py b/tests/test_main.py index 4ae00d3..a6a9421 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,14 +1,32 @@ from argparse import Namespace +from datetime import datetime import subprocess import sys import pytest import compiler_admin.main -from compiler_admin.main import main, __name__ as MODULE +from compiler_admin.main import main, prior_month_start, prior_month_end, TZINFO, __name__ as MODULE from compiler_admin.services.google import DOMAIN +@pytest.fixture +def mock_local_now(mocker): + dt = datetime(2024, 9, 25, tzinfo=TZINFO) + mocker.patch(f"{MODULE}.local_now", return_value=dt) + return dt + + +@pytest.fixture +def mock_start(mock_local_now): + return datetime(2024, 8, 1, tzinfo=TZINFO) + + +@pytest.fixture +def mock_end(mock_local_now): + return datetime(2024, 8, 31, tzinfo=TZINFO) + + @pytest.fixture def mock_commands_info(mock_commands_info): return mock_commands_info(MODULE) @@ -29,6 +47,18 @@ def mock_commands_user(mock_commands_user): return mock_commands_user(MODULE) +def test_prior_month_start(mock_start): + start = prior_month_start() + + assert start == mock_start + + +def test_prior_month_end(mock_end): + end = prior_month_end() + + assert end == mock_end + + def test_main_info(mock_commands_info): main(argv=["info"]) @@ -84,6 +114,90 @@ def test_main_time_convert_default(mock_commands_time): ) +@pytest.mark.usefixtures("mock_local_now") +def test_main_time_download_default(mock_commands_time, mock_start, mock_end): + main(argv=["time", "download"]) + + mock_commands_time.assert_called_once() + call_args = mock_commands_time.call_args.args + assert ( + Namespace( + func=mock_commands_time, + command="time", + subcommand="download", + start=mock_start, + end=mock_end, + output=sys.stdout, + client_ids=None, + project_ids=None, + task_ids=None, + user_ids=None, + ) + in call_args + ) + + +def test_main_time_download_args(mock_commands_time): + main( + argv=[ + "time", + "download", + "--start", + "2024-01-01", + "--end", + "2024-01-31", + "--output", + "file.csv", + "--client", + "1", + "--client", + "2", + "--client", + "3", + "--project", + "1", + "--project", + "2", + "--project", + "3", + "--task", + "1", + "--task", + "2", + "--task", + "3", + "--user", + "1", + "--user", + "2", + "--user", + "3", + ] + ) + + expected_start = TZINFO.localize(datetime(2024, 1, 1)) + expected_end = TZINFO.localize(datetime(2024, 1, 31)) + ids = [1, 2, 3] + + mock_commands_time.assert_called_once() + call_args = mock_commands_time.call_args.args + assert ( + Namespace( + func=mock_commands_time, + command="time", + subcommand="download", + start=expected_start, + end=expected_end, + output="file.csv", + client_ids=ids, + project_ids=ids, + task_ids=ids, + user_ids=ids, + ) + in call_args + ) + + def test_main_time_convert_client(mock_commands_time): main(argv=["time", "convert", "--client", "client123"])