Error Logs for failed tranformer requests (#360)

* Added CLI option "logs" to redirect user to transform ids error logs

* removed the URL optional opening, added info and error level, removed the table

* Added loglevel in models for info and error

* Removed Log Level from models.py

* Logic to add the filters in the URL, _a parameters for request id, log level and g parameters for time frame

* edited the logs cli option to use the log url from transformation status message

Author: ketan96-m

servicex/app/transforms.py

@@ -28,6 +28,9 @@
 import asyncio
 from pathlib import Path
 from typing import Optional, List
+import webbrowser
+import re
+from enum import Enum
 
 import rich
 import typer
@@ -138,3 +141,83 @@ async def download_with_progress(filename) -> Path:
 
     for path in result_files:
         print(path.as_posix())
+
+
+class TimeFrame(str, Enum):
+    r"""
+    Time Frame levels: 'day', 'week' & 'month'
+    """
+    day = ("day",)
+    week = ("week",)
+    month = ("month")
+
+
+class LogLevel(str, Enum):
+    r"""
+    Level of the log messages: INFO & ERROR
+    """
+    info = ("INFO",)
+    error = ("ERROR",)
+
+
+def add_query(key, value):
+    """
+    Creates query string from the key and value pairs
+    """
+    query_string = "(query:(match_phrase:({0}:'{1}')))".format(key, value)
+    return query_string
+
+
+def select_time(time_frame=TimeFrame.day):
+    """
+    Takes input as 'day','week','month' and returns the time filter
+    """
+    time_string = time_frame
+    if time_frame.lower() == TimeFrame.day:
+        time_string = "time:(from:now%2Fd,to:now%2Fd)"
+    elif time_frame.lower() == TimeFrame.week:
+        time_string = "time:(from:now%2Fw,to:now%2Fw)"
+    elif time_frame.lower() == TimeFrame.month:
+        time_string = "time:(from:now-30d%2Fd,to:now)"
+    else:
+        rich.print("Got a time frame apart from 'day', 'week', 'month'")
+    return time_string
+
+
+def create_kibana_link_parameters(log_url, transform_id=None, log_level=None, time_frame=None):
+    """
+    Create the _a and _g parameters for the kibana dashboard link
+    """
+    a_parameter = f"&_a=(filters:!({add_query('requestId', transform_id)},"\
+                  f"{add_query('level', log_level.value.lower())}))"
+    g_parameter = f"&_g=({select_time(time_frame.value.lower())})"
+    kibana_link = re.sub(r"\&\_g\=\(\)", g_parameter + a_parameter, log_url)
+    return kibana_link
+
+
+@transforms_app.command(no_args_is_help=True)
+def logs(
+    url: Optional[str] = url_cli_option,
+    backend: Optional[str] = backend_cli_option,
+    transform_id: str = typer.Option(None, "-t", "--transform-id", help="Transform ID"),
+    log_level: Optional[LogLevel] = typer.Option("ERROR", "-l", "--log-level",
+                                                 help="Level of Logs",
+                                                 case_sensitive=False),
+    time_frame: Optional[TimeFrame] = typer.Option("month", "-f", "--time-frame",
+                                                   help="Time Frame",
+                                                   case_sensitive=False)
+):
+    """
+    Open the URL to the Kibana dashboard of the logs of a tranformer
+    """
+    sx = ServiceXClient(backend=backend, url=url)
+    transforms = sx.get_transform_status(transform_id)
+    if transforms and transforms.request_id == transform_id:
+        kibana_link = create_kibana_link_parameters(transforms.log_url,
+                                                    transform_id=transform_id,
+                                                    log_level=log_level,
+                                                    time_frame=time_frame)
+        print(kibana_link)
+        webbrowser.open(kibana_link)
+    else:
+        rich.print("Invalid Request ID")

tests/test_servicex_app_transforms.py

@@ -0,0 +1,60 @@
+from servicex.app.transforms import LogLevel, TimeFrame
+from servicex.app.transforms import add_query, select_time, create_kibana_link_parameters
+
+
+def test_add_query():
+    key = "abc"
+    value = "123-345-567"
+    query = "(query:(match_phrase:(abc:'123-345-567')))"
+    assert add_query(key, value) == query
+
+    key = "requestId"
+    value = "d2ede739-9779-4075-95b1-0c7fae1de408"
+    query = "(query:(match_phrase:(requestId:'d2ede739-9779-4075-95b1-0c7fae1de408')))"
+    assert add_query(key, value) == query
+
+
+def test_select_time():
+    time_frame = TimeFrame.week
+    time_filter = "time:(from:now%2Fw,to:now%2Fw)"
+    assert time_filter == select_time(time_frame)
+
+    time_frame = "month"
+    time_filter = "time:(from:now-30d%2Fd,to:now)"
+    assert time_filter == select_time(time_frame)
+
+    time_frame = "daY"
+    time_filter = "time:(from:now%2Fd,to:now%2Fd)"
+    assert time_filter == select_time(time_frame)
+
+
+def test_create_kibana_link_parameters():
+    initial_log_url = "https://atlas-kibana.mwt2.org:5601/s/servicex/app"\
+                      "/dashboards?auth_provider_hint=anonymous1#/view/"\
+                      "2d2b3b40-f34e-11ed-a6d8-9f6a16cd6d78?embed=true&_g=()"\
+                      "&show-time-filter=true&hide-filter-bar=true"
+    transform_id = "d2ede739-9779-4075-95b1-0c7fae1de408"
+    log_level = LogLevel.error
+    time_frame = TimeFrame.day
+    final_url = "https://atlas-kibana.mwt2.org:5601/s/servicex/app/dashboards?"\
+                "auth_provider_hint=anonymous1#/view/2d2b3b40-f34e-11ed-a6d8-9f6a16cd6d78?"\
+                "embed=true&_g=(time:(from:now%2Fd,to:now%2Fd))"\
+                "&_a=(filters:!((query:(match_phrase:"\
+                "(requestId:'d2ede739-9779-4075-95b1-0c7fae1de408'))),"\
+                "(query:(match_phrase:(level:'error')))))&show-time-filter=true"\
+                "&hide-filter-bar=true"
+    assert create_kibana_link_parameters(initial_log_url, transform_id,
+                                         log_level, time_frame) == final_url
+
+    transform_id = "93713b34-2f0b-4d53-8412-8afa98626516"
+    log_level = LogLevel.info
+    time_frame = TimeFrame.month
+    final_url = "https://atlas-kibana.mwt2.org:5601/s/servicex/app/dashboards?"\
+                "auth_provider_hint=anonymous1#/view/2d2b3b40-f34e-11ed-a6d8-9f6a16cd6d78?"\
+                "embed=true&_g=(time:(from:now-30d%2Fd,to:now))"\
+                "&_a=(filters:!((query:(match_phrase:"\
+                "(requestId:'93713b34-2f0b-4d53-8412-8afa98626516'))),"\
+                "(query:(match_phrase:(level:'info')))))&show-time-filter=true"\
+                "&hide-filter-bar=true"
+    assert create_kibana_link_parameters(initial_log_url, transform_id,
+                                         log_level, time_frame) == final_url