From 14646fcce30dc272dd36c01240d219cad0367eff Mon Sep 17 00:00:00 2001 From: Konrad Dysput Date: Fri, 13 Sep 2024 17:51:59 +0200 Subject: [PATCH 1/7] Refactor collect and send algorithm --- backtracepython/child.py | 98 -------------------------- backtracepython/client.py | 73 ++++++++----------- backtracepython/report.py | 42 ++++++----- backtracepython/report_queue.py | 51 ++++++++++++++ backtracepython/request_handler.py | 65 +++++++++++++++++ backtracepython/source_code_handler.py | 42 +++++++++++ example/example.py | 30 +++++--- example/test.txt | 1 + requirements.txt | 2 + setup.py | 6 +- tests/exe/send_report.py | 5 +- tests/test_basic_flow.py | 2 +- 12 files changed, 240 insertions(+), 177 deletions(-) delete mode 100644 backtracepython/child.py create mode 100644 backtracepython/report_queue.py create mode 100644 backtracepython/request_handler.py create mode 100644 backtracepython/source_code_handler.py create mode 100644 example/test.txt diff --git a/backtracepython/child.py b/backtracepython/child.py deleted file mode 100644 index 62e067d..0000000 --- a/backtracepython/child.py +++ /dev/null @@ -1,98 +0,0 @@ -from __future__ import print_function - -import os -import sys - -import simplejson as json - -if sys.version_info.major >= 3: - from urllib.request import Request - from urllib.request import urlopen -else: - from urllib2 import urlopen - from urllib2 import Request - - -class globs: - tab_width = None - debug_backtrace = None - context_line_count = None - - -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, **kwargs) - - -def post_json(full_url, obj): - payload = json.dumps(obj, ignore_nan=True, bigint_as_string=True) - if globs.debug_backtrace: - data = "Submitting a payload to {},\n {}\n".format(full_url, payload) - eprint(data) - - payload = payload.encode("utf-8") - headers = { - "Content-Type": "application/json", - "Content-Length": len(payload), - } - req = Request(full_url, payload, headers) - resp = urlopen(req) - if resp.code != 200: - raise Exception(resp.code, resp.read()) - - -def read_file_or_none(file_path): - try: - with open(file_path) as f: - return f.read() - except: - return None - - -def create_source_object(source_path, min_line, max_line): - ext = os.path.splitext(source_path)[1] - if ext != ".pyc": - text = read_file_or_none(source_path) - if text is not None: - lines = text.split("\n") - line_start = max(min_line - globs.context_line_count - 1, 0) - line_end = min(len(lines), max_line + globs.context_line_count + 1) - return { - "text": "\n".join(lines[line_start:line_end]), - "startLine": line_start + 1, - "startColumn": 1, - "path": source_path, - "tabWidth": globs.tab_width, - } - return {"path": source_path} - - -def collect_source_code(report, source_code_dict): - out_source_code = {} - for key in source_code_dict: - item = source_code_dict[key] - out_source_code[key] = create_source_object( - item["path"], item["minLine"], item["maxLine"] - ) - report["sourceCode"] = out_source_code - - -def prepare_and_send_report(msg): - globs.tab_width = msg["tab_width"] - globs.debug_backtrace = msg["debug_backtrace"] - globs.context_line_count = msg["context_line_count"] - - report = msg["report"] - source_code = msg["source_code"] - collect_source_code(report, source_code) - - post_json(msg["endpoint"], report) - - -for line in sys.stdin: - msg = json.loads(line) - if msg["id"] == "terminate": - sys.exit(0) - elif msg["id"] == "send": - prepare_and_send_report(msg) - else: - raise Exception("invalid message id", msg["id"]) diff --git a/backtracepython/client.py b/backtracepython/client.py index 46ee862..c29c157 100644 --- a/backtracepython/client.py +++ b/backtracepython/client.py @@ -1,5 +1,3 @@ -import os -import subprocess import sys import simplejson as json @@ -7,6 +5,9 @@ from backtracepython.attributes.attribute_manager import attribute_manager from .report import BacktraceReport +from .report_queue import ReportQueue +from .request_handler import BacktraceRequestHandler +from .source_code_handler import SourceCodeHandler if sys.version_info.major >= 3: from urllib.parse import urlencode @@ -18,50 +19,34 @@ class globs: endpoint = None next_except_hook = None debug_backtrace = False - timeout = None - tab_width = None - attributes = {} - context_line_count = None worker = None - - -child_py_path = os.path.join(os.path.dirname(__file__), "child.py") + attachments = [] + handler = None def get_attributes(): return attribute_manager.get() -def send_worker_report(report, source_code): - send_worker_msg( - { - "id": "send", - "report": report, - "context_line_count": globs.context_line_count, - "timeout": globs.timeout, - "endpoint": globs.endpoint, - "tab_width": globs.tab_width, - "debug_backtrace": globs.debug_backtrace, - "source_code": source_code, - } - ) +def send(report, attachments=[]): + if globs.handler is None: + return False - -def send_worker_msg(msg): - payload = json.dumps(msg, ignore_nan=True).encode("utf-8") - globs.worker.stdin.write(payload) - globs.worker.stdin.write("\n".encode("utf-8")) - globs.worker.stdin.flush() + globs.handler.add( + report.get_data(), report.get_attachments() + globs.attachments + attachments + ) + return True def create_and_send_report(ex_type, ex_value, ex_traceback): report = BacktraceReport() report.set_exception(ex_type, ex_value, ex_traceback) report.set_attribute("error.type", "Unhandled exception") - report.send() + globs.handler.process(report.get_data(), globs.attachments) def bt_except_hook(ex_type, ex_value, ex_traceback): + print("captured unahndled exception") if globs.debug_backtrace: # Go back to normal exceptions while we do our work here. sys.excepthook = globs.next_except_hook @@ -88,17 +73,19 @@ def initialize(**kwargs): kwargs["endpoint"], kwargs.get("token", None) ) globs.debug_backtrace = kwargs.get("debug_backtrace", False) - globs.timeout = kwargs.get("timeout", 4) - globs.tab_width = kwargs.get("tab_width", 8) - globs.context_line_count = kwargs.get("context_line_count", 200) - + globs.attachments = kwargs.get("attachments", []) attribute_manager.add(kwargs.get("attributes", {})) - stdio_value = None if globs.debug_backtrace else subprocess.PIPE - globs.worker = subprocess.Popen( - [sys.executable, child_py_path], - stdin=subprocess.PIPE, - stdout=stdio_value, - stderr=stdio_value, + + globs.handler = ReportQueue( + BacktraceRequestHandler( + globs.endpoint, + kwargs.get("timeout", 4), + kwargs.get("ignore_ssl_certificate", False), + globs.debug_backtrace, + ), + SourceCodeHandler( + kwargs.get("tab_width", 8), kwargs.get("context_line_count", 200) + ), ) disable_global_handler = kwargs.get("disable_global_handler", False) @@ -123,11 +110,9 @@ def construct_submission_url(endpoint, token): def finalize(): - send_worker_msg({"id": "terminate"}) - if not globs.debug_backtrace: - globs.worker.stdout.close() - globs.worker.stderr.close() - globs.worker.wait() + if globs.handler is None: + return + globs.handler.dispose() def send_last_exception(**kwargs): diff --git a/backtracepython/report.py b/backtracepython/report.py index 691e446..b7a86ac 100644 --- a/backtracepython/report.py +++ b/backtracepython/report.py @@ -11,23 +11,22 @@ def add_source_code(source_path, source_code_dict, source_path_dict, line): - try: - the_id = source_path_dict[source_path] - except KeyError: - the_id = str(uuid.uuid4()) - source_path_dict[source_path] = the_id - source_code_dict[the_id] = { + + if source_path in source_path_dict: + source_code_info = source_code_dict[source_path] + if line < source_code_info["minLine"]: + source_code_info["minLine"] = line + if line > source_code_info["maxLine"]: + source_code_info["maxLine"] = line + + else: + source_code_dict[source_path] = { "minLine": line, "maxLine": line, "path": source_path, } - return the_id - if line < source_code_dict[the_id]["minLine"]: - source_code_dict[the_id]["minLine"] = line - if line > source_code_dict[the_id]["maxLine"]: - source_code_dict[the_id]["maxLine"] = line - return the_id + return source_path def process_frame(tb_frame, line, source_code_dict, source_path_dict): @@ -35,9 +34,8 @@ def process_frame(tb_frame, line, source_code_dict, source_path_dict): frame = { "funcName": tb_frame.f_code.co_name, "line": line, - "sourceCode": add_source_code( - source_file, source_code_dict, source_path_dict, line - ), + "library": source_file, + "sourceCode": source_file, } return frame @@ -70,6 +68,7 @@ def __init__(self): self.source_code = {} self.source_path_dict = {} entry_source_code_id = None + self.attachments = [] import __main__ cwd_path = os.path.abspath(os.getcwd()) @@ -155,9 +154,18 @@ def log(self, line): } ) + def add_attachment(self, attachment_path): + self.attachments.append(attachment_path) + + def get_attachments(self): + return self.attachments + + def get_data(self): + return self.report + def send(self): if len(self.log_lines) != 0 and "Log" not in self.report["annotations"]: self.report["annotations"]["Log"] = self.log_lines - from backtracepython.client import send_worker_report + from backtracepython.client import send - send_worker_report(self.report, self.source_code) + send(self) diff --git a/backtracepython/report_queue.py b/backtracepython/report_queue.py new file mode 100644 index 0000000..9165a55 --- /dev/null +++ b/backtracepython/report_queue.py @@ -0,0 +1,51 @@ +import sys +import threading + +if sys.version_info.major >= 3: + import queue +else: + import Queue as queue + + +class ReportQueue: + def __init__(self, request_handler, source_code_handler): + self.request_handler = request_handler + self.source_code_handler = source_code_handler + + # report submission tasks queue + self.report_queue = queue.Queue() + + # Create and start a single worker thread + self.worker_thread = threading.Thread(target=self._worker) + self.worker_thread.daemon = True + self.active = True + self.worker_thread.start() + + def _worker(self): + while True: + report_data = self.report_queue.get() + if report_data is None or self.active == False: + self.report_queue.task_done() + break + report, attachments = report_data + self.process(report, attachments) + self.report_queue.task_done() + + def add(self, report, attachments): + self.report_queue.put((report, attachments)) + + # Immediately process the report and skip the queue process + # Use this method to handle importa data before application exit + def process(self, report, attachments): + self.source_code_handler.collect(report) + self.request_handler.send(report, attachments) + + def __del__(self): + self.dispose() + + def dispose(self): + # Put a sentinel value to stop the worker thread + self.active = False + self.report_queue.put_nowait(None) + self.report_queue.join() + self.worker_thread.join() diff --git a/backtracepython/request_handler.py b/backtracepython/request_handler.py new file mode 100644 index 0000000..3029b15 --- /dev/null +++ b/backtracepython/request_handler.py @@ -0,0 +1,65 @@ +import os + +import requests +import simplejson as json + + +class BacktraceRequestHandler: + def __init__(self, submission_url, timeout, ignore_ssl_certificate, debug): + self.submission_url = submission_url + self.timeout = timeout + self.ignore_ssl_certificate = ignore_ssl_certificate + self.debug = debug + + def send(self, report, attachments): + payload = json.dumps(report, ignore_nan=True, bigint_as_string=True) + self.debug_api( + "Submitting a payload to {},\n {}\n".format(self.submission_url, payload) + ) + + files = {"upload_file": payload} + + for attachment in attachments: + if not os.path.exists(attachment): + continue + try: + files["attachment_" + os.path.basename(attachment)] = ( + attachment, + open(attachment, "rb"), + "application/octet-stream", + ) + except Exception as e: + self.debug_api( + "Cannot add attachment {}. Reason {}".format(attachment, str(e)) + ) + continue + + try: + with requests.post( + url=self.submission_url, + files=files, + stream=True, + verify=not self.ignore_ssl_certificate, + timeout=self.timeout, + ) as response: + if response.status_code != 200: + response_body = json.loads(response.text) + result_rx = response_body["_rxid"] + self.debug_api("Report available with rxId {}".format(result_rx)) + return result_rx + self.debug_api( + "Received invalid status code {}. Data: {}".format( + response.status_code, response.text + ) + ) + return None + except Exception as e: + self.debug_api("Received submission failure. Reason: {}".format(str(e))) + finally: + return None + + def debug_api(self, message): + if not self.debug: + return + + print(message) diff --git a/backtracepython/source_code_handler.py b/backtracepython/source_code_handler.py new file mode 100644 index 0000000..5f85ce7 --- /dev/null +++ b/backtracepython/source_code_handler.py @@ -0,0 +1,42 @@ +class SourceCodeHandler: + def __init__(self, tab_width, context_line_count): + self.tab_width = tab_width + self.context_line_count = context_line_count + self.line_offset = 5 + + def collect(self, report): + if not "threads" in report or not "sourceCode" in report: + return report + threads = report["threads"] + + if not "mainThread" in report or not report["mainThread"] in threads: + return report + + main_thread = threads[report["mainThread"]] + + if not "stack" in main_thread: + return report + + main_thread_stack = main_thread["stack"] + + source_code = {} + + for frame in main_thread_stack: + new_min_line = max(frame.line - self.line_offset, 0) + new_max_line = frame.line + self.line_offset + + if not frame["sourceCode"] in source_code: + source_code[frame["sourceCode"]] = { + "minLine": new_min_line, + "maxLine": new_max_line, + "path": frame["sourceCode"], + } + else: + source = source_code[frame["sourceCode"]] + if new_min_line < source["minLine"]: + source["minLIne"] = new_min_line + if new_max_line > source["maxLine"]: + source["maxLine"] = new_min_line + + report["sourceCode"] = source_code + return report diff --git a/example/example.py b/example/example.py index 00f668e..bde14bc 100644 --- a/example/example.py +++ b/example/example.py @@ -7,18 +7,26 @@ def open_file(name): open(name).read() +current_location = os.path.dirname(os.path.abspath(__file__)) +backtrace.initialize( + endpoint=os.getenv( + "BACKTRACE_SUBMISSION_URL", + "https://submit.backtrace.io/your-universe/token/json", + ), + attributes={ + "application": "example-app", + "application.version": "1.0.0", + "version": "1.0.0", + }, + attachments=[ + os.path.join(current_location, "test.txt"), + os.path.join(current_location, "not-existing-file"), + ], + debug_backtrace=True, +) + + def main(): - backtrace.initialize( - endpoint=os.getenv( - "BACKTRACE_SUBMISSION_URL", - '"https://submit.backtrace.io/your-universe/token/json"', - ), - attributes={ - "application": "example-app", - "application.version": "1.0.0", - "version": "1.0.0", - }, - ) # send an exception from the try/catch block try: diff --git a/example/test.txt b/example/test.txt new file mode 100644 index 0000000..d6e9540 --- /dev/null +++ b/example/test.txt @@ -0,0 +1 @@ +Attachment content diff --git a/requirements.txt b/requirements.txt index 6f4311d..a0aa2c6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,5 @@ simplejson==3.19.3 pytest tox black; python_version > '3.0' +requests + diff --git a/setup.py b/setup.py index f147442..c560f8c 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,6 @@ from setuptools import find_packages, setup - setup( name="backtracepython", version="0.3.3", @@ -12,10 +11,7 @@ packages=find_packages(), test_suite="tests", url="https://github.com/backtrace-labs/backtrace-python", - install_requires=[ - "six", - "simplejson", - ], + install_requires=["six", "simplejson", "requests"], extras_require={ "test": ["pytest"], }, diff --git a/tests/exe/send_report.py b/tests/exe/send_report.py index 5d10651..a5b2a57 100644 --- a/tests/exe/send_report.py +++ b/tests/exe/send_report.py @@ -1,4 +1,6 @@ -import sys, os +import os +import sys +import time root_dir = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) sys.path.insert(0, root_dir) @@ -23,3 +25,4 @@ def do_the_thing(): do_the_thing() +time.sleep(1) diff --git a/tests/test_basic_flow.py b/tests/test_basic_flow.py index 191e06d..5ff3e3c 100644 --- a/tests/test_basic_flow.py +++ b/tests/test_basic_flow.py @@ -69,7 +69,7 @@ def check_send_report(obj): def check_threads(obj): if sys.version_info.major >= 3: - assert len(obj["threads"]) == 4 + assert len(obj["threads"]) == 5 def run_one_test(check_fn, exe_name): From 5d0cd81cdefbc1ea44eaa96fc0c563d98db30abf Mon Sep 17 00:00:00 2001 From: Konrad Dysput Date: Fri, 13 Sep 2024 17:54:33 +0200 Subject: [PATCH 2/7] Clean up client --- backtracepython/client.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/backtracepython/client.py b/backtracepython/client.py index c29c157..b35e7a1 100644 --- a/backtracepython/client.py +++ b/backtracepython/client.py @@ -1,7 +1,5 @@ import sys -import simplejson as json - from backtracepython.attributes.attribute_manager import attribute_manager from .report import BacktraceReport @@ -46,7 +44,6 @@ def create_and_send_report(ex_type, ex_value, ex_traceback): def bt_except_hook(ex_type, ex_value, ex_traceback): - print("captured unahndled exception") if globs.debug_backtrace: # Go back to normal exceptions while we do our work here. sys.excepthook = globs.next_except_hook From 74f2ec3db09296b4927071d60f401f5d934bb97f Mon Sep 17 00:00:00 2001 From: Konrad Dysput Date: Mon, 16 Sep 2024 12:40:45 +0200 Subject: [PATCH 3/7] Update backtracepython/request_handler.py Co-authored-by: Sebastian Alex --- backtracepython/request_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backtracepython/request_handler.py b/backtracepython/request_handler.py index 3029b15..107ed58 100644 --- a/backtracepython/request_handler.py +++ b/backtracepython/request_handler.py @@ -30,7 +30,7 @@ def send(self, report, attachments): ) except Exception as e: self.debug_api( - "Cannot add attachment {}. Reason {}".format(attachment, str(e)) + "Cannot add attachment {}: {}".format(attachment, str(e)) ) continue From 016c42ae0410654516a9c5e341d1ee25b7c248e8 Mon Sep 17 00:00:00 2001 From: Konrad Dysput Date: Mon, 16 Sep 2024 13:31:39 +0200 Subject: [PATCH 4/7] Tests should use formdata. Source code handler implementation. Implemented code review feedback --- backtracepython/report_queue.py | 2 +- backtracepython/request_handler.py | 13 ++++--- backtracepython/source_code_handler.py | 53 ++++++++++++++++++++++---- tests/test_basic_flow.py | 9 ++++- 4 files changed, 62 insertions(+), 15 deletions(-) diff --git a/backtracepython/report_queue.py b/backtracepython/report_queue.py index 9165a55..e263cf0 100644 --- a/backtracepython/report_queue.py +++ b/backtracepython/report_queue.py @@ -32,7 +32,7 @@ def _worker(self): self.report_queue.task_done() def add(self, report, attachments): - self.report_queue.put((report, attachments)) + self.report_queue.put_nowait((report, attachments)) # Immediately process the report and skip the queue process # Use this method to handle importa data before application exit diff --git a/backtracepython/request_handler.py b/backtracepython/request_handler.py index 107ed58..aa83156 100644 --- a/backtracepython/request_handler.py +++ b/backtracepython/request_handler.py @@ -1,4 +1,5 @@ import os +import sys import requests import simplejson as json @@ -20,10 +21,15 @@ def send(self, report, attachments): files = {"upload_file": payload} for attachment in attachments: + attachment_name = "attachment_" + os.path.basename(attachment) + + if attachment_name in files: + continue + if not os.path.exists(attachment): continue try: - files["attachment_" + os.path.basename(attachment)] = ( + files[attachment_name] = ( attachment, open(attachment, "rb"), "application/octet-stream", @@ -52,14 +58,11 @@ def send(self, report, attachments): response.status_code, response.text ) ) - return None except Exception as e: self.debug_api("Received submission failure. Reason: {}".format(str(e))) - finally: - return None def debug_api(self, message): if not self.debug: return - print(message) + print(message, file=sys.stderr) diff --git a/backtracepython/source_code_handler.py b/backtracepython/source_code_handler.py index 5f85ce7..e88d36e 100644 --- a/backtracepython/source_code_handler.py +++ b/backtracepython/source_code_handler.py @@ -1,11 +1,13 @@ +import os + + class SourceCodeHandler: def __init__(self, tab_width, context_line_count): self.tab_width = tab_width self.context_line_count = context_line_count - self.line_offset = 5 def collect(self, report): - if not "threads" in report or not "sourceCode" in report: + if not "threads" in report: return report threads = report["threads"] @@ -22,21 +24,56 @@ def collect(self, report): source_code = {} for frame in main_thread_stack: - new_min_line = max(frame.line - self.line_offset, 0) - new_max_line = frame.line + self.line_offset + new_min_line = max(frame["line"] - self.context_line_count - 1, 0) + new_max_line = frame["line"] + self.context_line_count + 1 if not frame["sourceCode"] in source_code: source_code[frame["sourceCode"]] = { - "minLine": new_min_line, + "startLine": new_min_line + 1, + "startColumn": 1, "maxLine": new_max_line, "path": frame["sourceCode"], + "tabWidth": self.tab_width, } else: source = source_code[frame["sourceCode"]] - if new_min_line < source["minLine"]: - source["minLIne"] = new_min_line + if new_min_line < source["startLine"]: + source["startLine"] = new_min_line + 1 if new_max_line > source["maxLine"]: - source["maxLine"] = new_min_line + source["maxLine"] = new_max_line + + for source_code_path in source_code: + source = source_code[source_code_path] + source_code_content = self.read_source( + source_code_path, source["startLine"] - 1, source["maxLine"] + ) + if source_code_content is None: + source_code.pop(source_code_path) + continue + source["text"] = source_code_content + # clean up the source code integration + source.pop("maxLine") report["sourceCode"] = source_code return report + + def read_source(self, source_code_path, start, end): + extension = os.path.splitext(source_code_path)[1] + if extension == ".pyc": + return + + file_content = self.read_file_or_none(source_code_path) + if file_content is None: + return + + lines = file_content.split("\n") + + max_line = min(end, len(lines)) + return "\n".join(lines[start:max_line]) + + def read_file_or_none(self, file_path): + try: + with open(file_path) as f: + return f.read() + except: + return None diff --git a/tests/test_basic_flow.py b/tests/test_basic_flow.py index 5ff3e3c..6740ded 100644 --- a/tests/test_basic_flow.py +++ b/tests/test_basic_flow.py @@ -1,6 +1,7 @@ import os import subprocess import sys +from cgi import FieldStorage import simplejson as json @@ -82,7 +83,13 @@ class RequestHandler(BaseHTTPRequestHandler): def do_POST(self): self.send_response(200) self.end_headers() - payload = self.rfile.read() + form = FieldStorage( + fp=self.rfile, + headers=self.headers, + environ={"REQUEST_METHOD": "POST"}, + ) + + payload = form["upload_file"].file.read() json_string = payload.decode("utf-8", "strict") non_local.json_object = json.loads(json_string) From f5b8708d9bc8c29663bc00c71cf37551105ec807 Mon Sep 17 00:00:00 2001 From: Konrad Dysput Date: Tue, 17 Sep 2024 13:35:53 +0200 Subject: [PATCH 5/7] Refactor stack trace algorithm --- backtracepython/client.py | 8 -- backtracepython/report.py | 150 +++++++++++++++++--------------------- 2 files changed, 67 insertions(+), 91 deletions(-) diff --git a/backtracepython/client.py b/backtracepython/client.py index b35e7a1..db68711 100644 --- a/backtracepython/client.py +++ b/backtracepython/client.py @@ -121,16 +121,8 @@ def send_last_exception(**kwargs): report.send() -def make_an_exception(): - try: - raise Exception - except: - return sys.exc_info() - - def send_report(msg, **kwargs): report = BacktraceReport() - report.set_exception(*make_an_exception()) report.set_dict_attributes(kwargs.get("attributes", {})) report.set_dict_annotations(kwargs.get("annotations", {})) report.set_attribute("error.message", msg) diff --git a/backtracepython/report.py b/backtracepython/report.py index b7a86ac..ad7feaa 100644 --- a/backtracepython/report.py +++ b/backtracepython/report.py @@ -10,77 +10,12 @@ from .version import version_string -def add_source_code(source_path, source_code_dict, source_path_dict, line): - - if source_path in source_path_dict: - source_code_info = source_code_dict[source_path] - if line < source_code_info["minLine"]: - source_code_info["minLine"] = line - if line > source_code_info["maxLine"]: - source_code_info["maxLine"] = line - - else: - source_code_dict[source_path] = { - "minLine": line, - "maxLine": line, - "path": source_path, - } - - return source_path - - -def process_frame(tb_frame, line, source_code_dict, source_path_dict): - source_file = os.path.abspath(tb_frame.f_code.co_filename) - frame = { - "funcName": tb_frame.f_code.co_name, - "line": line, - "library": source_file, - "sourceCode": source_file, - } - return frame - - -def get_main_thread(): - if sys.version_info.major >= 3: - return threading.main_thread() - first = None - for thread in threading.enumerate(): - if thread.name == "MainThread": - return thread - if first is None: - first = thread - return first - - -def walk_tb_backwards(tb): - while tb is not None: - yield tb.tb_frame, tb.tb_lineno - tb = tb.tb_next - - -def walk_tb(tb): - return reversed(list(walk_tb_backwards(tb))) - - class BacktraceReport: def __init__(self): self.fault_thread = threading.current_thread() self.source_code = {} self.source_path_dict = {} - entry_source_code_id = None self.attachments = [] - import __main__ - - cwd_path = os.path.abspath(os.getcwd()) - entry_thread = get_main_thread() - if hasattr(__main__, "__file__"): - entry_source_code_id = ( - add_source_code( - __main__.__file__, self.source_code, self.source_path_dict, 1 - ) - if hasattr(__main__, "__file__") - else None - ) init_attrs = {"error.type": "Exception"} init_attrs.update(attribute_manager.get()) @@ -95,38 +30,87 @@ def __init__(self): "agent": "backtrace-python", "agentVersion": version_string, "mainThread": str(self.fault_thread.ident), - "entryThread": str(entry_thread.ident), - "cwd": cwd_path, "attributes": init_attrs, "annotations": { "Environment Variables": dict(os.environ), }, + "threads": self.generate_stack_trace(), } - if entry_source_code_id is not None: - self.report["entrySourceCode"] = entry_source_code_id def set_exception(self, garbage, ex_value, ex_traceback): self.report["classifiers"] = [ex_value.__class__.__name__] self.report["attributes"]["error.message"] = str(ex_value) + # update faulting thread with information from the error + fault_thread_id = str(self.fault_thread.ident) + if not fault_thread_id in self.report["threads"]: + self.report["threads"][fault_thread_id] = { + "name": self.fault_thread.name, + "stack": [], + "fault": True, + } + + faulting_thread = self.report["threads"][fault_thread_id] + + faulting_thread["stack"] = self.convert_stack_trace( + self.traverse_exception_stack(ex_traceback), False + ) + faulting_thread["fault"] = True + + def generate_stack_trace(self): + current_frames = sys._current_frames() threads = {} for thread in threading.enumerate(): - if thread.ident == self.fault_thread.ident: - threads[str(self.fault_thread.ident)] = { - "name": self.fault_thread.name, - "stack": [ - process_frame( - frame, line, self.source_code, self.source_path_dict - ) - for frame, line in walk_tb(ex_traceback) - ], - } - else: - threads[str(thread.ident)] = { - "name": thread.name, + thread_frame = current_frames.get(thread.ident) + is_main_thread = thread.name == "MainThread" + threads[str(thread.ident)] = { + "name": thread.name, + "stack": self.convert_stack_trace( + self.traverse_process_thread_stack(thread_frame), is_main_thread + ), + "fault": is_main_thread, + } + + return threads + + def traverse_exception_stack(self, traceback): + stack = [] + while traceback: + stack.append({"frame": traceback.tb_frame, "line": traceback.tb_lineno}) + traceback = traceback.tb_next + return reversed(stack) + + def traverse_process_thread_stack(self, thread_frame): + stack = [] + while thread_frame: + stack.append({"frame": thread_frame, "line": thread_frame.f_lineno}) + thread_frame = thread_frame.f_back + return stack + + def convert_stack_trace(self, thread_stack_trace, skip_backtrace_module): + stack_trace = [] + + for thread_stack_frame in thread_stack_trace: + # do not generate frames from our modules when we're reporting messages + thread_frame = thread_stack_frame["frame"] + if skip_backtrace_module: + module = thread_frame.f_globals.get("__name__") + + if module is not None and module.startswith("backtracepython"): + continue + + source_file = os.path.abspath(thread_frame.f_code.co_filename) + + stack_trace.append( + { + "funcName": thread_frame.f_code.co_name, + "line": thread_frame.f_lineno, + "library": source_file, + "sourceCode": source_file, } + ) - self.report["threads"] = threads + return stack_trace def capture_last_exception(self): self.set_exception(*sys.exc_info()) From 3d83f2a82807a36b71c1289699183983190661f3 Mon Sep 17 00:00:00 2001 From: Konrad Dysput Date: Tue, 17 Sep 2024 13:39:27 +0200 Subject: [PATCH 6/7] Use future --- backtracepython/request_handler.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backtracepython/request_handler.py b/backtracepython/request_handler.py index aa83156..385e9cf 100644 --- a/backtracepython/request_handler.py +++ b/backtracepython/request_handler.py @@ -1,3 +1,5 @@ +from __future__ import print_function + import os import sys From ebac11d6b0b751f21044d5a2c3a47d3bb8745736 Mon Sep 17 00:00:00 2001 From: Konrad Dysput Date: Tue, 17 Sep 2024 15:45:21 +0200 Subject: [PATCH 7/7] Move parameters to arguments --- backtracepython/request_handler.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/backtracepython/request_handler.py b/backtracepython/request_handler.py index 385e9cf..033da9f 100644 --- a/backtracepython/request_handler.py +++ b/backtracepython/request_handler.py @@ -17,7 +17,7 @@ def __init__(self, submission_url, timeout, ignore_ssl_certificate, debug): def send(self, report, attachments): payload = json.dumps(report, ignore_nan=True, bigint_as_string=True) self.debug_api( - "Submitting a payload to {},\n {}\n".format(self.submission_url, payload) + "Submitting a payload to {},\n {}\n", self.submission_url, payload ) files = {"upload_file": payload} @@ -37,9 +37,7 @@ def send(self, report, attachments): "application/octet-stream", ) except Exception as e: - self.debug_api( - "Cannot add attachment {}: {}".format(attachment, str(e)) - ) + self.debug_api("Cannot add attachment {}: {}", attachment, str(e)) continue try: @@ -53,18 +51,18 @@ def send(self, report, attachments): if response.status_code != 200: response_body = json.loads(response.text) result_rx = response_body["_rxid"] - self.debug_api("Report available with rxId {}".format(result_rx)) + self.debug_api("Report available with rxId {}", result_rx) return result_rx self.debug_api( - "Received invalid status code {}. Data: {}".format( - response.status_code, response.text - ) + "Received invalid status code {}. Data: {}", + response.status_code, + response.text, ) except Exception as e: - self.debug_api("Received submission failure. Reason: {}".format(str(e))) + self.debug_api("Received submission failure. Reason: {}", str(e)) - def debug_api(self, message): + def debug_api(self, message, *args): if not self.debug: return - print(message, file=sys.stderr) + print(message.format(*args), file=sys.stderr)