Refactor collect and send algorithm

backtracepython/client.py

@@ -1,12 +1,13 @@
-import os
-import subprocess
 import sys
 
 import simplejson as json
 
 from backtracepython.attributes.attribute_manager import attribute_manager
 
 from .report import BacktraceReport
+from .report_queue import ReportQueue
+from .request_handler import BacktraceRequestHandler
+from .source_code_handler import SourceCodeHandler
 
 if sys.version_info.major >= 3:
     from urllib.parse import urlencode
@@ -18,50 +19,34 @@ class globs:
     endpoint = None
     next_except_hook = None
     debug_backtrace = False
-    timeout = None
-    tab_width = None
-    attributes = {}
-    context_line_count = None
     worker = None
-
-
-child_py_path = os.path.join(os.path.dirname(__file__), "child.py")
+    attachments = []
+    handler = None
 
 
 def get_attributes():
     return attribute_manager.get()
 
 
-def send_worker_report(report, source_code):
-    send_worker_msg(
-        {
-            "id": "send",
-            "report": report,
-            "context_line_count": globs.context_line_count,
-            "timeout": globs.timeout,
-            "endpoint": globs.endpoint,
-            "tab_width": globs.tab_width,
-            "debug_backtrace": globs.debug_backtrace,
-            "source_code": source_code,
-        }
-    )
+def send(report, attachments=[]):
+    if globs.handler is None:
+        return False
 
-
-def send_worker_msg(msg):
-    payload = json.dumps(msg, ignore_nan=True).encode("utf-8")
-    globs.worker.stdin.write(payload)
-    globs.worker.stdin.write("\n".encode("utf-8"))
-    globs.worker.stdin.flush()
+    globs.handler.add(
+        report.get_data(), report.get_attachments() + globs.attachments + attachments
+    )
+    return True
 
 
 def create_and_send_report(ex_type, ex_value, ex_traceback):
     report = BacktraceReport()
     report.set_exception(ex_type, ex_value, ex_traceback)
     report.set_attribute("error.type", "Unhandled exception")
-    report.send()
+    globs.handler.process(report.get_data(), globs.attachments)
 
 
 def bt_except_hook(ex_type, ex_value, ex_traceback):
+    print("captured unahndled exception")
     if globs.debug_backtrace:
         # Go back to normal exceptions while we do our work here.
         sys.excepthook = globs.next_except_hook
@@ -88,17 +73,19 @@ def initialize(**kwargs):
         kwargs["endpoint"], kwargs.get("token", None)
     )
     globs.debug_backtrace = kwargs.get("debug_backtrace", False)
-    globs.timeout = kwargs.get("timeout", 4)
-    globs.tab_width = kwargs.get("tab_width", 8)
-    globs.context_line_count = kwargs.get("context_line_count", 200)
-
+    globs.attachments = kwargs.get("attachments", [])
     attribute_manager.add(kwargs.get("attributes", {}))
-    stdio_value = None if globs.debug_backtrace else subprocess.PIPE
-    globs.worker = subprocess.Popen(
-        [sys.executable, child_py_path],
-        stdin=subprocess.PIPE,
-        stdout=stdio_value,
-        stderr=stdio_value,
+
+    globs.handler = ReportQueue(
+        BacktraceRequestHandler(
+            globs.endpoint,
+            kwargs.get("timeout", 4),
+            kwargs.get("ignore_ssl_certificate", False),
+            globs.debug_backtrace,
+        ),
+        SourceCodeHandler(
+            kwargs.get("tab_width", 8), kwargs.get("context_line_count", 200)
+        ),
     )
 
     disable_global_handler = kwargs.get("disable_global_handler", False)
@@ -123,11 +110,9 @@ def construct_submission_url(endpoint, token):
 
 
 def finalize():
-    send_worker_msg({"id": "terminate"})
-    if not globs.debug_backtrace:
-        globs.worker.stdout.close()
-        globs.worker.stderr.close()
-    globs.worker.wait()
+    if globs.handler is None:
+        return
+    globs.handler.dispose()
 
 
 def send_last_exception(**kwargs):

backtracepython/report.py

@@ -11,33 +11,31 @@
 
 
 def add_source_code(source_path, source_code_dict, source_path_dict, line):
-    try:
-        the_id = source_path_dict[source_path]
-    except KeyError:
-        the_id = str(uuid.uuid4())
-        source_path_dict[source_path] = the_id
-        source_code_dict[the_id] = {
+
+    if source_path in source_path_dict:
+        source_code_info = source_code_dict[source_path]
+        if line < source_code_info["minLine"]:
+            source_code_info["minLine"] = line
+        if line > source_code_info["maxLine"]:
+            source_code_info["maxLine"] = line
+
+    else:
+        source_code_dict[source_path] = {
             "minLine": line,
             "maxLine": line,
             "path": source_path,
         }
-        return the_id
 
-    if line < source_code_dict[the_id]["minLine"]:
-        source_code_dict[the_id]["minLine"] = line
-    if line > source_code_dict[the_id]["maxLine"]:
-        source_code_dict[the_id]["maxLine"] = line
-    return the_id
+    return source_path
 
 
 def process_frame(tb_frame, line, source_code_dict, source_path_dict):
     source_file = os.path.abspath(tb_frame.f_code.co_filename)
     frame = {
         "funcName": tb_frame.f_code.co_name,
         "line": line,
-        "sourceCode": add_source_code(
-            source_file, source_code_dict, source_path_dict, line
-        ),
+        "library": source_file,
+        "sourceCode": source_file,
     }
     return frame
 
@@ -70,6 +68,7 @@ def __init__(self):
         self.source_code = {}
         self.source_path_dict = {}
         entry_source_code_id = None
+        self.attachments = []
         import __main__
 
         cwd_path = os.path.abspath(os.getcwd())
@@ -155,9 +154,18 @@ def log(self, line):
             }
         )
 
+    def add_attachment(self, attachment_path):
+        self.attachments.append(attachment_path)
+
+    def get_attachments(self):
+        return self.attachments
+
+    def get_data(self):
+        return self.report
+
     def send(self):
         if len(self.log_lines) != 0 and "Log" not in self.report["annotations"]:
             self.report["annotations"]["Log"] = self.log_lines
-        from backtracepython.client import send_worker_report
+        from backtracepython.client import send
 
-        send_worker_report(self.report, self.source_code)
+        send(self)

backtracepython/report_queue.py

@@ -0,0 +1,51 @@
+import sys
+import threading
+
+if sys.version_info.major >= 3:
+    import queue
+else:
+    import Queue as queue
+
+
+class ReportQueue:
+    def __init__(self, request_handler, source_code_handler):
+        self.request_handler = request_handler
+        self.source_code_handler = source_code_handler
+
+        # report submission tasks queue
+        self.report_queue = queue.Queue()
+
+        # Create and start a single worker thread
+        self.worker_thread = threading.Thread(target=self._worker)
+        self.worker_thread.daemon = True
+        self.active = True
+        self.worker_thread.start()
+
+    def _worker(self):
+        while True:
+            report_data = self.report_queue.get()
+            if report_data is None or self.active == False:
+                self.report_queue.task_done()
+                break
+            report, attachments = report_data
+            self.process(report, attachments)
+            self.report_queue.task_done()
+
+    def add(self, report, attachments):
+        self.report_queue.put((report, attachments))
+
+    # Immediately process the report and skip the queue process
+    # Use this method to handle importa data before application exit
+    def process(self, report, attachments):
+        self.source_code_handler.collect(report)
+        self.request_handler.send(report, attachments)
+
+    def __del__(self):
+        self.dispose()
+
+    def dispose(self):
+        # Put a sentinel value to stop the worker thread
+        self.active = False
+        self.report_queue.put_nowait(None)
+        self.report_queue.join()
+        self.worker_thread.join()