From 0935d1961aeceb9f85eea7e0c7a8525b1b696f64 Mon Sep 17 00:00:00 2001 From: i-jey Date: Tue, 21 Jan 2025 14:57:43 -0800 Subject: [PATCH 01/25] Reset counter after adjusting autobrightness --- ulc_mm_package/hardware/scope_routines.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ulc_mm_package/hardware/scope_routines.py b/ulc_mm_package/hardware/scope_routines.py index 43bfe6406..39bab02fb 100644 --- a/ulc_mm_package/hardware/scope_routines.py +++ b/ulc_mm_package/hardware/scope_routines.py @@ -208,7 +208,11 @@ def count_parasitemia( def count_parasitemia_periodic_wrapper( self, mscope: MalariaScope, - ) -> Generator[List[AsyncInferenceResult], Tuple[np.ndarray, Optional[int]], None,]: + ) -> Generator[ + List[AsyncInferenceResult], + Tuple[np.ndarray, Optional[int]], + None, + ]: while True: img, counts = yield mscope.cell_diagnosis_model.get_asyn_results() mscope.cell_diagnosis_model(img, counts) @@ -386,6 +390,7 @@ def periodic_autobrightness_routine( if counter >= processing_constants.PERIODIC_AB_PERIOD_NUM_FRAMES: autobrightness.autobrightness_pid_control(img) curr_img_brightness = autobrightness.prev_mean_img_brightness + counter = 0 def checkPressureDifference( self, mscope: MalariaScope, ambient_pressure: float From 1159339c93766d2f09a970ef652a2419fac0e39e Mon Sep 17 00:00:00 2001 From: i-jey Date: Tue, 21 Jan 2025 15:02:37 -0800 Subject: [PATCH 02/25] Oops, need to fmt with an older version --- ulc_mm_package/hardware/scope_routines.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ulc_mm_package/hardware/scope_routines.py b/ulc_mm_package/hardware/scope_routines.py index 39bab02fb..b4c6b64a8 100644 --- a/ulc_mm_package/hardware/scope_routines.py +++ b/ulc_mm_package/hardware/scope_routines.py @@ -208,11 +208,7 @@ def count_parasitemia( def count_parasitemia_periodic_wrapper( self, mscope: MalariaScope, - ) -> Generator[ - List[AsyncInferenceResult], - Tuple[np.ndarray, Optional[int]], - None, - ]: + ) -> Generator[List[AsyncInferenceResult], Tuple[np.ndarray, Optional[int]], None,]: while True: img, counts = yield mscope.cell_diagnosis_model.get_asyn_results() mscope.cell_diagnosis_model(img, counts) From d44aed5c4b7cf2167193b6c236b4baf705dafbae Mon Sep 17 00:00:00 2001 From: i-jey Date: Thu, 30 Jan 2025 21:34:40 -0800 Subject: [PATCH 03/25] Add monitoring and dialog box exception if a pressure leak is detected during a run. --- ulc_mm_package/QtGUI/oracle.py | 13 ++++++- ulc_mm_package/QtGUI/scope_op.py | 11 ++++++ ulc_mm_package/hardware/hardware_constants.py | 1 + ulc_mm_package/hardware/scope_routines.py | 36 +++++++++++++++++++ 4 files changed, 60 insertions(+), 1 deletion(-) diff --git a/ulc_mm_package/QtGUI/oracle.py b/ulc_mm_package/QtGUI/oracle.py index d016734f4..005b7cf0d 100644 --- a/ulc_mm_package/QtGUI/oracle.py +++ b/ulc_mm_package/QtGUI/oracle.py @@ -294,6 +294,7 @@ def _init_sigslots(self): self.scopeop.reload_pause.connect(self.reload_pause_handler) self.scopeop.lid_open_pause.connect(self.lid_open_pause_handler) + self.scopeop.pressure_leak_pause.connect(self.pressure_leak_pause_handler) self.scopeop.create_timers.connect(self.acquisition.create_timers) self.scopeop.start_timers.connect(self.acquisition.start_timers) @@ -392,6 +393,17 @@ def lid_open_pause_handler(self): self.scopeop.to_pause() self.unpause() + def pressure_leak_pause_handler(self): + if self.scopeop.state not in NO_PAUSE_STATES: + self.scopeop.to_pause() + self.display_message( + QMessageBox.Icon.Information, + "Pressure leak detected - pausing...", + 'Please open the lid and reseat the CAP module, a pressure leak has been detected. Press "OK" to resume.', + buttons=Buttons.OK, + ) + self.unpause() + def general_pause_handler( self, icon=QMessageBox.Icon.Information, @@ -428,7 +440,6 @@ def general_pause_handler( buttons=Buttons.OK, image=IMAGE_RELOAD_PATH, ) - self.close_lid_display_message() self.unpause() def unpause(self): diff --git a/ulc_mm_package/QtGUI/scope_op.py b/ulc_mm_package/QtGUI/scope_op.py index a38bf3a06..308e42826 100644 --- a/ulc_mm_package/QtGUI/scope_op.py +++ b/ulc_mm_package/QtGUI/scope_op.py @@ -92,6 +92,7 @@ class ScopeOp(QObject, NamedMachine): reload_pause = pyqtSignal(str, str) lid_open_pause = pyqtSignal() + pressure_leak_pause = pyqtSignal() create_timers = pyqtSignal() start_timers = pyqtSignal() @@ -412,6 +413,9 @@ def _check_pressure_seal(self, *args): self.logger.info( f"Pressure check ✅. Ambient absolute pressure: {self.ambient_pressure:.2f} mBar. Gauge pressure = {pdiff:.2f} mBar." ) + self.pressure_monitoring_routine = ( + self.routines.pressure_monitoring_routine(self.ambient_pressure) + ) if self.state == "pressure_check": self.next_state() except PressureSensorBusy as e: @@ -971,6 +975,7 @@ def run_experiment(self, img, timestamp) -> None: self.img_metadata["motor_pos"] = self.mscope.motor.getCurrentPosition() try: pressure, status = self.mscope.pneumatic_module.getPressure() + self.pressure_monitoring_routine.send(pressure) ( self.img_metadata["pressure_hpa"], self.img_metadata["pressure_status_flag"], @@ -978,6 +983,12 @@ def run_experiment(self, img, timestamp) -> None: except PressureSensorStaleValue as e: ## TODO??? self.logger.info(f"Stale pressure sensor value - {e}") + except PressureLeak as e: + self.logger.warning( + f"Pressure leak detected. Current pressure: {pressure:.2f}mBar (gauge: {pressure - self.ambient_pressure:.2f}mBar)." + ) + self.pressure_leak_pause.emit() + return self.img_metadata["led_pwm_val"] = self.mscope.led.pwm_duty_cycle self.img_metadata[ diff --git a/ulc_mm_package/hardware/hardware_constants.py b/ulc_mm_package/hardware/hardware_constants.py index 277bd7b26..7cf44a736 100644 --- a/ulc_mm_package/hardware/hardware_constants.py +++ b/ulc_mm_package/hardware/hardware_constants.py @@ -78,6 +78,7 @@ MPRLS_PWR = 22 MIN_PRESSURE_DIFF = 330 # In units of hPa +PRESSURE_EWMA_ALPHA = 0.1 # ================ Fan constants ================ # FAN_GPIO = 5 CAM_FAN_1 = 23 diff --git a/ulc_mm_package/hardware/scope_routines.py b/ulc_mm_package/hardware/scope_routines.py index b4c6b64a8..bee2a20e1 100644 --- a/ulc_mm_package/hardware/scope_routines.py +++ b/ulc_mm_package/hardware/scope_routines.py @@ -31,6 +31,7 @@ from ulc_mm_package.hardware.motorcontroller import Direction, MotorControllerError from ulc_mm_package.hardware.hardware_constants import ( MIN_PRESSURE_DIFF, + PRESSURE_EWMA_ALPHA, FOCUS_EWMA_ALPHA, ) from ulc_mm_package.image_processing.classic_focus import OOF, ClassicImageFocus @@ -433,6 +434,41 @@ def checkPressureDifference( ) return pressure_diff + @init_generator + def pressure_monitoring_routine( + self, ambient_pressure: float + ) -> Generator[None, float, None]: + """ + Monitor the pressure and raise an exception if it drops below the minimum required pressure difference. + + Parameters + ---------- + mscope: MalariaScope + + Exceptions + ---------- + PressureLeak: + Raised if the pressure difference is less than the minimum required (as set in `hardware_constants.py` via MIN_PRESSURE_DIFF). + """ + + pressure_ewma_filter = EWMAFiltering(PRESSURE_EWMA_ALPHA) + pressure_ewma_filter.set_init_val(ambient_pressure) + period_num = pressure_ewma_filter.get_adjustment_period_ewma() + counter = 0 + + while True: + counter += 1 + curr_pressure = yield + filtered_pressure = pressure_ewma_filter.update_and_get_val(curr_pressure) + gauge_pressure = ambient_pressure - filtered_pressure + + if counter > period_num: + if gauge_pressure < MIN_PRESSURE_DIFF: + raise PressureLeak( + f"Pressure leak detected. Could only generate {gauge_pressure:.3f}mBar of pressure difference (ambient is at: {ambient_pressure:.2f}mBar)." + ) + counter = 0 + @init_generator def find_cells_routine( self, From 60da54804418886fb0150d4b13503edaedfd41f3 Mon Sep 17 00:00:00 2001 From: i-jey Date: Thu, 30 Jan 2025 21:36:23 -0800 Subject: [PATCH 04/25] remove extraneous e --- ulc_mm_package/QtGUI/scope_op.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ulc_mm_package/QtGUI/scope_op.py b/ulc_mm_package/QtGUI/scope_op.py index 308e42826..5717c9af5 100644 --- a/ulc_mm_package/QtGUI/scope_op.py +++ b/ulc_mm_package/QtGUI/scope_op.py @@ -983,7 +983,7 @@ def run_experiment(self, img, timestamp) -> None: except PressureSensorStaleValue as e: ## TODO??? self.logger.info(f"Stale pressure sensor value - {e}") - except PressureLeak as e: + except PressureLeak: self.logger.warning( f"Pressure leak detected. Current pressure: {pressure:.2f}mBar (gauge: {pressure - self.ambient_pressure:.2f}mBar)." ) From 74eb3386e8c51e906b1d65003a3bf130e9c1faff Mon Sep 17 00:00:00 2001 From: i-jey Date: Wed, 19 Feb 2025 17:19:47 -0800 Subject: [PATCH 05/25] remove old pyngrok dependency (no longer used) --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 30197b643..84685f0e0 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,6 @@ def readme(): "typer==0.4.1", "tqdm==4.63.0", "transitions==0.8.11", - "pyngrok==7.0.3", "numba==0.56.0", "Jinja2==3.1.3", "xhtml2pdf==0.2.11", From 748c2fd63cc1f5008f4f0641e2336e167ebc12e2 Mon Sep 17 00:00:00 2001 From: i-jey Date: Wed, 19 Feb 2025 17:31:26 -0800 Subject: [PATCH 06/25] Remove unused ngrok utils --- ulc_mm_package/QtGUI/dev_run.py | 10 -- ulc_mm_package/QtGUI/oracle.py | 23 +-- ulc_mm_package/scope_constants.py | 4 - ulc_mm_package/utilities/email_utils.py | 180 ------------------- ulc_mm_package/utilities/ngrok_utils.py | 226 ------------------------ 5 files changed, 2 insertions(+), 441 deletions(-) delete mode 100644 ulc_mm_package/utilities/email_utils.py delete mode 100644 ulc_mm_package/utilities/ngrok_utils.py diff --git a/ulc_mm_package/QtGUI/dev_run.py b/ulc_mm_package/QtGUI/dev_run.py index 5499276ff..9b8d7c079 100644 --- a/ulc_mm_package/QtGUI/dev_run.py +++ b/ulc_mm_package/QtGUI/dev_run.py @@ -49,9 +49,6 @@ from ulc_mm_package.image_processing.processing_constants import FLOWRATE -from ulc_mm_package.utilities.ngrok_utils import make_tcp_tunnel, NgrokError -from ulc_mm_package.utilities.email_utils import send_ngrok_email - from ulc_mm_package.neural_nets.AutofocusInference import AutoFocus import ulc_mm_package.neural_nets.neural_network_constants as nn_constants @@ -636,13 +633,6 @@ def __init__(self, *args, **kwargs): # Misc self.fan.turn_on_all() self.btnExit.clicked.connect(self.exit) - try: - ngrok_address = make_tcp_tunnel() - send_ngrok_email() - except NgrokError as e: - print(f"Ngrok error : {e}") - ngrok_address = "-ngrok error-" - self.lblngrok.setText(f"{ngrok_address}") # Set slider min/max self.min_exposure_us = 100 diff --git a/ulc_mm_package/QtGUI/oracle.py b/ulc_mm_package/QtGUI/oracle.py index d016734f4..0dccb90f8 100644 --- a/ulc_mm_package/QtGUI/oracle.py +++ b/ulc_mm_package/QtGUI/oracle.py @@ -1,4 +1,4 @@ -""" High-level state machine manager. +"""High-level state machine manager. The Oracle sees all and knows all. It owns all GUI windows, threads, and worker objects (ScopeOp and Acquisition). @@ -70,8 +70,6 @@ AUTOFOCUS_MODEL_DIR, YOGO_MODEL_DIR, ) -from ulc_mm_package.utilities.email_utils import send_ngrok_email, EmailError -from ulc_mm_package.utilities.ngrok_utils import make_tcp_tunnel, NgrokError from ulc_mm_package.QtGUI.scope_op import ScopeOp from ulc_mm_package.QtGUI.form_gui import FormGUI @@ -159,24 +157,7 @@ def __init__(self): self.next_state() def _init_tcp(self): - try: - tcp_addr = make_tcp_tunnel() - self.logger.info(f"SSH address is {tcp_addr}.") - self.liveview_window.update_tcp(tcp_addr) - send_ngrok_email() - except NgrokError as e: - self.logger.warning( - f"SSH address could not be found - {e}. This can be safely ignored." - ) - self.liveview_window.update_tcp("unavailable") - except EmailError as e: - self.logger.warning( - f"SSH address could not be emailed - {e}. This can be safely ignored." - ) - except Exception as e: - self.logger.warning( - f"Unexpected error while setting up TCP: {e}. This can be safely ignored." - ) + self.liveview_window.update_tcp("unavailable") def _check_lock(self): if path.isfile(LOCKFILE): diff --git a/ulc_mm_package/scope_constants.py b/ulc_mm_package/scope_constants.py index 14b7e4a9c..a973bdba0 100644 --- a/ulc_mm_package/scope_constants.py +++ b/ulc_mm_package/scope_constants.py @@ -200,10 +200,6 @@ def IMG_HEIGHT(self) -> int: if VERBOSE: PER_IMAGE_METADATA_KEYS.extend(PER_IMAGE_TIMING_KEYS) -# ================ Environment variables ================ # -NGROK_AUTH_TOKEN_ENV_VAR = "NGROK_AUTH_TOKEN" -EMAIL_PW_TOKEN = "GMAIL_TOKEN" - # ================ SSD directory constants ================ # SSD_NAME = "SamsungSSD" if SIMULATION: diff --git a/ulc_mm_package/utilities/email_utils.py b/ulc_mm_package/utilities/email_utils.py deleted file mode 100644 index f02ccbb86..000000000 --- a/ulc_mm_package/utilities/email_utils.py +++ /dev/null @@ -1,180 +0,0 @@ -import os -import logging -from datetime import datetime -import smtplib -import email.message -import socket - -from ulc_mm_package.scope_constants import EMAIL_PW_TOKEN -from ulc_mm_package.utilities.ngrok_utils import make_tcp_tunnel, NgrokError - - -DEFAULT_EMAIL_LINE = ( - "A million miles away and it's you who has the key to my tcp tunnel <3" # Default -) -DATE_FMT = "%Y-%m-%d" - -logger = logging.getLogger(__name__) - - -class EmailError(Exception): - pass - - -class EmailPWNotSet(Exception): - pass - - -def send_email(sender: str, receiver: str, subject: str, payload: str) -> None: - """Send an email - - Parameters - ---------- - - Exceptions - ---------- - EmailError - EmailPWNotSet - Raised if the GMAIL_TOKEN environment variable is not set in /home/pi/.bashrc - """ - - # Set up email object - msg = email.message.EmailMessage() - msg["From"] = sender - msg["To"] = receiver - msg["Subject"] = subject - msg.add_header("Content-Type", "text") - msg.set_content(payload) - - # creates SMTP session, start TLS - try: - s = smtplib.SMTP("smtp.gmail.com", 587) - s.starttls() - except Exception as e: - raise EmailError(f"Errored when trying to start tls - {e}") - - # Authentication - try: - token = _get_pw() - s.login(sender, token) - except EmailError: - raise - - s.send_message(msg) - - -def send_ngrok_email( - sender: str = "lfmscope@gmail.com", - receiver: str = "lfmscope@gmail.com", -) -> None: - """Send an email with the ngrok address. - - Parameters - ---------- - sender: str - Sending email - receiver: str - Receipient email - - Exceptions - ---------- - NgrokError: - Raised if there's an issue creating/returning the ngrok address. - """ - - try: - ngrok_addr: str = make_tcp_tunnel() - except NgrokError: - raise - scope_name = _get_scope_name() - subject = f"{scope_name} - {ngrok_addr}" - curr_time = datetime.now().strftime("%Y-%m-%d-%H%M%S") - msg = ( - f"Current time: {curr_time}\n" - f"Scope : {scope_name}\n" - f"ngrok address : {ngrok_addr}\n" - f"{_load_saga()}" - ) - try: - send_email(sender, receiver, subject, msg) - except: - raise - - -def _get_pw() -> str: - """ - Get the gmail device-specific password stored in the environment variables. - - Returns - ------- - str - """ - - pw = os.environ.get(EMAIL_PW_TOKEN) - if pw is None: - raise EmailError( - f"{EMAIL_PW_TOKEN} environment variable not set.\n" - "You can set the email token in the .bashrc file by:\n" - "Open the file with: nano /home/pi/.bashrc\n" - "then add the following line (without the '<' '>' signs) to the file:\n" - f"export {EMAIL_PW_TOKEN}=" - ) - return pw - - -def _get_scope_name() -> str: - """Return the hostname (i.e lfm-ohmu)""" - - return socket.gethostname() - - -def _parse_date_str(datetime_str: str, fmt: str = DATE_FMT) -> datetime: - return datetime.strptime(datetime_str, fmt) - - -def _get_days_since_inception(reset: bool = False) -> int: - if reset: - start_date = os.environ.get("INCEPTION", datetime.now().strftime(DATE_FMT)) - os.environ["INCEPTION"] = datetime.now().strftime(DATE_FMT) - return 0 - else: - start_date = os.environ.get("INCEPTION", datetime.now().strftime(DATE_FMT)) - curr_date = datetime.now().strftime(DATE_FMT) - start_datetime, curr_datetime = _parse_date_str(start_date), _parse_date_str( - curr_date - ) - return (curr_datetime - start_datetime).days - - -def _get_saga_line() -> str: - import csv - - file_list = [x for x in os.listdir(".") if "_saga.txt" in x] - file = file_list[0] if len(file_list) > 0 else None - if file is None: - _get_days_since_inception(reset=True) - saga_counter = int(os.environ.get("SAGA_COUNTER", "0")) - os.environ["SAGA_COUNTER"] = "0" - return DEFAULT_EMAIL_LINE - saga_counter = int(os.environ.get("SAGA_COUNTER", "0")) - with open(file, "r") as f: - reader = csv.reader(f, delimiter="^") - for i, row in enumerate(reader): - if i == saga_counter: - os.environ["SAGA_COUNTER"] = str(saga_counter + 1) - return row[0] - return DEFAULT_EMAIL_LINE - - -def _load_saga() -> str: - days_since_inception = _get_days_since_inception() - line = _get_saga_line() - return f"Day {days_since_inception}: {line}" - - -if __name__ == "__main__": - try: - send_ngrok_email() - except NgrokError as e: - print(f"Error when sending email - NgrokError: {e}") - except EmailError as e: - print(f"Error when sending email - EmailError: {e}") diff --git a/ulc_mm_package/utilities/ngrok_utils.py b/ulc_mm_package/utilities/ngrok_utils.py deleted file mode 100644 index 64e47fe48..000000000 --- a/ulc_mm_package/utilities/ngrok_utils.py +++ /dev/null @@ -1,226 +0,0 @@ -import os -import json -import logging -import subprocess - -from urllib.error import URLError -from urllib.request import urlopen -from typing import Dict - -from pyngrok import ngrok, conf - -from ulc_mm_package.scope_constants import NGROK_AUTH_TOKEN_ENV_VAR - -logger = logging.getLogger(__name__) - - -class NgrokError(Exception): - pass - - -class AuthTokenNotSet(NgrokError): - pass - - -def _get_ngrok_json() -> Dict: - """Calls the ngrok localhost status page and returns the status dictionary. - - Returns - ------- - Dict: - Dictionary of ngrok status - - Exceptions - ---------- - URLError: - If ngrok is not running, there will be nothing at this URL. - """ - - addr = "http://127.0.0.1:4040/api/tunnels" - try: - content = urlopen(addr).read().decode("utf-8") - return json.loads(content) - except URLError as e: - logger.info("Address unavailable - ngrok is not on.") - raise e - - -def is_ngrok_running() -> bool: - """Check whether ngrok is running. - - Returns - ------- - bool: - Is ngrok running - """ - - addr = "http://127.0.0.1:4040/api/tunnels" - try: - _ = urlopen(addr).read().decode("utf-8") - return True - except URLError: - return False - - -def _get_addr_from_json(json_dict: Dict): - """Get the public_url from the json.""" - return json_dict["tunnels"][0]["public_url"] - - -def get_addr() -> str: - """Get the public accessible ngrok URL. - - Note: - The user should first check that ngrok is running (`is_ngrok_running()`) - - Returns - ------- - str: - ngrok public url - """ - ngrok_json = _get_ngrok_json() - return _get_addr_from_json(ngrok_json) - - -def _make_tcp_tunnel() -> ngrok.NgrokTunnel: - """Attempt to create an ngrok tcp tunnel. Return an existing tunnel if one is already open. - - Returns - ------- - pyngrok.NgrokTunnel: - pyngrok object - - Exceptions - ---------- - NgrokError: - Unable to create the tunnel. - """ - try: - return ngrok.connect(22, "tcp") - except ngrok.PyngrokError as e: - raise NgrokError(e) - - -def _get_public_url_from_ngrok_tunnel_obj(tunnel_obj: ngrok.NgrokTunnel) -> str: - """Ingest a pyngrok object and return the publicly accessible URL. - - Parameters - ---------- - pyngrok.NgrokTunnel obj - - Returns - ------- - str - """ - return tunnel_obj.public_url - - -def _kill_old_ngrok_sessions() -> None: - """Ensure any old ngrok tunnels are terminated. - - The free-tier account is limited to one active tunnel. Ensure that any stale - sessions are terminated before a new one is made. - - Exceptions - ---------- - None: - Catch-all which logs the exception+traceback - """ - - try: - # Redirect subprocess output to DEVNULL to avoid cluttering the console. - subprocess.run( - ["killall", "ngrok"], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT - ) - except Exception as e: - logger.exception(f"Unknown failure when attempting to `killall ngrok`: {e}") - - -def _create_new_tunnel() -> str: - _kill_old_ngrok_sessions() - set_ngrok_auth_token() - return _get_public_url_from_ngrok_tunnel_obj(_make_tcp_tunnel()) - - -def make_tcp_tunnel() -> str: - """Returns the publicly accessible ngrok ssh address. - - Returns - ------- - str: - Publicly accessible ngrok URL. - - Exceptions - ---------- - NgrokError: - Unable to create the ngrok tunnel. - """ - - try: - # Check for existing ngrok tunnel - if is_ngrok_running(): - try: - addr = get_addr() - return addr - except (URLError, IndexError): - logger.error( - "ngrok is running but unable to get address from api/tunnels." - ) - _kill_old_ngrok_sessions() - set_ngrok_auth_token() - return _get_public_url_from_ngrok_tunnel_obj(_make_tcp_tunnel()) - else: - # Create a new tunnel - try: - return _create_new_tunnel() - except NgrokError: - raise - except: - raise NgrokError( - "NgrokError : existing ngrok tunnel detected but errored out during either is_ngrok_running() or get_addr()." - ) - - -def _get_ngrok_auth_token() -> str: - """Get the ngrok token stored in the environment variable. - - Returns - ------- - str - """ - - token = os.environ.get(NGROK_AUTH_TOKEN_ENV_VAR) - if token is None: - raise AuthTokenNotSet( - f"{NGROK_AUTH_TOKEN_ENV_VAR} environment variable not set.\n" - "You can set the ngrok token in the .bashrc file by:\n" - "Open the file with: nano /home/pi/.bashrc\n" - "then add the following line (without the '<' '>' signs) to the file:\n" - f"export {NGROK_AUTH_TOKEN_ENV_VAR}=" - ) - else: - return token - - -def _set_ngrok_auth_token(token: str) -> None: - """Set the ngrok token.""" - - ngrok.set_auth_token(token) - conf.get_default().auth_token = token - - -def set_ngrok_auth_token(): - """Attempt to set the ngrok token. - - Exceptions - ---------- - NgrokError.AuthTokenNotSet - """ - - token = _get_ngrok_auth_token() - _set_ngrok_auth_token(token) - - -if __name__ == "__main__": - print(f"{make_tcp_tunnel()}") - input("Press enter to exit and close the tunnel...") From 56ff6fb1d4802d020d116af054964c22b333f284 Mon Sep 17 00:00:00 2001 From: i-jey Date: Fri, 28 Feb 2025 15:58:11 -0800 Subject: [PATCH 07/25] Setting NCS performance hit will potentially allow for more frequent polling of SSAF, and thus improve our ability to track focal drift. May want to tweak EWMA parameters to be less conservative since we can now receive more samples. Needs to be tested extensively on scope however, in particular we need to monitor for any memory increases in case the NCS cannot keep up and we start building up a queue. --- ulc_mm_package/neural_nets/NCSModel.py | 3 +++ ulc_mm_package/neural_nets/neural_network_constants.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/ulc_mm_package/neural_nets/NCSModel.py b/ulc_mm_package/neural_nets/NCSModel.py index c7e318665..880b69bc5 100644 --- a/ulc_mm_package/neural_nets/NCSModel.py +++ b/ulc_mm_package/neural_nets/NCSModel.py @@ -116,6 +116,9 @@ def _compile_model( compiled_model = self.core.compile_model( model, self.device_name, + config = { + "PERFORMANCE_HINT": "THROUGHPUT", + } ) self.connected = True return compiled_model diff --git a/ulc_mm_package/neural_nets/neural_network_constants.py b/ulc_mm_package/neural_nets/neural_network_constants.py index 63a7e789e..3cf17fe75 100644 --- a/ulc_mm_package/neural_nets/neural_network_constants.py +++ b/ulc_mm_package/neural_nets/neural_network_constants.py @@ -7,11 +7,11 @@ # ================ Autofocus constants ================ # -AF_PERIOD_S = 0.5 +AF_PERIOD_S = 0.1 AF_PERIOD_NUM = int( AF_PERIOD_S * ACQUISITION_FPS ) # Used for periodic (ie. EWMA) autofocus -AF_BATCH_SIZE = 10 # Used for single shot autofocus +AF_BATCH_SIZE = 50 # Used for single shot autofocus AF_THRESHOLD = 2 AF_QSIZE = 10 # For AF_PERIOD_S = 0.5, we have a max delay of 5 sec From 3f4fd5f4a27a557e18e88ec3958757dc23973207 Mon Sep 17 00:00:00 2001 From: i-jey Date: Fri, 28 Feb 2025 16:05:52 -0800 Subject: [PATCH 08/25] formatting --- ulc_mm_package/neural_nets/NCSModel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ulc_mm_package/neural_nets/NCSModel.py b/ulc_mm_package/neural_nets/NCSModel.py index 880b69bc5..2adfca7e2 100644 --- a/ulc_mm_package/neural_nets/NCSModel.py +++ b/ulc_mm_package/neural_nets/NCSModel.py @@ -116,9 +116,9 @@ def _compile_model( compiled_model = self.core.compile_model( model, self.device_name, - config = { + config={ "PERFORMANCE_HINT": "THROUGHPUT", - } + }, ) self.connected = True return compiled_model From 42bd123f7a77643d7ba5abe352ff535c923d2b60 Mon Sep 17 00:00:00 2001 From: i-jey Date: Tue, 4 Mar 2025 13:20:22 -0800 Subject: [PATCH 09/25] formatting --- ulc_mm_package/QtGUI/acquisition.py | 2 +- ulc_mm_package/QtGUI/form_gui.py | 2 +- ulc_mm_package/QtGUI/liveview_gui.py | 2 +- ulc_mm_package/QtGUI/scope_op.py | 22 +++++++++++++++++++--- 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/ulc_mm_package/QtGUI/acquisition.py b/ulc_mm_package/QtGUI/acquisition.py index 2ab5ed18f..2f26e148f 100644 --- a/ulc_mm_package/QtGUI/acquisition.py +++ b/ulc_mm_package/QtGUI/acquisition.py @@ -1,4 +1,4 @@ -""" Image manager +"""Image manager Receives images from the camera and sends them to Liveview and ScopeOp. diff --git a/ulc_mm_package/QtGUI/form_gui.py b/ulc_mm_package/QtGUI/form_gui.py index 1d54f8372..95a34f1be 100644 --- a/ulc_mm_package/QtGUI/form_gui.py +++ b/ulc_mm_package/QtGUI/form_gui.py @@ -1,4 +1,4 @@ -""" Experiment form GUI window +"""Experiment form GUI window Takes user input and exports experiment metadata. diff --git a/ulc_mm_package/QtGUI/liveview_gui.py b/ulc_mm_package/QtGUI/liveview_gui.py index 1d30836e7..2b2b3718f 100644 --- a/ulc_mm_package/QtGUI/liveview_gui.py +++ b/ulc_mm_package/QtGUI/liveview_gui.py @@ -1,4 +1,4 @@ -""" Liveview GUI window +"""Liveview GUI window Displays camera preview and conveys info to user during runs.""" diff --git a/ulc_mm_package/QtGUI/scope_op.py b/ulc_mm_package/QtGUI/scope_op.py index a38bf3a06..9b909a77f 100644 --- a/ulc_mm_package/QtGUI/scope_op.py +++ b/ulc_mm_package/QtGUI/scope_op.py @@ -1,4 +1,4 @@ -""" Mid-level/hardware state machine manager +"""Mid-level/hardware state machine manager Controls hardware (ie. the Scope) operations. Manages hardware routines and interactions with Oracle and Acquisition. @@ -875,6 +875,9 @@ def run_experiment(self, img, timestamp) -> None: self.mscope.cell_diagnosis_model.work_queue_size(), ) + # ------------------------------------ + # Get and process YOGO results + # ------------------------------------ t0 = perf_counter() for result in prev_yogo_results: self.mscope.predictions_handler.add_yogo_pred(result) @@ -903,6 +906,9 @@ def run_experiment(self, img, timestamp) -> None: t1 = perf_counter() self._update_metadata_if_verbose("yogo_result_mgmt", t1 - t0) + # ------------------------------------ + # Run periodic singleshot autofocus routine + # ------------------------------------ t0 = perf_counter() resized_img = cv2.resize(img, IMG_RESIZED_DIMS, interpolation=cv2.INTER_CUBIC) try: @@ -937,8 +943,10 @@ def run_experiment(self, img, timestamp) -> None: if filtered_focus_err is not None: self.filtered_focus_err = filtered_focus_err + # ------------------------------------ + # Get classic image sharpness metric + # ------------------------------------ t0 = perf_counter() - # Downsample image for use in flowrate + classic image focus metric img_ds_10x = downsample_image(img, 10) try: @@ -951,6 +959,10 @@ def run_experiment(self, img, timestamp) -> None: ) self.oof_to_motor_sweep() return + + # ------------------------------------ + # Run flow control routine + # ------------------------------------ try: self.flowrate, _ = self.flowcontrol_routine.send((img_ds_10x, timestamp)) except Exception as e: @@ -963,11 +975,15 @@ def run_experiment(self, img, timestamp) -> None: t1 = perf_counter() self._update_metadata_if_verbose("flowrate_dt", t1 - t0) + # ------------------------------------ # Run periodic autobrightness routine + # ------------------------------------ curr_mean_pixel_val = self.periodic_autobrightness_routine.send(resized_img) + # ------------------------------------ + # Update remaining metadata in per-image csv + # ------------------------------------ t0 = perf_counter() - # Update remaining metadata self.img_metadata["motor_pos"] = self.mscope.motor.getCurrentPosition() try: pressure, status = self.mscope.pneumatic_module.getPressure() From 5833fd82e860d3e198f48e1bb203e7303a91f302 Mon Sep 17 00:00:00 2001 From: i-jey Date: Tue, 4 Mar 2025 15:28:15 -0800 Subject: [PATCH 10/25] Force the predictions handler to reset as opposed to relying on gc --- ulc_mm_package/hardware/scope.py | 2 +- ulc_mm_package/neural_nets/predictions_handler.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ulc_mm_package/hardware/scope.py b/ulc_mm_package/hardware/scope.py index 478a6137b..e337eb1a3 100644 --- a/ulc_mm_package/hardware/scope.py +++ b/ulc_mm_package/hardware/scope.py @@ -153,7 +153,7 @@ def reset_for_end_experiment(self) -> None: self.cell_diagnosis_model.reset(wait_for_jobs=False) # Reset predictions handler - self.predictions_handler: PredictionsHandler = PredictionsHandler() + self.predictions_handler.reset() def shutoff(self): self.logger.info("Shutting off scope hardware.") diff --git a/ulc_mm_package/neural_nets/predictions_handler.py b/ulc_mm_package/neural_nets/predictions_handler.py index 80196964b..534524ef6 100644 --- a/ulc_mm_package/neural_nets/predictions_handler.py +++ b/ulc_mm_package/neural_nets/predictions_handler.py @@ -71,6 +71,9 @@ def __init__(self): # Setup heatmap masking self.heatmaps = np.zeros((len(YOGO_CLASS_LIST), sy * sx)) + def reset(self): + self.__init__() + def add_raw_pred_to_heatmap(self, yogo_res: AsyncInferenceResult) -> None: """Add the raw YOGO prediction to the heatmap. From 2b2588a82215f6bf50dab15c66fb8c4e9e498982 Mon Sep 17 00:00:00 2001 From: i-jey Date: Wed, 5 Mar 2025 12:11:34 -0800 Subject: [PATCH 11/25] Address potential memory leaks --- ulc_mm_package/neural_nets/predictions_handler.py | 12 ++++++++++++ .../summary_report/parasitemia_visualization.py | 1 + 2 files changed, 13 insertions(+) diff --git a/ulc_mm_package/neural_nets/predictions_handler.py b/ulc_mm_package/neural_nets/predictions_handler.py index 534524ef6..9eca37e94 100644 --- a/ulc_mm_package/neural_nets/predictions_handler.py +++ b/ulc_mm_package/neural_nets/predictions_handler.py @@ -72,6 +72,18 @@ def __init__(self): self.heatmaps = np.zeros((len(YOGO_CLASS_LIST), sy * sx)) def reset(self): + self.pred_tensors.fill(0) + self.new_pred_pointer = 0 + self.max_confs = {x: [] for x in self.class_ids} + self.curr_min_of_max_confs_by_class = { + x: HIGH_CONF_THRESH - 1e-6 for x in self.class_ids + } + self.min_confs = {x: [] for x in self.class_ids} + self.curr_max_of_min_confs_by_class = { + x: HIGH_CONF_THRESH for x in self.class_ids + } + self.heatmaps.fill(0) + self.__init__() def add_raw_pred_to_heatmap(self, yogo_res: AsyncInferenceResult) -> None: diff --git a/ulc_mm_package/summary_report/parasitemia_visualization.py b/ulc_mm_package/summary_report/parasitemia_visualization.py index 83ab46b2c..728eb31ce 100644 --- a/ulc_mm_package/summary_report/parasitemia_visualization.py +++ b/ulc_mm_package/summary_report/parasitemia_visualization.py @@ -169,6 +169,7 @@ def make_parasitemia_plot(parasitemia, err, savefile): fig.subplots_adjust(left=0.1, right=0.9, top=0.8, bottom=0.1) fig.tight_layout() plt.savefig(savefile) + plt.close() if __name__ == "__main__": From 4e78eca873da4a8ec65af8f04aaf577177c2acb2 Mon Sep 17 00:00:00 2001 From: i-jey Date: Wed, 5 Mar 2025 12:20:42 -0800 Subject: [PATCH 12/25] remove unnecessary __init__ call in PredictionsHandler reset --- ulc_mm_package/neural_nets/predictions_handler.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ulc_mm_package/neural_nets/predictions_handler.py b/ulc_mm_package/neural_nets/predictions_handler.py index 9eca37e94..ba3fd01e3 100644 --- a/ulc_mm_package/neural_nets/predictions_handler.py +++ b/ulc_mm_package/neural_nets/predictions_handler.py @@ -84,8 +84,6 @@ def reset(self): } self.heatmaps.fill(0) - self.__init__() - def add_raw_pred_to_heatmap(self, yogo_res: AsyncInferenceResult) -> None: """Add the raw YOGO prediction to the heatmap. From 3b7166f4e9b15c2001b543482b94baa461d0ccf2 Mon Sep 17 00:00:00 2001 From: i-jey Date: Fri, 7 Mar 2025 11:09:34 -0800 Subject: [PATCH 13/25] close per img metadata file --- ulc_mm_package/image_processing/data_storage.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ulc_mm_package/image_processing/data_storage.py b/ulc_mm_package/image_processing/data_storage.py index 8d5e6f7ac..942c7e3f3 100644 --- a/ulc_mm_package/image_processing/data_storage.py +++ b/ulc_mm_package/image_processing/data_storage.py @@ -298,16 +298,16 @@ def close( summary_report_dir / f"{self.time_str}_per_image_metadata_plot.jpg" ) - per_img_metadata_file = open(self.per_img_metadata_filename, "r") counts_plot_loc = str(summary_report_dir / "counts.jpg") conf_plot_loc = str(summary_report_dir / "confs.jpg") objectness_plot_loc = str(summary_report_dir / "objectness.jpg") # Only generate additional plots if DEBUG_REPORT environment variable is set to True if DEBUG_REPORT: - make_per_image_metadata_plots( - per_img_metadata_file, per_image_metadata_plot_save_loc - ) + with open(self.per_img_metadata_filename, "r") as per_img_metadata_file: + make_per_image_metadata_plots( + per_img_metadata_file, per_image_metadata_plot_save_loc + ) try: make_cell_count_plot(pred_tensors, counts_plot_loc) From 819a5e75c44b8685b62e384bef09ddcfa3062e8e Mon Sep 17 00:00:00 2001 From: i-jey Date: Fri, 7 Mar 2025 14:15:35 -0800 Subject: [PATCH 14/25] This was the main problem - we should not instantiate a new FlowController instance, use mscope's existing one --- ulc_mm_package/hardware/scope_routines.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ulc_mm_package/hardware/scope_routines.py b/ulc_mm_package/hardware/scope_routines.py index b4c6b64a8..8434a0d4d 100644 --- a/ulc_mm_package/hardware/scope_routines.py +++ b/ulc_mm_package/hardware/scope_routines.py @@ -486,7 +486,7 @@ def find_cells_routine( # Maximum number of times to run check for cells routine before aborting max_attempts = 3 cell_finder = CellFinder() - flow_controller = FlowController(mscope.pneumatic_module) + flow_controller = mscope.flow_controller img = yield # Initial check for cells, return current motor position if cells found From 76bb1deaf4897ba14c86429907ee4cb55f750b4a Mon Sep 17 00:00:00 2001 From: i-jey Date: Fri, 7 Mar 2025 14:21:24 -0800 Subject: [PATCH 15/25] remove unused import --- ulc_mm_package/hardware/scope_routines.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ulc_mm_package/hardware/scope_routines.py b/ulc_mm_package/hardware/scope_routines.py index 8434a0d4d..865a19982 100644 --- a/ulc_mm_package/hardware/scope_routines.py +++ b/ulc_mm_package/hardware/scope_routines.py @@ -15,7 +15,6 @@ checkLedWorking, ) from ulc_mm_package.image_processing.flow_control import ( - FlowController, CantReachTargetFlowrate, ) from ulc_mm_package.image_processing.cell_finder import ( From d03f4cbf4e4b41228e2e2cfd8c2d0a09c8109938 Mon Sep 17 00:00:00 2001 From: i-jey Date: Fri, 7 Mar 2025 15:10:38 -0800 Subject: [PATCH 16/25] Set Autofocus batch size for pre-flow/post-flow to 20 (down from 50 which is excessive, up from 10 which is what we had previously) --- ulc_mm_package/neural_nets/neural_network_constants.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ulc_mm_package/neural_nets/neural_network_constants.py b/ulc_mm_package/neural_nets/neural_network_constants.py index 3cf17fe75..9c4e5bf6e 100644 --- a/ulc_mm_package/neural_nets/neural_network_constants.py +++ b/ulc_mm_package/neural_nets/neural_network_constants.py @@ -7,14 +7,14 @@ # ================ Autofocus constants ================ # -AF_PERIOD_S = 0.1 +AF_PERIOD_S = 0.1 # (10 imgs/sec) AF_PERIOD_NUM = int( AF_PERIOD_S * ACQUISITION_FPS ) # Used for periodic (ie. EWMA) autofocus -AF_BATCH_SIZE = 50 # Used for single shot autofocus +AF_BATCH_SIZE = 20 # Used for single shot autofocus AF_THRESHOLD = 2 -AF_QSIZE = 10 # For AF_PERIOD_S = 0.5, we have a max delay of 5 sec +AF_QSIZE = 25 AUTOFOCUS_MODEL_NAME = "fast-cosmos-557" AUTOFOCUS_MODEL_DIR = str( From 42914d4e29509c6c692dcba1226d07e0d39c2c8a Mon Sep 17 00:00:00 2001 From: i-jey Date: Fri, 7 Mar 2025 15:39:23 -0800 Subject: [PATCH 17/25] reset flow controller prior to starting cell finder --- ulc_mm_package/hardware/scope_routines.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ulc_mm_package/hardware/scope_routines.py b/ulc_mm_package/hardware/scope_routines.py index 865a19982..1a821768b 100644 --- a/ulc_mm_package/hardware/scope_routines.py +++ b/ulc_mm_package/hardware/scope_routines.py @@ -485,7 +485,9 @@ def find_cells_routine( # Maximum number of times to run check for cells routine before aborting max_attempts = 3 cell_finder = CellFinder() + mscope.flow_controller.reset() flow_controller = mscope.flow_controller + img = yield # Initial check for cells, return current motor position if cells found From a939cfcd21ec47cab6ac9bdecaac8ca196fa8023 Mon Sep 17 00:00:00 2001 From: i-jey Date: Fri, 7 Mar 2025 15:48:37 -0800 Subject: [PATCH 18/25] Double the fast flow pneumatic module step size just so we can get up to speed faster. Set it back to normal once fast flow is done. --- ulc_mm_package/hardware/scope_routines.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ulc_mm_package/hardware/scope_routines.py b/ulc_mm_package/hardware/scope_routines.py index 1a821768b..01a6b6891 100644 --- a/ulc_mm_package/hardware/scope_routines.py +++ b/ulc_mm_package/hardware/scope_routines.py @@ -247,6 +247,7 @@ def flow_control_routine( flow_controller.set_alpha( processing_constants.FLOW_CONTROL_EWMA_ALPHA * 2 ) # Double the alpha, ~halve the half life + flow_controller.pneumatic_module.min_step_size *= 2 while True: img, timestamp = yield flow_val, syringe_can_move @@ -269,6 +270,7 @@ def flow_control_routine( if fast_flow: if flow_error is not None: if flow_error == 0: + flow_controller.pneumatic_module.min_step_size /= 2 return flow_val @init_generator From b2fa3e38abeebf7ab4631234e302ba5aadeea79d Mon Sep 17 00:00:00 2001 From: i-jey Date: Fri, 7 Mar 2025 15:49:14 -0800 Subject: [PATCH 19/25] fmt --- ulc_mm_package/neural_nets/neural_network_constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ulc_mm_package/neural_nets/neural_network_constants.py b/ulc_mm_package/neural_nets/neural_network_constants.py index 9c4e5bf6e..9f8c4f8ac 100644 --- a/ulc_mm_package/neural_nets/neural_network_constants.py +++ b/ulc_mm_package/neural_nets/neural_network_constants.py @@ -7,7 +7,7 @@ # ================ Autofocus constants ================ # -AF_PERIOD_S = 0.1 # (10 imgs/sec) +AF_PERIOD_S = 0.1 # (10 imgs/sec) AF_PERIOD_NUM = int( AF_PERIOD_S * ACQUISITION_FPS ) # Used for periodic (ie. EWMA) autofocus From 796d854fa5c07271f2b2f3632b1ca2f497ac1c15 Mon Sep 17 00:00:00 2001 From: i-jey Date: Fri, 7 Mar 2025 15:49:24 -0800 Subject: [PATCH 20/25] fmt --- ulc_mm_package/neural_nets/neural_network_constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ulc_mm_package/neural_nets/neural_network_constants.py b/ulc_mm_package/neural_nets/neural_network_constants.py index 9c4e5bf6e..9f8c4f8ac 100644 --- a/ulc_mm_package/neural_nets/neural_network_constants.py +++ b/ulc_mm_package/neural_nets/neural_network_constants.py @@ -7,7 +7,7 @@ # ================ Autofocus constants ================ # -AF_PERIOD_S = 0.1 # (10 imgs/sec) +AF_PERIOD_S = 0.1 # (10 imgs/sec) AF_PERIOD_NUM = int( AF_PERIOD_S * ACQUISITION_FPS ) # Used for periodic (ie. EWMA) autofocus From 6ceaccaacda33a509c34a41731c88c360b0393ff Mon Sep 17 00:00:00 2001 From: i-jey Date: Fri, 7 Mar 2025 16:13:12 -0800 Subject: [PATCH 21/25] Add in a small check to CellFinder to ensure the motor is not in motion prior to starting --- ulc_mm_package/hardware/scope_routines.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ulc_mm_package/hardware/scope_routines.py b/ulc_mm_package/hardware/scope_routines.py index 01a6b6891..ccd9b98de 100644 --- a/ulc_mm_package/hardware/scope_routines.py +++ b/ulc_mm_package/hardware/scope_routines.py @@ -499,6 +499,11 @@ def find_cells_routine( except NoCellsFound: cell_finder.reset() + # Defensive check, ensure the motor isn't moving (say for example, + # if CellFinder was triggered by an OOF exception and SSAF just triggered a motor move) + while mscope.motor.is_locked(): + pass + while True: """ 1. Pull syringe for pull_time seconds (unless deliberately skipped) From 97f47d8a525bbcf5dbc7e6d83d2d6be43f1906b6 Mon Sep 17 00:00:00 2001 From: i-jey Date: Mon, 10 Mar 2025 11:54:43 -0700 Subject: [PATCH 22/25] Revert "Merge branch 'develop' into pressure_leak_warning" This reverts commit 18022466e8a8636d5b11bf58606d9e2e1a57d4c1, reversing changes made to 60da54804418886fb0150d4b13503edaedfd41f3. --- setup.py | 1 + ulc_mm_package/QtGUI/acquisition.py | 2 +- ulc_mm_package/QtGUI/dev_run.py | 10 + ulc_mm_package/QtGUI/form_gui.py | 2 +- ulc_mm_package/QtGUI/liveview_gui.py | 2 +- ulc_mm_package/QtGUI/oracle.py | 23 +- ulc_mm_package/QtGUI/scope_op.py | 22 +- ulc_mm_package/hardware/scope.py | 2 +- ulc_mm_package/hardware/scope_routines.py | 12 +- .../image_processing/data_storage.py | 8 +- ulc_mm_package/neural_nets/NCSModel.py | 3 - .../neural_nets/neural_network_constants.py | 6 +- .../neural_nets/predictions_handler.py | 13 - ulc_mm_package/scope_constants.py | 4 + .../parasitemia_visualization.py | 1 - ulc_mm_package/utilities/email_utils.py | 180 ++++++++++++++ ulc_mm_package/utilities/ngrok_utils.py | 226 ++++++++++++++++++ 17 files changed, 458 insertions(+), 59 deletions(-) create mode 100644 ulc_mm_package/utilities/email_utils.py create mode 100644 ulc_mm_package/utilities/ngrok_utils.py diff --git a/setup.py b/setup.py index 84685f0e0..30197b643 100644 --- a/setup.py +++ b/setup.py @@ -43,6 +43,7 @@ def readme(): "typer==0.4.1", "tqdm==4.63.0", "transitions==0.8.11", + "pyngrok==7.0.3", "numba==0.56.0", "Jinja2==3.1.3", "xhtml2pdf==0.2.11", diff --git a/ulc_mm_package/QtGUI/acquisition.py b/ulc_mm_package/QtGUI/acquisition.py index 2f26e148f..2ab5ed18f 100644 --- a/ulc_mm_package/QtGUI/acquisition.py +++ b/ulc_mm_package/QtGUI/acquisition.py @@ -1,4 +1,4 @@ -"""Image manager +""" Image manager Receives images from the camera and sends them to Liveview and ScopeOp. diff --git a/ulc_mm_package/QtGUI/dev_run.py b/ulc_mm_package/QtGUI/dev_run.py index 9b8d7c079..5499276ff 100644 --- a/ulc_mm_package/QtGUI/dev_run.py +++ b/ulc_mm_package/QtGUI/dev_run.py @@ -49,6 +49,9 @@ from ulc_mm_package.image_processing.processing_constants import FLOWRATE +from ulc_mm_package.utilities.ngrok_utils import make_tcp_tunnel, NgrokError +from ulc_mm_package.utilities.email_utils import send_ngrok_email + from ulc_mm_package.neural_nets.AutofocusInference import AutoFocus import ulc_mm_package.neural_nets.neural_network_constants as nn_constants @@ -633,6 +636,13 @@ def __init__(self, *args, **kwargs): # Misc self.fan.turn_on_all() self.btnExit.clicked.connect(self.exit) + try: + ngrok_address = make_tcp_tunnel() + send_ngrok_email() + except NgrokError as e: + print(f"Ngrok error : {e}") + ngrok_address = "-ngrok error-" + self.lblngrok.setText(f"{ngrok_address}") # Set slider min/max self.min_exposure_us = 100 diff --git a/ulc_mm_package/QtGUI/form_gui.py b/ulc_mm_package/QtGUI/form_gui.py index 95a34f1be..1d54f8372 100644 --- a/ulc_mm_package/QtGUI/form_gui.py +++ b/ulc_mm_package/QtGUI/form_gui.py @@ -1,4 +1,4 @@ -"""Experiment form GUI window +""" Experiment form GUI window Takes user input and exports experiment metadata. diff --git a/ulc_mm_package/QtGUI/liveview_gui.py b/ulc_mm_package/QtGUI/liveview_gui.py index 2b2b3718f..1d30836e7 100644 --- a/ulc_mm_package/QtGUI/liveview_gui.py +++ b/ulc_mm_package/QtGUI/liveview_gui.py @@ -1,4 +1,4 @@ -"""Liveview GUI window +""" Liveview GUI window Displays camera preview and conveys info to user during runs.""" diff --git a/ulc_mm_package/QtGUI/oracle.py b/ulc_mm_package/QtGUI/oracle.py index 3ae483163..005b7cf0d 100644 --- a/ulc_mm_package/QtGUI/oracle.py +++ b/ulc_mm_package/QtGUI/oracle.py @@ -1,4 +1,4 @@ -"""High-level state machine manager. +""" High-level state machine manager. The Oracle sees all and knows all. It owns all GUI windows, threads, and worker objects (ScopeOp and Acquisition). @@ -70,6 +70,8 @@ AUTOFOCUS_MODEL_DIR, YOGO_MODEL_DIR, ) +from ulc_mm_package.utilities.email_utils import send_ngrok_email, EmailError +from ulc_mm_package.utilities.ngrok_utils import make_tcp_tunnel, NgrokError from ulc_mm_package.QtGUI.scope_op import ScopeOp from ulc_mm_package.QtGUI.form_gui import FormGUI @@ -157,7 +159,24 @@ def __init__(self): self.next_state() def _init_tcp(self): - self.liveview_window.update_tcp("unavailable") + try: + tcp_addr = make_tcp_tunnel() + self.logger.info(f"SSH address is {tcp_addr}.") + self.liveview_window.update_tcp(tcp_addr) + send_ngrok_email() + except NgrokError as e: + self.logger.warning( + f"SSH address could not be found - {e}. This can be safely ignored." + ) + self.liveview_window.update_tcp("unavailable") + except EmailError as e: + self.logger.warning( + f"SSH address could not be emailed - {e}. This can be safely ignored." + ) + except Exception as e: + self.logger.warning( + f"Unexpected error while setting up TCP: {e}. This can be safely ignored." + ) def _check_lock(self): if path.isfile(LOCKFILE): diff --git a/ulc_mm_package/QtGUI/scope_op.py b/ulc_mm_package/QtGUI/scope_op.py index c7ecb092b..5717c9af5 100644 --- a/ulc_mm_package/QtGUI/scope_op.py +++ b/ulc_mm_package/QtGUI/scope_op.py @@ -1,4 +1,4 @@ -"""Mid-level/hardware state machine manager +""" Mid-level/hardware state machine manager Controls hardware (ie. the Scope) operations. Manages hardware routines and interactions with Oracle and Acquisition. @@ -879,9 +879,6 @@ def run_experiment(self, img, timestamp) -> None: self.mscope.cell_diagnosis_model.work_queue_size(), ) - # ------------------------------------ - # Get and process YOGO results - # ------------------------------------ t0 = perf_counter() for result in prev_yogo_results: self.mscope.predictions_handler.add_yogo_pred(result) @@ -910,9 +907,6 @@ def run_experiment(self, img, timestamp) -> None: t1 = perf_counter() self._update_metadata_if_verbose("yogo_result_mgmt", t1 - t0) - # ------------------------------------ - # Run periodic singleshot autofocus routine - # ------------------------------------ t0 = perf_counter() resized_img = cv2.resize(img, IMG_RESIZED_DIMS, interpolation=cv2.INTER_CUBIC) try: @@ -947,10 +941,8 @@ def run_experiment(self, img, timestamp) -> None: if filtered_focus_err is not None: self.filtered_focus_err = filtered_focus_err - # ------------------------------------ - # Get classic image sharpness metric - # ------------------------------------ t0 = perf_counter() + # Downsample image for use in flowrate + classic image focus metric img_ds_10x = downsample_image(img, 10) try: @@ -963,10 +955,6 @@ def run_experiment(self, img, timestamp) -> None: ) self.oof_to_motor_sweep() return - - # ------------------------------------ - # Run flow control routine - # ------------------------------------ try: self.flowrate, _ = self.flowcontrol_routine.send((img_ds_10x, timestamp)) except Exception as e: @@ -979,15 +967,11 @@ def run_experiment(self, img, timestamp) -> None: t1 = perf_counter() self._update_metadata_if_verbose("flowrate_dt", t1 - t0) - # ------------------------------------ # Run periodic autobrightness routine - # ------------------------------------ curr_mean_pixel_val = self.periodic_autobrightness_routine.send(resized_img) - # ------------------------------------ - # Update remaining metadata in per-image csv - # ------------------------------------ t0 = perf_counter() + # Update remaining metadata self.img_metadata["motor_pos"] = self.mscope.motor.getCurrentPosition() try: pressure, status = self.mscope.pneumatic_module.getPressure() diff --git a/ulc_mm_package/hardware/scope.py b/ulc_mm_package/hardware/scope.py index e337eb1a3..478a6137b 100644 --- a/ulc_mm_package/hardware/scope.py +++ b/ulc_mm_package/hardware/scope.py @@ -153,7 +153,7 @@ def reset_for_end_experiment(self) -> None: self.cell_diagnosis_model.reset(wait_for_jobs=False) # Reset predictions handler - self.predictions_handler.reset() + self.predictions_handler: PredictionsHandler = PredictionsHandler() def shutoff(self): self.logger.info("Shutting off scope hardware.") diff --git a/ulc_mm_package/hardware/scope_routines.py b/ulc_mm_package/hardware/scope_routines.py index 61521b850..bee2a20e1 100644 --- a/ulc_mm_package/hardware/scope_routines.py +++ b/ulc_mm_package/hardware/scope_routines.py @@ -15,6 +15,7 @@ checkLedWorking, ) from ulc_mm_package.image_processing.flow_control import ( + FlowController, CantReachTargetFlowrate, ) from ulc_mm_package.image_processing.cell_finder import ( @@ -248,7 +249,6 @@ def flow_control_routine( flow_controller.set_alpha( processing_constants.FLOW_CONTROL_EWMA_ALPHA * 2 ) # Double the alpha, ~halve the half life - flow_controller.pneumatic_module.min_step_size *= 2 while True: img, timestamp = yield flow_val, syringe_can_move @@ -271,7 +271,6 @@ def flow_control_routine( if fast_flow: if flow_error is not None: if flow_error == 0: - flow_controller.pneumatic_module.min_step_size /= 2 return flow_val @init_generator @@ -523,9 +522,7 @@ def find_cells_routine( # Maximum number of times to run check for cells routine before aborting max_attempts = 3 cell_finder = CellFinder() - mscope.flow_controller.reset() - flow_controller = mscope.flow_controller - + flow_controller = FlowController(mscope.pneumatic_module) img = yield # Initial check for cells, return current motor position if cells found @@ -535,11 +532,6 @@ def find_cells_routine( except NoCellsFound: cell_finder.reset() - # Defensive check, ensure the motor isn't moving (say for example, - # if CellFinder was triggered by an OOF exception and SSAF just triggered a motor move) - while mscope.motor.is_locked(): - pass - while True: """ 1. Pull syringe for pull_time seconds (unless deliberately skipped) diff --git a/ulc_mm_package/image_processing/data_storage.py b/ulc_mm_package/image_processing/data_storage.py index 942c7e3f3..8d5e6f7ac 100644 --- a/ulc_mm_package/image_processing/data_storage.py +++ b/ulc_mm_package/image_processing/data_storage.py @@ -298,16 +298,16 @@ def close( summary_report_dir / f"{self.time_str}_per_image_metadata_plot.jpg" ) + per_img_metadata_file = open(self.per_img_metadata_filename, "r") counts_plot_loc = str(summary_report_dir / "counts.jpg") conf_plot_loc = str(summary_report_dir / "confs.jpg") objectness_plot_loc = str(summary_report_dir / "objectness.jpg") # Only generate additional plots if DEBUG_REPORT environment variable is set to True if DEBUG_REPORT: - with open(self.per_img_metadata_filename, "r") as per_img_metadata_file: - make_per_image_metadata_plots( - per_img_metadata_file, per_image_metadata_plot_save_loc - ) + make_per_image_metadata_plots( + per_img_metadata_file, per_image_metadata_plot_save_loc + ) try: make_cell_count_plot(pred_tensors, counts_plot_loc) diff --git a/ulc_mm_package/neural_nets/NCSModel.py b/ulc_mm_package/neural_nets/NCSModel.py index 2adfca7e2..c7e318665 100644 --- a/ulc_mm_package/neural_nets/NCSModel.py +++ b/ulc_mm_package/neural_nets/NCSModel.py @@ -116,9 +116,6 @@ def _compile_model( compiled_model = self.core.compile_model( model, self.device_name, - config={ - "PERFORMANCE_HINT": "THROUGHPUT", - }, ) self.connected = True return compiled_model diff --git a/ulc_mm_package/neural_nets/neural_network_constants.py b/ulc_mm_package/neural_nets/neural_network_constants.py index 9f8c4f8ac..63a7e789e 100644 --- a/ulc_mm_package/neural_nets/neural_network_constants.py +++ b/ulc_mm_package/neural_nets/neural_network_constants.py @@ -7,14 +7,14 @@ # ================ Autofocus constants ================ # -AF_PERIOD_S = 0.1 # (10 imgs/sec) +AF_PERIOD_S = 0.5 AF_PERIOD_NUM = int( AF_PERIOD_S * ACQUISITION_FPS ) # Used for periodic (ie. EWMA) autofocus -AF_BATCH_SIZE = 20 # Used for single shot autofocus +AF_BATCH_SIZE = 10 # Used for single shot autofocus AF_THRESHOLD = 2 -AF_QSIZE = 25 +AF_QSIZE = 10 # For AF_PERIOD_S = 0.5, we have a max delay of 5 sec AUTOFOCUS_MODEL_NAME = "fast-cosmos-557" AUTOFOCUS_MODEL_DIR = str( diff --git a/ulc_mm_package/neural_nets/predictions_handler.py b/ulc_mm_package/neural_nets/predictions_handler.py index ba3fd01e3..80196964b 100644 --- a/ulc_mm_package/neural_nets/predictions_handler.py +++ b/ulc_mm_package/neural_nets/predictions_handler.py @@ -71,19 +71,6 @@ def __init__(self): # Setup heatmap masking self.heatmaps = np.zeros((len(YOGO_CLASS_LIST), sy * sx)) - def reset(self): - self.pred_tensors.fill(0) - self.new_pred_pointer = 0 - self.max_confs = {x: [] for x in self.class_ids} - self.curr_min_of_max_confs_by_class = { - x: HIGH_CONF_THRESH - 1e-6 for x in self.class_ids - } - self.min_confs = {x: [] for x in self.class_ids} - self.curr_max_of_min_confs_by_class = { - x: HIGH_CONF_THRESH for x in self.class_ids - } - self.heatmaps.fill(0) - def add_raw_pred_to_heatmap(self, yogo_res: AsyncInferenceResult) -> None: """Add the raw YOGO prediction to the heatmap. diff --git a/ulc_mm_package/scope_constants.py b/ulc_mm_package/scope_constants.py index a973bdba0..14b7e4a9c 100644 --- a/ulc_mm_package/scope_constants.py +++ b/ulc_mm_package/scope_constants.py @@ -200,6 +200,10 @@ def IMG_HEIGHT(self) -> int: if VERBOSE: PER_IMAGE_METADATA_KEYS.extend(PER_IMAGE_TIMING_KEYS) +# ================ Environment variables ================ # +NGROK_AUTH_TOKEN_ENV_VAR = "NGROK_AUTH_TOKEN" +EMAIL_PW_TOKEN = "GMAIL_TOKEN" + # ================ SSD directory constants ================ # SSD_NAME = "SamsungSSD" if SIMULATION: diff --git a/ulc_mm_package/summary_report/parasitemia_visualization.py b/ulc_mm_package/summary_report/parasitemia_visualization.py index 728eb31ce..83ab46b2c 100644 --- a/ulc_mm_package/summary_report/parasitemia_visualization.py +++ b/ulc_mm_package/summary_report/parasitemia_visualization.py @@ -169,7 +169,6 @@ def make_parasitemia_plot(parasitemia, err, savefile): fig.subplots_adjust(left=0.1, right=0.9, top=0.8, bottom=0.1) fig.tight_layout() plt.savefig(savefile) - plt.close() if __name__ == "__main__": diff --git a/ulc_mm_package/utilities/email_utils.py b/ulc_mm_package/utilities/email_utils.py new file mode 100644 index 000000000..f02ccbb86 --- /dev/null +++ b/ulc_mm_package/utilities/email_utils.py @@ -0,0 +1,180 @@ +import os +import logging +from datetime import datetime +import smtplib +import email.message +import socket + +from ulc_mm_package.scope_constants import EMAIL_PW_TOKEN +from ulc_mm_package.utilities.ngrok_utils import make_tcp_tunnel, NgrokError + + +DEFAULT_EMAIL_LINE = ( + "A million miles away and it's you who has the key to my tcp tunnel <3" # Default +) +DATE_FMT = "%Y-%m-%d" + +logger = logging.getLogger(__name__) + + +class EmailError(Exception): + pass + + +class EmailPWNotSet(Exception): + pass + + +def send_email(sender: str, receiver: str, subject: str, payload: str) -> None: + """Send an email + + Parameters + ---------- + + Exceptions + ---------- + EmailError - EmailPWNotSet + Raised if the GMAIL_TOKEN environment variable is not set in /home/pi/.bashrc + """ + + # Set up email object + msg = email.message.EmailMessage() + msg["From"] = sender + msg["To"] = receiver + msg["Subject"] = subject + msg.add_header("Content-Type", "text") + msg.set_content(payload) + + # creates SMTP session, start TLS + try: + s = smtplib.SMTP("smtp.gmail.com", 587) + s.starttls() + except Exception as e: + raise EmailError(f"Errored when trying to start tls - {e}") + + # Authentication + try: + token = _get_pw() + s.login(sender, token) + except EmailError: + raise + + s.send_message(msg) + + +def send_ngrok_email( + sender: str = "lfmscope@gmail.com", + receiver: str = "lfmscope@gmail.com", +) -> None: + """Send an email with the ngrok address. + + Parameters + ---------- + sender: str + Sending email + receiver: str + Receipient email + + Exceptions + ---------- + NgrokError: + Raised if there's an issue creating/returning the ngrok address. + """ + + try: + ngrok_addr: str = make_tcp_tunnel() + except NgrokError: + raise + scope_name = _get_scope_name() + subject = f"{scope_name} - {ngrok_addr}" + curr_time = datetime.now().strftime("%Y-%m-%d-%H%M%S") + msg = ( + f"Current time: {curr_time}\n" + f"Scope : {scope_name}\n" + f"ngrok address : {ngrok_addr}\n" + f"{_load_saga()}" + ) + try: + send_email(sender, receiver, subject, msg) + except: + raise + + +def _get_pw() -> str: + """ + Get the gmail device-specific password stored in the environment variables. + + Returns + ------- + str + """ + + pw = os.environ.get(EMAIL_PW_TOKEN) + if pw is None: + raise EmailError( + f"{EMAIL_PW_TOKEN} environment variable not set.\n" + "You can set the email token in the .bashrc file by:\n" + "Open the file with: nano /home/pi/.bashrc\n" + "then add the following line (without the '<' '>' signs) to the file:\n" + f"export {EMAIL_PW_TOKEN}=" + ) + return pw + + +def _get_scope_name() -> str: + """Return the hostname (i.e lfm-ohmu)""" + + return socket.gethostname() + + +def _parse_date_str(datetime_str: str, fmt: str = DATE_FMT) -> datetime: + return datetime.strptime(datetime_str, fmt) + + +def _get_days_since_inception(reset: bool = False) -> int: + if reset: + start_date = os.environ.get("INCEPTION", datetime.now().strftime(DATE_FMT)) + os.environ["INCEPTION"] = datetime.now().strftime(DATE_FMT) + return 0 + else: + start_date = os.environ.get("INCEPTION", datetime.now().strftime(DATE_FMT)) + curr_date = datetime.now().strftime(DATE_FMT) + start_datetime, curr_datetime = _parse_date_str(start_date), _parse_date_str( + curr_date + ) + return (curr_datetime - start_datetime).days + + +def _get_saga_line() -> str: + import csv + + file_list = [x for x in os.listdir(".") if "_saga.txt" in x] + file = file_list[0] if len(file_list) > 0 else None + if file is None: + _get_days_since_inception(reset=True) + saga_counter = int(os.environ.get("SAGA_COUNTER", "0")) + os.environ["SAGA_COUNTER"] = "0" + return DEFAULT_EMAIL_LINE + saga_counter = int(os.environ.get("SAGA_COUNTER", "0")) + with open(file, "r") as f: + reader = csv.reader(f, delimiter="^") + for i, row in enumerate(reader): + if i == saga_counter: + os.environ["SAGA_COUNTER"] = str(saga_counter + 1) + return row[0] + return DEFAULT_EMAIL_LINE + + +def _load_saga() -> str: + days_since_inception = _get_days_since_inception() + line = _get_saga_line() + return f"Day {days_since_inception}: {line}" + + +if __name__ == "__main__": + try: + send_ngrok_email() + except NgrokError as e: + print(f"Error when sending email - NgrokError: {e}") + except EmailError as e: + print(f"Error when sending email - EmailError: {e}") diff --git a/ulc_mm_package/utilities/ngrok_utils.py b/ulc_mm_package/utilities/ngrok_utils.py new file mode 100644 index 000000000..64e47fe48 --- /dev/null +++ b/ulc_mm_package/utilities/ngrok_utils.py @@ -0,0 +1,226 @@ +import os +import json +import logging +import subprocess + +from urllib.error import URLError +from urllib.request import urlopen +from typing import Dict + +from pyngrok import ngrok, conf + +from ulc_mm_package.scope_constants import NGROK_AUTH_TOKEN_ENV_VAR + +logger = logging.getLogger(__name__) + + +class NgrokError(Exception): + pass + + +class AuthTokenNotSet(NgrokError): + pass + + +def _get_ngrok_json() -> Dict: + """Calls the ngrok localhost status page and returns the status dictionary. + + Returns + ------- + Dict: + Dictionary of ngrok status + + Exceptions + ---------- + URLError: + If ngrok is not running, there will be nothing at this URL. + """ + + addr = "http://127.0.0.1:4040/api/tunnels" + try: + content = urlopen(addr).read().decode("utf-8") + return json.loads(content) + except URLError as e: + logger.info("Address unavailable - ngrok is not on.") + raise e + + +def is_ngrok_running() -> bool: + """Check whether ngrok is running. + + Returns + ------- + bool: + Is ngrok running + """ + + addr = "http://127.0.0.1:4040/api/tunnels" + try: + _ = urlopen(addr).read().decode("utf-8") + return True + except URLError: + return False + + +def _get_addr_from_json(json_dict: Dict): + """Get the public_url from the json.""" + return json_dict["tunnels"][0]["public_url"] + + +def get_addr() -> str: + """Get the public accessible ngrok URL. + + Note: + The user should first check that ngrok is running (`is_ngrok_running()`) + + Returns + ------- + str: + ngrok public url + """ + ngrok_json = _get_ngrok_json() + return _get_addr_from_json(ngrok_json) + + +def _make_tcp_tunnel() -> ngrok.NgrokTunnel: + """Attempt to create an ngrok tcp tunnel. Return an existing tunnel if one is already open. + + Returns + ------- + pyngrok.NgrokTunnel: + pyngrok object + + Exceptions + ---------- + NgrokError: + Unable to create the tunnel. + """ + try: + return ngrok.connect(22, "tcp") + except ngrok.PyngrokError as e: + raise NgrokError(e) + + +def _get_public_url_from_ngrok_tunnel_obj(tunnel_obj: ngrok.NgrokTunnel) -> str: + """Ingest a pyngrok object and return the publicly accessible URL. + + Parameters + ---------- + pyngrok.NgrokTunnel obj + + Returns + ------- + str + """ + return tunnel_obj.public_url + + +def _kill_old_ngrok_sessions() -> None: + """Ensure any old ngrok tunnels are terminated. + + The free-tier account is limited to one active tunnel. Ensure that any stale + sessions are terminated before a new one is made. + + Exceptions + ---------- + None: + Catch-all which logs the exception+traceback + """ + + try: + # Redirect subprocess output to DEVNULL to avoid cluttering the console. + subprocess.run( + ["killall", "ngrok"], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT + ) + except Exception as e: + logger.exception(f"Unknown failure when attempting to `killall ngrok`: {e}") + + +def _create_new_tunnel() -> str: + _kill_old_ngrok_sessions() + set_ngrok_auth_token() + return _get_public_url_from_ngrok_tunnel_obj(_make_tcp_tunnel()) + + +def make_tcp_tunnel() -> str: + """Returns the publicly accessible ngrok ssh address. + + Returns + ------- + str: + Publicly accessible ngrok URL. + + Exceptions + ---------- + NgrokError: + Unable to create the ngrok tunnel. + """ + + try: + # Check for existing ngrok tunnel + if is_ngrok_running(): + try: + addr = get_addr() + return addr + except (URLError, IndexError): + logger.error( + "ngrok is running but unable to get address from api/tunnels." + ) + _kill_old_ngrok_sessions() + set_ngrok_auth_token() + return _get_public_url_from_ngrok_tunnel_obj(_make_tcp_tunnel()) + else: + # Create a new tunnel + try: + return _create_new_tunnel() + except NgrokError: + raise + except: + raise NgrokError( + "NgrokError : existing ngrok tunnel detected but errored out during either is_ngrok_running() or get_addr()." + ) + + +def _get_ngrok_auth_token() -> str: + """Get the ngrok token stored in the environment variable. + + Returns + ------- + str + """ + + token = os.environ.get(NGROK_AUTH_TOKEN_ENV_VAR) + if token is None: + raise AuthTokenNotSet( + f"{NGROK_AUTH_TOKEN_ENV_VAR} environment variable not set.\n" + "You can set the ngrok token in the .bashrc file by:\n" + "Open the file with: nano /home/pi/.bashrc\n" + "then add the following line (without the '<' '>' signs) to the file:\n" + f"export {NGROK_AUTH_TOKEN_ENV_VAR}=" + ) + else: + return token + + +def _set_ngrok_auth_token(token: str) -> None: + """Set the ngrok token.""" + + ngrok.set_auth_token(token) + conf.get_default().auth_token = token + + +def set_ngrok_auth_token(): + """Attempt to set the ngrok token. + + Exceptions + ---------- + NgrokError.AuthTokenNotSet + """ + + token = _get_ngrok_auth_token() + _set_ngrok_auth_token(token) + + +if __name__ == "__main__": + print(f"{make_tcp_tunnel()}") + input("Press enter to exit and close the tunnel...") From 09eff4d66e0f419243e4d105ad9240011466fd56 Mon Sep 17 00:00:00 2001 From: i-jey Date: Mon, 10 Mar 2025 12:22:09 -0700 Subject: [PATCH 23/25] rollback to PR607 merge --- ulc_mm_package/QtGUI/acquisition.py | 2 +- ulc_mm_package/QtGUI/dev_run.py | 10 ---- ulc_mm_package/QtGUI/form_gui.py | 2 +- ulc_mm_package/QtGUI/liveview_gui.py | 2 +- ulc_mm_package/QtGUI/oracle.py | 36 ++------------ ulc_mm_package/QtGUI/scope_op.py | 33 +++++++------ ulc_mm_package/hardware/hardware_constants.py | 1 - ulc_mm_package/hardware/scope.py | 2 +- ulc_mm_package/hardware/scope_routines.py | 48 ++++--------------- .../image_processing/data_storage.py | 8 ++-- ulc_mm_package/neural_nets/NCSModel.py | 3 ++ .../neural_nets/neural_network_constants.py | 6 +-- .../neural_nets/predictions_handler.py | 13 +++++ ulc_mm_package/scope_constants.py | 4 -- .../parasitemia_visualization.py | 1 + 15 files changed, 60 insertions(+), 111 deletions(-) diff --git a/ulc_mm_package/QtGUI/acquisition.py b/ulc_mm_package/QtGUI/acquisition.py index 2ab5ed18f..2f26e148f 100644 --- a/ulc_mm_package/QtGUI/acquisition.py +++ b/ulc_mm_package/QtGUI/acquisition.py @@ -1,4 +1,4 @@ -""" Image manager +"""Image manager Receives images from the camera and sends them to Liveview and ScopeOp. diff --git a/ulc_mm_package/QtGUI/dev_run.py b/ulc_mm_package/QtGUI/dev_run.py index 5499276ff..9b8d7c079 100644 --- a/ulc_mm_package/QtGUI/dev_run.py +++ b/ulc_mm_package/QtGUI/dev_run.py @@ -49,9 +49,6 @@ from ulc_mm_package.image_processing.processing_constants import FLOWRATE -from ulc_mm_package.utilities.ngrok_utils import make_tcp_tunnel, NgrokError -from ulc_mm_package.utilities.email_utils import send_ngrok_email - from ulc_mm_package.neural_nets.AutofocusInference import AutoFocus import ulc_mm_package.neural_nets.neural_network_constants as nn_constants @@ -636,13 +633,6 @@ def __init__(self, *args, **kwargs): # Misc self.fan.turn_on_all() self.btnExit.clicked.connect(self.exit) - try: - ngrok_address = make_tcp_tunnel() - send_ngrok_email() - except NgrokError as e: - print(f"Ngrok error : {e}") - ngrok_address = "-ngrok error-" - self.lblngrok.setText(f"{ngrok_address}") # Set slider min/max self.min_exposure_us = 100 diff --git a/ulc_mm_package/QtGUI/form_gui.py b/ulc_mm_package/QtGUI/form_gui.py index 1d54f8372..95a34f1be 100644 --- a/ulc_mm_package/QtGUI/form_gui.py +++ b/ulc_mm_package/QtGUI/form_gui.py @@ -1,4 +1,4 @@ -""" Experiment form GUI window +"""Experiment form GUI window Takes user input and exports experiment metadata. diff --git a/ulc_mm_package/QtGUI/liveview_gui.py b/ulc_mm_package/QtGUI/liveview_gui.py index 1d30836e7..2b2b3718f 100644 --- a/ulc_mm_package/QtGUI/liveview_gui.py +++ b/ulc_mm_package/QtGUI/liveview_gui.py @@ -1,4 +1,4 @@ -""" Liveview GUI window +"""Liveview GUI window Displays camera preview and conveys info to user during runs.""" diff --git a/ulc_mm_package/QtGUI/oracle.py b/ulc_mm_package/QtGUI/oracle.py index 005b7cf0d..0dccb90f8 100644 --- a/ulc_mm_package/QtGUI/oracle.py +++ b/ulc_mm_package/QtGUI/oracle.py @@ -1,4 +1,4 @@ -""" High-level state machine manager. +"""High-level state machine manager. The Oracle sees all and knows all. It owns all GUI windows, threads, and worker objects (ScopeOp and Acquisition). @@ -70,8 +70,6 @@ AUTOFOCUS_MODEL_DIR, YOGO_MODEL_DIR, ) -from ulc_mm_package.utilities.email_utils import send_ngrok_email, EmailError -from ulc_mm_package.utilities.ngrok_utils import make_tcp_tunnel, NgrokError from ulc_mm_package.QtGUI.scope_op import ScopeOp from ulc_mm_package.QtGUI.form_gui import FormGUI @@ -159,24 +157,7 @@ def __init__(self): self.next_state() def _init_tcp(self): - try: - tcp_addr = make_tcp_tunnel() - self.logger.info(f"SSH address is {tcp_addr}.") - self.liveview_window.update_tcp(tcp_addr) - send_ngrok_email() - except NgrokError as e: - self.logger.warning( - f"SSH address could not be found - {e}. This can be safely ignored." - ) - self.liveview_window.update_tcp("unavailable") - except EmailError as e: - self.logger.warning( - f"SSH address could not be emailed - {e}. This can be safely ignored." - ) - except Exception as e: - self.logger.warning( - f"Unexpected error while setting up TCP: {e}. This can be safely ignored." - ) + self.liveview_window.update_tcp("unavailable") def _check_lock(self): if path.isfile(LOCKFILE): @@ -294,7 +275,6 @@ def _init_sigslots(self): self.scopeop.reload_pause.connect(self.reload_pause_handler) self.scopeop.lid_open_pause.connect(self.lid_open_pause_handler) - self.scopeop.pressure_leak_pause.connect(self.pressure_leak_pause_handler) self.scopeop.create_timers.connect(self.acquisition.create_timers) self.scopeop.start_timers.connect(self.acquisition.start_timers) @@ -393,17 +373,6 @@ def lid_open_pause_handler(self): self.scopeop.to_pause() self.unpause() - def pressure_leak_pause_handler(self): - if self.scopeop.state not in NO_PAUSE_STATES: - self.scopeop.to_pause() - self.display_message( - QMessageBox.Icon.Information, - "Pressure leak detected - pausing...", - 'Please open the lid and reseat the CAP module, a pressure leak has been detected. Press "OK" to resume.', - buttons=Buttons.OK, - ) - self.unpause() - def general_pause_handler( self, icon=QMessageBox.Icon.Information, @@ -440,6 +409,7 @@ def general_pause_handler( buttons=Buttons.OK, image=IMAGE_RELOAD_PATH, ) + self.close_lid_display_message() self.unpause() def unpause(self): diff --git a/ulc_mm_package/QtGUI/scope_op.py b/ulc_mm_package/QtGUI/scope_op.py index 5717c9af5..9b909a77f 100644 --- a/ulc_mm_package/QtGUI/scope_op.py +++ b/ulc_mm_package/QtGUI/scope_op.py @@ -1,4 +1,4 @@ -""" Mid-level/hardware state machine manager +"""Mid-level/hardware state machine manager Controls hardware (ie. the Scope) operations. Manages hardware routines and interactions with Oracle and Acquisition. @@ -92,7 +92,6 @@ class ScopeOp(QObject, NamedMachine): reload_pause = pyqtSignal(str, str) lid_open_pause = pyqtSignal() - pressure_leak_pause = pyqtSignal() create_timers = pyqtSignal() start_timers = pyqtSignal() @@ -413,9 +412,6 @@ def _check_pressure_seal(self, *args): self.logger.info( f"Pressure check ✅. Ambient absolute pressure: {self.ambient_pressure:.2f} mBar. Gauge pressure = {pdiff:.2f} mBar." ) - self.pressure_monitoring_routine = ( - self.routines.pressure_monitoring_routine(self.ambient_pressure) - ) if self.state == "pressure_check": self.next_state() except PressureSensorBusy as e: @@ -879,6 +875,9 @@ def run_experiment(self, img, timestamp) -> None: self.mscope.cell_diagnosis_model.work_queue_size(), ) + # ------------------------------------ + # Get and process YOGO results + # ------------------------------------ t0 = perf_counter() for result in prev_yogo_results: self.mscope.predictions_handler.add_yogo_pred(result) @@ -907,6 +906,9 @@ def run_experiment(self, img, timestamp) -> None: t1 = perf_counter() self._update_metadata_if_verbose("yogo_result_mgmt", t1 - t0) + # ------------------------------------ + # Run periodic singleshot autofocus routine + # ------------------------------------ t0 = perf_counter() resized_img = cv2.resize(img, IMG_RESIZED_DIMS, interpolation=cv2.INTER_CUBIC) try: @@ -941,8 +943,10 @@ def run_experiment(self, img, timestamp) -> None: if filtered_focus_err is not None: self.filtered_focus_err = filtered_focus_err + # ------------------------------------ + # Get classic image sharpness metric + # ------------------------------------ t0 = perf_counter() - # Downsample image for use in flowrate + classic image focus metric img_ds_10x = downsample_image(img, 10) try: @@ -955,6 +959,10 @@ def run_experiment(self, img, timestamp) -> None: ) self.oof_to_motor_sweep() return + + # ------------------------------------ + # Run flow control routine + # ------------------------------------ try: self.flowrate, _ = self.flowcontrol_routine.send((img_ds_10x, timestamp)) except Exception as e: @@ -967,15 +975,18 @@ def run_experiment(self, img, timestamp) -> None: t1 = perf_counter() self._update_metadata_if_verbose("flowrate_dt", t1 - t0) + # ------------------------------------ # Run periodic autobrightness routine + # ------------------------------------ curr_mean_pixel_val = self.periodic_autobrightness_routine.send(resized_img) + # ------------------------------------ + # Update remaining metadata in per-image csv + # ------------------------------------ t0 = perf_counter() - # Update remaining metadata self.img_metadata["motor_pos"] = self.mscope.motor.getCurrentPosition() try: pressure, status = self.mscope.pneumatic_module.getPressure() - self.pressure_monitoring_routine.send(pressure) ( self.img_metadata["pressure_hpa"], self.img_metadata["pressure_status_flag"], @@ -983,12 +994,6 @@ def run_experiment(self, img, timestamp) -> None: except PressureSensorStaleValue as e: ## TODO??? self.logger.info(f"Stale pressure sensor value - {e}") - except PressureLeak: - self.logger.warning( - f"Pressure leak detected. Current pressure: {pressure:.2f}mBar (gauge: {pressure - self.ambient_pressure:.2f}mBar)." - ) - self.pressure_leak_pause.emit() - return self.img_metadata["led_pwm_val"] = self.mscope.led.pwm_duty_cycle self.img_metadata[ diff --git a/ulc_mm_package/hardware/hardware_constants.py b/ulc_mm_package/hardware/hardware_constants.py index 7cf44a736..277bd7b26 100644 --- a/ulc_mm_package/hardware/hardware_constants.py +++ b/ulc_mm_package/hardware/hardware_constants.py @@ -78,7 +78,6 @@ MPRLS_PWR = 22 MIN_PRESSURE_DIFF = 330 # In units of hPa -PRESSURE_EWMA_ALPHA = 0.1 # ================ Fan constants ================ # FAN_GPIO = 5 CAM_FAN_1 = 23 diff --git a/ulc_mm_package/hardware/scope.py b/ulc_mm_package/hardware/scope.py index 478a6137b..e337eb1a3 100644 --- a/ulc_mm_package/hardware/scope.py +++ b/ulc_mm_package/hardware/scope.py @@ -153,7 +153,7 @@ def reset_for_end_experiment(self) -> None: self.cell_diagnosis_model.reset(wait_for_jobs=False) # Reset predictions handler - self.predictions_handler: PredictionsHandler = PredictionsHandler() + self.predictions_handler.reset() def shutoff(self): self.logger.info("Shutting off scope hardware.") diff --git a/ulc_mm_package/hardware/scope_routines.py b/ulc_mm_package/hardware/scope_routines.py index bee2a20e1..ccd9b98de 100644 --- a/ulc_mm_package/hardware/scope_routines.py +++ b/ulc_mm_package/hardware/scope_routines.py @@ -15,7 +15,6 @@ checkLedWorking, ) from ulc_mm_package.image_processing.flow_control import ( - FlowController, CantReachTargetFlowrate, ) from ulc_mm_package.image_processing.cell_finder import ( @@ -31,7 +30,6 @@ from ulc_mm_package.hardware.motorcontroller import Direction, MotorControllerError from ulc_mm_package.hardware.hardware_constants import ( MIN_PRESSURE_DIFF, - PRESSURE_EWMA_ALPHA, FOCUS_EWMA_ALPHA, ) from ulc_mm_package.image_processing.classic_focus import OOF, ClassicImageFocus @@ -249,6 +247,7 @@ def flow_control_routine( flow_controller.set_alpha( processing_constants.FLOW_CONTROL_EWMA_ALPHA * 2 ) # Double the alpha, ~halve the half life + flow_controller.pneumatic_module.min_step_size *= 2 while True: img, timestamp = yield flow_val, syringe_can_move @@ -271,6 +270,7 @@ def flow_control_routine( if fast_flow: if flow_error is not None: if flow_error == 0: + flow_controller.pneumatic_module.min_step_size /= 2 return flow_val @init_generator @@ -434,41 +434,6 @@ def checkPressureDifference( ) return pressure_diff - @init_generator - def pressure_monitoring_routine( - self, ambient_pressure: float - ) -> Generator[None, float, None]: - """ - Monitor the pressure and raise an exception if it drops below the minimum required pressure difference. - - Parameters - ---------- - mscope: MalariaScope - - Exceptions - ---------- - PressureLeak: - Raised if the pressure difference is less than the minimum required (as set in `hardware_constants.py` via MIN_PRESSURE_DIFF). - """ - - pressure_ewma_filter = EWMAFiltering(PRESSURE_EWMA_ALPHA) - pressure_ewma_filter.set_init_val(ambient_pressure) - period_num = pressure_ewma_filter.get_adjustment_period_ewma() - counter = 0 - - while True: - counter += 1 - curr_pressure = yield - filtered_pressure = pressure_ewma_filter.update_and_get_val(curr_pressure) - gauge_pressure = ambient_pressure - filtered_pressure - - if counter > period_num: - if gauge_pressure < MIN_PRESSURE_DIFF: - raise PressureLeak( - f"Pressure leak detected. Could only generate {gauge_pressure:.3f}mBar of pressure difference (ambient is at: {ambient_pressure:.2f}mBar)." - ) - counter = 0 - @init_generator def find_cells_routine( self, @@ -522,7 +487,9 @@ def find_cells_routine( # Maximum number of times to run check for cells routine before aborting max_attempts = 3 cell_finder = CellFinder() - flow_controller = FlowController(mscope.pneumatic_module) + mscope.flow_controller.reset() + flow_controller = mscope.flow_controller + img = yield # Initial check for cells, return current motor position if cells found @@ -532,6 +499,11 @@ def find_cells_routine( except NoCellsFound: cell_finder.reset() + # Defensive check, ensure the motor isn't moving (say for example, + # if CellFinder was triggered by an OOF exception and SSAF just triggered a motor move) + while mscope.motor.is_locked(): + pass + while True: """ 1. Pull syringe for pull_time seconds (unless deliberately skipped) diff --git a/ulc_mm_package/image_processing/data_storage.py b/ulc_mm_package/image_processing/data_storage.py index 8d5e6f7ac..942c7e3f3 100644 --- a/ulc_mm_package/image_processing/data_storage.py +++ b/ulc_mm_package/image_processing/data_storage.py @@ -298,16 +298,16 @@ def close( summary_report_dir / f"{self.time_str}_per_image_metadata_plot.jpg" ) - per_img_metadata_file = open(self.per_img_metadata_filename, "r") counts_plot_loc = str(summary_report_dir / "counts.jpg") conf_plot_loc = str(summary_report_dir / "confs.jpg") objectness_plot_loc = str(summary_report_dir / "objectness.jpg") # Only generate additional plots if DEBUG_REPORT environment variable is set to True if DEBUG_REPORT: - make_per_image_metadata_plots( - per_img_metadata_file, per_image_metadata_plot_save_loc - ) + with open(self.per_img_metadata_filename, "r") as per_img_metadata_file: + make_per_image_metadata_plots( + per_img_metadata_file, per_image_metadata_plot_save_loc + ) try: make_cell_count_plot(pred_tensors, counts_plot_loc) diff --git a/ulc_mm_package/neural_nets/NCSModel.py b/ulc_mm_package/neural_nets/NCSModel.py index c7e318665..2adfca7e2 100644 --- a/ulc_mm_package/neural_nets/NCSModel.py +++ b/ulc_mm_package/neural_nets/NCSModel.py @@ -116,6 +116,9 @@ def _compile_model( compiled_model = self.core.compile_model( model, self.device_name, + config={ + "PERFORMANCE_HINT": "THROUGHPUT", + }, ) self.connected = True return compiled_model diff --git a/ulc_mm_package/neural_nets/neural_network_constants.py b/ulc_mm_package/neural_nets/neural_network_constants.py index 63a7e789e..9f8c4f8ac 100644 --- a/ulc_mm_package/neural_nets/neural_network_constants.py +++ b/ulc_mm_package/neural_nets/neural_network_constants.py @@ -7,14 +7,14 @@ # ================ Autofocus constants ================ # -AF_PERIOD_S = 0.5 +AF_PERIOD_S = 0.1 # (10 imgs/sec) AF_PERIOD_NUM = int( AF_PERIOD_S * ACQUISITION_FPS ) # Used for periodic (ie. EWMA) autofocus -AF_BATCH_SIZE = 10 # Used for single shot autofocus +AF_BATCH_SIZE = 20 # Used for single shot autofocus AF_THRESHOLD = 2 -AF_QSIZE = 10 # For AF_PERIOD_S = 0.5, we have a max delay of 5 sec +AF_QSIZE = 25 AUTOFOCUS_MODEL_NAME = "fast-cosmos-557" AUTOFOCUS_MODEL_DIR = str( diff --git a/ulc_mm_package/neural_nets/predictions_handler.py b/ulc_mm_package/neural_nets/predictions_handler.py index 80196964b..ba3fd01e3 100644 --- a/ulc_mm_package/neural_nets/predictions_handler.py +++ b/ulc_mm_package/neural_nets/predictions_handler.py @@ -71,6 +71,19 @@ def __init__(self): # Setup heatmap masking self.heatmaps = np.zeros((len(YOGO_CLASS_LIST), sy * sx)) + def reset(self): + self.pred_tensors.fill(0) + self.new_pred_pointer = 0 + self.max_confs = {x: [] for x in self.class_ids} + self.curr_min_of_max_confs_by_class = { + x: HIGH_CONF_THRESH - 1e-6 for x in self.class_ids + } + self.min_confs = {x: [] for x in self.class_ids} + self.curr_max_of_min_confs_by_class = { + x: HIGH_CONF_THRESH for x in self.class_ids + } + self.heatmaps.fill(0) + def add_raw_pred_to_heatmap(self, yogo_res: AsyncInferenceResult) -> None: """Add the raw YOGO prediction to the heatmap. diff --git a/ulc_mm_package/scope_constants.py b/ulc_mm_package/scope_constants.py index 14b7e4a9c..a973bdba0 100644 --- a/ulc_mm_package/scope_constants.py +++ b/ulc_mm_package/scope_constants.py @@ -200,10 +200,6 @@ def IMG_HEIGHT(self) -> int: if VERBOSE: PER_IMAGE_METADATA_KEYS.extend(PER_IMAGE_TIMING_KEYS) -# ================ Environment variables ================ # -NGROK_AUTH_TOKEN_ENV_VAR = "NGROK_AUTH_TOKEN" -EMAIL_PW_TOKEN = "GMAIL_TOKEN" - # ================ SSD directory constants ================ # SSD_NAME = "SamsungSSD" if SIMULATION: diff --git a/ulc_mm_package/summary_report/parasitemia_visualization.py b/ulc_mm_package/summary_report/parasitemia_visualization.py index 83ab46b2c..728eb31ce 100644 --- a/ulc_mm_package/summary_report/parasitemia_visualization.py +++ b/ulc_mm_package/summary_report/parasitemia_visualization.py @@ -169,6 +169,7 @@ def make_parasitemia_plot(parasitemia, err, savefile): fig.subplots_adjust(left=0.1, right=0.9, top=0.8, bottom=0.1) fig.tight_layout() plt.savefig(savefile) + plt.close() if __name__ == "__main__": From 7f19eb838ee0148ae60f255c0d29b29fda09138d Mon Sep 17 00:00:00 2001 From: i-jey Date: Mon, 10 Mar 2025 12:25:18 -0700 Subject: [PATCH 24/25] remove old unused files --- ulc_mm_package/utilities/email_utils.py | 180 ------------------- ulc_mm_package/utilities/ngrok_utils.py | 226 ------------------------ 2 files changed, 406 deletions(-) delete mode 100644 ulc_mm_package/utilities/email_utils.py delete mode 100644 ulc_mm_package/utilities/ngrok_utils.py diff --git a/ulc_mm_package/utilities/email_utils.py b/ulc_mm_package/utilities/email_utils.py deleted file mode 100644 index f02ccbb86..000000000 --- a/ulc_mm_package/utilities/email_utils.py +++ /dev/null @@ -1,180 +0,0 @@ -import os -import logging -from datetime import datetime -import smtplib -import email.message -import socket - -from ulc_mm_package.scope_constants import EMAIL_PW_TOKEN -from ulc_mm_package.utilities.ngrok_utils import make_tcp_tunnel, NgrokError - - -DEFAULT_EMAIL_LINE = ( - "A million miles away and it's you who has the key to my tcp tunnel <3" # Default -) -DATE_FMT = "%Y-%m-%d" - -logger = logging.getLogger(__name__) - - -class EmailError(Exception): - pass - - -class EmailPWNotSet(Exception): - pass - - -def send_email(sender: str, receiver: str, subject: str, payload: str) -> None: - """Send an email - - Parameters - ---------- - - Exceptions - ---------- - EmailError - EmailPWNotSet - Raised if the GMAIL_TOKEN environment variable is not set in /home/pi/.bashrc - """ - - # Set up email object - msg = email.message.EmailMessage() - msg["From"] = sender - msg["To"] = receiver - msg["Subject"] = subject - msg.add_header("Content-Type", "text") - msg.set_content(payload) - - # creates SMTP session, start TLS - try: - s = smtplib.SMTP("smtp.gmail.com", 587) - s.starttls() - except Exception as e: - raise EmailError(f"Errored when trying to start tls - {e}") - - # Authentication - try: - token = _get_pw() - s.login(sender, token) - except EmailError: - raise - - s.send_message(msg) - - -def send_ngrok_email( - sender: str = "lfmscope@gmail.com", - receiver: str = "lfmscope@gmail.com", -) -> None: - """Send an email with the ngrok address. - - Parameters - ---------- - sender: str - Sending email - receiver: str - Receipient email - - Exceptions - ---------- - NgrokError: - Raised if there's an issue creating/returning the ngrok address. - """ - - try: - ngrok_addr: str = make_tcp_tunnel() - except NgrokError: - raise - scope_name = _get_scope_name() - subject = f"{scope_name} - {ngrok_addr}" - curr_time = datetime.now().strftime("%Y-%m-%d-%H%M%S") - msg = ( - f"Current time: {curr_time}\n" - f"Scope : {scope_name}\n" - f"ngrok address : {ngrok_addr}\n" - f"{_load_saga()}" - ) - try: - send_email(sender, receiver, subject, msg) - except: - raise - - -def _get_pw() -> str: - """ - Get the gmail device-specific password stored in the environment variables. - - Returns - ------- - str - """ - - pw = os.environ.get(EMAIL_PW_TOKEN) - if pw is None: - raise EmailError( - f"{EMAIL_PW_TOKEN} environment variable not set.\n" - "You can set the email token in the .bashrc file by:\n" - "Open the file with: nano /home/pi/.bashrc\n" - "then add the following line (without the '<' '>' signs) to the file:\n" - f"export {EMAIL_PW_TOKEN}=" - ) - return pw - - -def _get_scope_name() -> str: - """Return the hostname (i.e lfm-ohmu)""" - - return socket.gethostname() - - -def _parse_date_str(datetime_str: str, fmt: str = DATE_FMT) -> datetime: - return datetime.strptime(datetime_str, fmt) - - -def _get_days_since_inception(reset: bool = False) -> int: - if reset: - start_date = os.environ.get("INCEPTION", datetime.now().strftime(DATE_FMT)) - os.environ["INCEPTION"] = datetime.now().strftime(DATE_FMT) - return 0 - else: - start_date = os.environ.get("INCEPTION", datetime.now().strftime(DATE_FMT)) - curr_date = datetime.now().strftime(DATE_FMT) - start_datetime, curr_datetime = _parse_date_str(start_date), _parse_date_str( - curr_date - ) - return (curr_datetime - start_datetime).days - - -def _get_saga_line() -> str: - import csv - - file_list = [x for x in os.listdir(".") if "_saga.txt" in x] - file = file_list[0] if len(file_list) > 0 else None - if file is None: - _get_days_since_inception(reset=True) - saga_counter = int(os.environ.get("SAGA_COUNTER", "0")) - os.environ["SAGA_COUNTER"] = "0" - return DEFAULT_EMAIL_LINE - saga_counter = int(os.environ.get("SAGA_COUNTER", "0")) - with open(file, "r") as f: - reader = csv.reader(f, delimiter="^") - for i, row in enumerate(reader): - if i == saga_counter: - os.environ["SAGA_COUNTER"] = str(saga_counter + 1) - return row[0] - return DEFAULT_EMAIL_LINE - - -def _load_saga() -> str: - days_since_inception = _get_days_since_inception() - line = _get_saga_line() - return f"Day {days_since_inception}: {line}" - - -if __name__ == "__main__": - try: - send_ngrok_email() - except NgrokError as e: - print(f"Error when sending email - NgrokError: {e}") - except EmailError as e: - print(f"Error when sending email - EmailError: {e}") diff --git a/ulc_mm_package/utilities/ngrok_utils.py b/ulc_mm_package/utilities/ngrok_utils.py deleted file mode 100644 index 64e47fe48..000000000 --- a/ulc_mm_package/utilities/ngrok_utils.py +++ /dev/null @@ -1,226 +0,0 @@ -import os -import json -import logging -import subprocess - -from urllib.error import URLError -from urllib.request import urlopen -from typing import Dict - -from pyngrok import ngrok, conf - -from ulc_mm_package.scope_constants import NGROK_AUTH_TOKEN_ENV_VAR - -logger = logging.getLogger(__name__) - - -class NgrokError(Exception): - pass - - -class AuthTokenNotSet(NgrokError): - pass - - -def _get_ngrok_json() -> Dict: - """Calls the ngrok localhost status page and returns the status dictionary. - - Returns - ------- - Dict: - Dictionary of ngrok status - - Exceptions - ---------- - URLError: - If ngrok is not running, there will be nothing at this URL. - """ - - addr = "http://127.0.0.1:4040/api/tunnels" - try: - content = urlopen(addr).read().decode("utf-8") - return json.loads(content) - except URLError as e: - logger.info("Address unavailable - ngrok is not on.") - raise e - - -def is_ngrok_running() -> bool: - """Check whether ngrok is running. - - Returns - ------- - bool: - Is ngrok running - """ - - addr = "http://127.0.0.1:4040/api/tunnels" - try: - _ = urlopen(addr).read().decode("utf-8") - return True - except URLError: - return False - - -def _get_addr_from_json(json_dict: Dict): - """Get the public_url from the json.""" - return json_dict["tunnels"][0]["public_url"] - - -def get_addr() -> str: - """Get the public accessible ngrok URL. - - Note: - The user should first check that ngrok is running (`is_ngrok_running()`) - - Returns - ------- - str: - ngrok public url - """ - ngrok_json = _get_ngrok_json() - return _get_addr_from_json(ngrok_json) - - -def _make_tcp_tunnel() -> ngrok.NgrokTunnel: - """Attempt to create an ngrok tcp tunnel. Return an existing tunnel if one is already open. - - Returns - ------- - pyngrok.NgrokTunnel: - pyngrok object - - Exceptions - ---------- - NgrokError: - Unable to create the tunnel. - """ - try: - return ngrok.connect(22, "tcp") - except ngrok.PyngrokError as e: - raise NgrokError(e) - - -def _get_public_url_from_ngrok_tunnel_obj(tunnel_obj: ngrok.NgrokTunnel) -> str: - """Ingest a pyngrok object and return the publicly accessible URL. - - Parameters - ---------- - pyngrok.NgrokTunnel obj - - Returns - ------- - str - """ - return tunnel_obj.public_url - - -def _kill_old_ngrok_sessions() -> None: - """Ensure any old ngrok tunnels are terminated. - - The free-tier account is limited to one active tunnel. Ensure that any stale - sessions are terminated before a new one is made. - - Exceptions - ---------- - None: - Catch-all which logs the exception+traceback - """ - - try: - # Redirect subprocess output to DEVNULL to avoid cluttering the console. - subprocess.run( - ["killall", "ngrok"], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT - ) - except Exception as e: - logger.exception(f"Unknown failure when attempting to `killall ngrok`: {e}") - - -def _create_new_tunnel() -> str: - _kill_old_ngrok_sessions() - set_ngrok_auth_token() - return _get_public_url_from_ngrok_tunnel_obj(_make_tcp_tunnel()) - - -def make_tcp_tunnel() -> str: - """Returns the publicly accessible ngrok ssh address. - - Returns - ------- - str: - Publicly accessible ngrok URL. - - Exceptions - ---------- - NgrokError: - Unable to create the ngrok tunnel. - """ - - try: - # Check for existing ngrok tunnel - if is_ngrok_running(): - try: - addr = get_addr() - return addr - except (URLError, IndexError): - logger.error( - "ngrok is running but unable to get address from api/tunnels." - ) - _kill_old_ngrok_sessions() - set_ngrok_auth_token() - return _get_public_url_from_ngrok_tunnel_obj(_make_tcp_tunnel()) - else: - # Create a new tunnel - try: - return _create_new_tunnel() - except NgrokError: - raise - except: - raise NgrokError( - "NgrokError : existing ngrok tunnel detected but errored out during either is_ngrok_running() or get_addr()." - ) - - -def _get_ngrok_auth_token() -> str: - """Get the ngrok token stored in the environment variable. - - Returns - ------- - str - """ - - token = os.environ.get(NGROK_AUTH_TOKEN_ENV_VAR) - if token is None: - raise AuthTokenNotSet( - f"{NGROK_AUTH_TOKEN_ENV_VAR} environment variable not set.\n" - "You can set the ngrok token in the .bashrc file by:\n" - "Open the file with: nano /home/pi/.bashrc\n" - "then add the following line (without the '<' '>' signs) to the file:\n" - f"export {NGROK_AUTH_TOKEN_ENV_VAR}=" - ) - else: - return token - - -def _set_ngrok_auth_token(token: str) -> None: - """Set the ngrok token.""" - - ngrok.set_auth_token(token) - conf.get_default().auth_token = token - - -def set_ngrok_auth_token(): - """Attempt to set the ngrok token. - - Exceptions - ---------- - NgrokError.AuthTokenNotSet - """ - - token = _get_ngrok_auth_token() - _set_ngrok_auth_token(token) - - -if __name__ == "__main__": - print(f"{make_tcp_tunnel()}") - input("Press enter to exit and close the tunnel...") From 4ef8ba4ff8495f5da8dd62cf063e0e1c90754040 Mon Sep 17 00:00:00 2001 From: i-jey Date: Wed, 12 Mar 2025 14:44:16 -0700 Subject: [PATCH 25/25] Context: We set the pneumatic module step size to be twice what it is normally when doing fast flow, so that the set up is quicker and more ergonomic for the user (i.e they don't need to wait for a long while for the syringe to hit its stop in the event that debris or stuck cells are messing up the flow estimation calculation. Fix: This properly resets the step size back to what it is normally supposed to be, in the event where fast flow does not reach the target flow rate. Previously it was only being reset when fast flow succeeded. --- ulc_mm_package/hardware/real/pneumatic_module.py | 3 ++- ulc_mm_package/hardware/scope_routines.py | 14 ++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/ulc_mm_package/hardware/real/pneumatic_module.py b/ulc_mm_package/hardware/real/pneumatic_module.py index 642dc35a1..36fe8bb2d 100644 --- a/ulc_mm_package/hardware/real/pneumatic_module.py +++ b/ulc_mm_package/hardware/real/pneumatic_module.py @@ -75,8 +75,9 @@ def __init__( ( self.min_duty_cycle, self.max_duty_cycle, - self.min_step_size, + self.default_min_step_size, ) = self.get_config_params() + self.min_step_size = self.default_min_step_size self.duty_cycle = self.max_duty_cycle self.prev_duty_cycle = self.duty_cycle diff --git a/ulc_mm_package/hardware/scope_routines.py b/ulc_mm_package/hardware/scope_routines.py index ccd9b98de..bf366606d 100644 --- a/ulc_mm_package/hardware/scope_routines.py +++ b/ulc_mm_package/hardware/scope_routines.py @@ -243,6 +243,7 @@ def flow_control_routine( mscope.flow_controller.reset() flow_controller = mscope.flow_controller flow_controller.set_target_flowrate(target_flowrate) + flow_controller.set_alpha(processing_constants.FLOW_CONTROL_EWMA_ALPHA) if fast_flow: flow_controller.set_alpha( processing_constants.FLOW_CONTROL_EWMA_ALPHA * 2 @@ -261,8 +262,15 @@ def flow_control_routine( flow_val, flow_error, syringe_can_move = flow_controller.control_flow( img, timestamp ) + + # This check is here so that we don't flood the logger with the same message repeatedly if (prev_can_move is True) and (syringe_can_move is False): - # This is here so that we don't flood the logger with the same message + # If we were in fast_flow, we need to reset the min_step_size + if fast_flow: + flow_controller.pneumatic_module.min_step_size = ( + flow_controller.pneumatic_module.default_min_step_size + ) + self.logger.error( "Can't reach target flowrate. Syringe at end of travel." ) @@ -270,7 +278,9 @@ def flow_control_routine( if fast_flow: if flow_error is not None: if flow_error == 0: - flow_controller.pneumatic_module.min_step_size /= 2 + flow_controller.pneumatic_module.min_step_size = ( + flow_controller.pneumatic_module.default_min_step_size + ) return flow_val @init_generator