Drop tqdm in favor of logger; normalize use of tabulate (#35)

* Drop tqdm in favor of logger; normalize use of tabulate

Changes are to accommodate #25, as webtools will need basic logging to handle websocket logging behavior required for front-end

* Missed reference

* Preserve logging bypass from existing branch

* Remove flask web package dependencies until PR#25

* Introduce decorate fn for redirecting logsteam to be later used in web app/tool

Refer to notes on other PR at 371a060

* Formatting fixes with pre-commit

Author: mcioffi

requirements-dev.txt

@@ -6,8 +6,7 @@ twine
 pre-commit
 pytest
 coverage
-tqdm
-Flask
+flask
 tabulate
 matplotlib
 -e .

train/__init__.py

@@ -1,12 +1,13 @@
 from .clean__check_label_consistency import check_label_consistency
 from .featuresearch import feature_search
 from .gridsearch import grid_search
-from .train_model import train_multiple, train_single
+from .train_model import set_redirect_log_stream, train_multiple, train_single
 
 __all__ = [
     "check_label_consistency",
     "feature_search",
     "grid_search",
+    "set_redirect_log_stream",
     "train_multiple",
     "train_single",
 ]

train/featuresearch.py

@@ -12,11 +12,11 @@
 import pycrfsuite
 from sklearn.model_selection import train_test_split
 from tabulate import tabulate
-from tqdm import tqdm
 
 from .train_model import DEFAULT_MODEL_LOCATION
 from .training_utils import (
     DataVectors,
+    convert_num_ordinal,
     evaluate,
     load_datasets,
 )
@@ -211,14 +211,17 @@ def feature_search(args: argparse.Namespace):
     logger.info(f"Grid search over {len(argument_sets)} feature sets.")
     logger.info(f"{args.seed} is the random seed used for the train/test split.")
 
+    eval_results = []
     with cf.ProcessPoolExecutor(max_workers=args.processes) as executor:
         futures = [
             executor.submit(train_model_feature_search, *a) for a in argument_sets
         ]
-        eval_results = [
-            future.result()
-            for future in tqdm(cf.as_completed(futures), total=len(futures))
-        ]
+        logger.info(
+            f"Queued for separate runs against {len(argument_sets)} feature sets"
+        )
+        for idx, future in enumerate(cf.as_completed(futures)):
+            logger.info(f"{convert_num_ordinal(idx + 1)} set completed")
+            eval_results.append(future.result())
 
     # Sort with highest sentence accuracy first
     eval_results = sorted(
@@ -248,4 +251,14 @@ def feature_search(args: argparse.Namespace):
             ]
         )
 
-    print(tabulate(table, headers=headers, tablefmt="simple_outline"))
+    print(
+        "\n"
+        + tabulate(
+            table,
+            headers=headers,
+            tablefmt="fancy_grid",
+            stralign="left",
+            numalign="right",
+        )
+        + "\n"
+    )

train/gridsearch.py

@@ -13,11 +13,11 @@
 import pycrfsuite
 from sklearn.model_selection import train_test_split
 from tabulate import tabulate
-from tqdm import tqdm
 
 from .train_model import DEFAULT_MODEL_LOCATION
 from .training_utils import (
     DataVectors,
+    convert_num_ordinal,
     evaluate,
     load_datasets,
 )
@@ -499,12 +499,13 @@ def grid_search(args: argparse.Namespace):
     logger.info(f"Grid search over {len(arguments)} hyperparameters combinations.")
     logger.info(f"{args.seed} is the random seed used for the train/test split.")
 
+    eval_results = []
     with cf.ProcessPoolExecutor(max_workers=args.processes) as executor:
         futures = [executor.submit(train_model_grid_search, *a) for a in arguments]
-        eval_results = [
-            future.result()
-            for future in tqdm(cf.as_completed(futures), total=len(futures))
-        ]
+        logger.info(f"Queued for separate runs against {len(args.algos)} algorithms")
+        for idx, future in enumerate(cf.as_completed(futures)):
+            logger.info(f"{convert_num_ordinal(idx + 1)} algorithm completed")
+            eval_results.append(future.result())
 
     # Sort with highest sentence accuracy first
     eval_results = sorted(
@@ -538,10 +539,14 @@ def grid_search(args: argparse.Namespace):
         )
 
     print(
-        tabulate(
+        "\n"
+        + tabulate(
             table,
             headers=headers,
             tablefmt="fancy_grid",
             maxcolwidths=[None, 130, None, None, None, None],
+            stralign="left",
+            numalign="right",
         )
+        + "\n"
     )

train/train_model.py

@@ -7,12 +7,12 @@
 from pathlib import Path
 from random import randint
 from statistics import mean, stdev
-from typing import Generator
+from typing import Generator, TextIO
 from uuid import uuid4
 
 import pycrfsuite
 from sklearn.model_selection import train_test_split
-from tqdm import tqdm
+from tabulate import tabulate
 
 from .test_results_to_detailed_results import test_results_to_detailed_results
 from .test_results_to_html import test_results_to_html
@@ -21,6 +21,7 @@
     DataVectors,
     Stats,
     confusion_matrix,
+    convert_num_ordinal,
     evaluate,
     load_datasets,
 )
@@ -29,7 +30,7 @@
 
 
 @contextmanager
-def change_log_level(level: int) -> Generator[None]:
+def change_log_level(level: int) -> Generator[None, None, None]:
     """Context manager to temporarily change logging level within the context.
 
     On exiting the context, the original level is restored.
@@ -50,6 +51,30 @@ def change_log_level(level: int) -> Generator[None]:
     logger.setLevel(original_level)
 
 
+@contextmanager
+def set_redirect_log_stream(io_stream: TextIO) -> Generator[None, None, None]:
+    """Context manager to accept io_stream for logging used in web app
+        Required by web app as it bypasses train.py where logging is configured
+
+    Parameters
+    ----------
+    io_stream : TextIO
+        io.IOString() stream
+
+    Yields
+    ------
+    Generator[None, None, None]
+        Generator, yielding None
+    """
+    logging.basicConfig(
+        stream=io_stream,
+        level=logging.INFO,
+        format="[%(levelname)s] (%(module)s) %(message)s",
+    )
+
+    yield
+
+
 def train_parser_model(
     vectors: DataVectors,
     split: float,
@@ -189,6 +214,15 @@ def train_parser_model(
     return stats
 
 
+def train_parser_model_bypass_logging(*kargs) -> Stats:
+    stats = None
+    with change_log_level(
+        logging.WARNING
+    ):  # Temporarily stop logging below WARNING for multi-processing
+        stats = train_parser_model(*kargs)
+    return stats
+
+
 def train_single(args: argparse.Namespace) -> None:
     """Train CRF model once.
 
@@ -222,15 +256,31 @@ def train_single(args: argparse.Namespace) -> None:
         combine_name_labels=args.combine_name_labels,
     )
 
-    print("Sentence-level results:")
-    print(f"\tAccuracy: {100 * stats.sentence.accuracy:.2f}%")
+    headers = ["Sentence-level results", "Word-level results"]
+    table = []
+
+    table.append(
+        [
+            f"Accuracy: {100 * stats.sentence.accuracy:.2f}%",
+            f"Accuracy: {100 * stats.token.accuracy:.2f}%\n"
+            f"Precision (micro) {100 * stats.token.weighted_avg.precision:.2f}%\n"
+            f"Recall (micro) {100 * stats.token.weighted_avg.recall:.2f}%\n"
+            f"F1 score (micro) {100 * stats.token.weighted_avg.f1_score:.2f}%",
+        ]
+    )
 
-    print()
-    print("Word-level results:")
-    print(f"\tAccuracy {100 * stats.token.accuracy:.2f}%")
-    print(f"\tPrecision (micro) {100 * stats.token.weighted_avg.precision:.2f}%")
-    print(f"\tRecall (micro) {100 * stats.token.weighted_avg.recall:.2f}%")
-    print(f"\tF1 score (micro) {100 * stats.token.weighted_avg.f1_score:.2f}%")
+    print(
+        "\n"
+        + tabulate(
+            table,
+            headers=headers,
+            tablefmt="fancy_grid",
+            maxcolwidths=[None, None],
+            stralign="left",
+            numalign="right",
+        )
+        + "\n"
+    )
 
 
 def train_multiple(args: argparse.Namespace) -> None:
@@ -272,13 +322,15 @@ def train_multiple(args: argparse.Namespace) -> None:
         for _ in range(args.runs)
     ]
 
-    with change_log_level(logging.WARNING):  # Temporarily stop logging below WARNING
-        with cf.ProcessPoolExecutor(max_workers=args.processes) as executor:
-            futures = [executor.submit(train_parser_model, *a) for a in arguments]
-            eval_results = [
-                future.result()
-                for future in tqdm(cf.as_completed(futures), total=len(futures))
-            ]
+    word_accuracies, sentence_accuracies, seeds, eval_results = [], [], [], []
+    with cf.ProcessPoolExecutor(max_workers=args.processes) as executor:
+        futures = [
+            executor.submit(train_parser_model_bypass_logging, *a) for a in arguments
+        ]
+        logger.info(f"Queued for {args.runs} separate runs")
+        for idx, future in enumerate(cf.as_completed(futures)):
+            logger.info(f"{convert_num_ordinal(idx + 1)} run completed")
+            eval_results.append(future.result())
 
     word_accuracies, sentence_accuracies, seeds = [], [], []
     for result in eval_results:
@@ -288,15 +340,9 @@ def train_multiple(args: argparse.Namespace) -> None:
 
     sentence_mean = 100 * mean(sentence_accuracies)
     sentence_uncertainty = 3 * 100 * stdev(sentence_accuracies)
-    print()
-    print("Average sentence-level accuracy:")
-    print(f"\t-> {sentence_mean:.2f}% ± {sentence_uncertainty:.2f}%")
 
     word_mean = 100 * mean(word_accuracies)
     word_uncertainty = 3 * 100 * stdev(word_accuracies)
-    print()
-    print("Average word-level accuracy:")
-    print(f"\t-> {word_mean:.2f}% ± {word_uncertainty:.2f}%")
 
     index_best = max(
         range(len(sentence_accuracies)), key=sentence_accuracies.__getitem__
@@ -310,6 +356,42 @@ def train_multiple(args: argparse.Namespace) -> None:
     min_sent = 100 * sentence_accuracies[index_worst]
     min_word = 100 * word_accuracies[index_worst]
     min_seed = seeds[index_worst]
-    print()
-    print(f"Best:  Sentence {max_sent:.2f}% / Word {max_word:.2f}% (Seed: {max_seed})")
-    print(f"Worst: Sentence {min_sent:.2f}% / Word {min_word:.2f}% (Seed: {min_seed})")
+
+    headers = ["Run", "Word/Token accuracy", "Sentence accuracy", "Seed"]
+
+    table = []
+    for idx, result in enumerate(eval_results):
+        table.append(
+            [
+                convert_num_ordinal(idx + 1),
+                f"{100 * result.token.accuracy:.2f}%",
+                f"{100 * result.sentence.accuracy:.2f}%",
+                f"{result.seed}",
+            ]
+        )
+
+    table.append(["-"] * len(headers))
+    table.append(
+        [
+            "Average",
+            f"{word_mean:.2f}% ± {word_uncertainty:.2f}%",
+            f"{sentence_mean:.2f}% ± {sentence_uncertainty:.2f}%",
+            f"{max_seed}",
+        ]
+    )
+    table.append(["-"] * len(headers))
+    table.append(["Best", f"{max_word:.2f}%", f"{max_sent:.2f}%", f"{max_seed}"])
+    table.append(["Worst", f"{min_word:.2f}%", f"{min_sent:.2f}%", f"{min_seed}"])
+
+    print(
+        "\n"
+        + tabulate(
+            table,
+            headers=headers,
+            tablefmt="fancy_grid",
+            maxcolwidths=[None, None, None, None],
+            stralign="left",
+            numalign="right",
+        )
+        + "\n"
+    )

train/training_utils.py

@@ -7,7 +7,7 @@
 from dataclasses import dataclass
 from functools import partial
 from itertools import chain, islice
-from typing import Any, Callable, Iterable
+from typing import Any, Callable, Iterable, Union
 
 from matplotlib import pyplot as plt
 from sklearn.metrics import (
@@ -415,3 +415,22 @@ def confusion_matrix(
     fig.savefig(figure_path)
     logger.info(f"Confusion matrix saved to {figure_path}.")
     plt.close(fig)
+
+
+def convert_num_ordinal(num: Union[int, float, str]) -> str:
+    """Converts a number (int) into its ordinal; falls back to input if unsuccessful
+
+    make_ordinal(0)   => '0th'
+    make_ordinal(3)   => '3rd'
+    make_ordinal(122) => '122nd'
+    make_ordinal(213) => '213th'
+    """
+    try:
+        n = int(num)
+        if 11 <= (n % 100) <= 13:
+            suffix = "th"
+        else:
+            suffix = ["th", "st", "nd", "rd", "th"][min(n % 10, 4)]
+        return str(n) + suffix
+    except TypeError or ValueError:
+        return str(num)