adding support for serial and fixing bug that driver might not be defined (#82)

* adding support for serial and fixing bug that driver might not be defined

Signed-off-by: vsoch <vsochat@stanford.edu>

Author: vsoch

CHANGELOG.md

@@ -12,6 +12,7 @@ and **Merged pull requests**. Critical items to know are:
 Referenced versions in headers are tagged on Github, in parentheses are for pypi.
 
 ## [vxx](https://github.com/urlstechie/urlschecker-python/tree/master) (master)
+ - serial option for debugging (0.0.32)
  - adding support for web driver for harder URLs (0.0.31)
  - use ANSI escape sequences for colors, fake-useragent for agents (0.0.30)
  - adding type hints to code, more tests and logging bug fix (0.0.29)

README.md

@@ -247,7 +247,12 @@ urlchecker check --file-types ".*,*.html" .
 
 **Note that while some patterns will work without quotes, it's recommended for most**
 to use them because if the shell expands any part of the pattern, it will not work as
-expected. By default, the urlchecker checks python and markdown.
+expected. By default, the urlchecker checks python and markdown. If a multiprocessing workers has an error,
+you can also add `--serial` to run in serial and test. The run will be slower, but it's useful for debugging.
+
+```bash
+$ urlchecker check . --files "content/docs/hacking/contributing/documentation/index.md" --serial
+```
 
 ### Check GitHub Repository
 

urlchecker/client/__init__.py

@@ -70,6 +70,12 @@ def get_parser():
         action="store_true",
     )
 
+    check.add_argument(
+        "--serial",
+        help="run checks in serial (no multiprocess)",
+        default=False,
+        action="store_true",
+    )
     check.add_argument(
         "--force-pass",
         help="force successful pass (return code 0) regardless of result",

urlchecker/client/check.py

@@ -65,6 +65,7 @@ def main(args, extra):
     print("               subfolder: %s" % args.subfolder)
     print("                  branch: %s" % args.branch)
     print("                 cleanup: %s" % args.cleanup)
+    print("                  serial: %s" % args.serial)
     print("              file types: %s" % file_types)
     print("                   files: %s" % files)
     print("               print all: %s" % (not args.no_print))
@@ -84,6 +85,7 @@ def main(args, extra):
         include_patterns=files,
         exclude_files=exclude_files,
         print_all=not args.no_print,
+        serial=args.serial,
     )
     check_results = checker.run(
         exclude_urls=exclude_urls,

urlchecker/core/check.py

@@ -20,7 +20,8 @@
 
 
 class UrlChecker:
-    """The UrlChecker can be instantiated by a client, and then used
+    """
+    The UrlChecker can be instantiated by a client, and then used
     to parse files, extract urls, and save results.
     """
 
@@ -31,6 +32,7 @@ def __init__(
         exclude_files: List[str] = None,
         print_all: bool = True,
         include_patterns: List[str] = None,
+        serial: bool = False,
     ):
         """
         initiate a url checker. At init we take in preferences for
@@ -43,6 +45,7 @@ def __init__(
             - print_all        (bool) : control var for whether to print all checked file names or only the ones with urls.
             - exclude_files    (list) : list of excluded files and patterns for flies.
             - include_patterns (list) : list of files and patterns to check.
+            - serial           (bool) : do checks in serial (no multiprocessing)
         """
         # Initiate results object, and checks lookup (holds UrlCheck) for each file
         self.results = {
@@ -61,6 +64,7 @@ def __init__(
         self.path = path
         self.file_types = file_types or [".py", ".md"]
         self.file_paths = []
+        self.serial = serial
 
         # get all file paths if a path is defined
         if path:
@@ -195,10 +199,11 @@ def run(
         random.shuffle(ports)
 
         # loop through files
+        results = {}
         for file_name in file_paths:
 
             # Export parameters and functions, use the same check task for all
-            tasks[file_name] = {
+            kwargs = {
                 "file_name": file_name,
                 "exclude_patterns": exclude_patterns,
                 "exclude_urls": exclude_urls,
@@ -207,9 +212,16 @@ def run(
                 "timeout": timeout,
                 "port": ports.pop(0),
             }
+
+            if self.serial:
+                results[file_name] = check_task(**kwargs)
+                continue
+
+            tasks[file_name] = kwargs
             funcs[file_name] = check_task
 
-        results = workers.run(funcs, tasks)
+        if not self.serial:
+            results = workers.run(funcs, tasks)  # type: ignore
         if not results:
             print("\U0001F914 There were no URLs to check.")
             sys.exit(0)

urlchecker/core/urlproc.py

@@ -242,7 +242,7 @@ def check_urls(
 
                 # Web driver doesn't have same issues with ssl
                 except Exception as e:
-                    if driver.check(url):
+                    if driver and driver.check(url):
                         response = requests.Response()
                         response.status_code = 200
                     else:

urlchecker/version.py

@@ -7,7 +7,7 @@
 
 """
 
-__version__ = "0.0.31"
+__version__ = "0.0.32"
 AUTHOR = "Ayoub Malek, Vanessa Sochat"
 AUTHOR_EMAIL = "superkogito@gmail.com, vsochat@stanford.edu"
 NAME = "urlchecker"