Add a series of scripts for fetching and parsing OpenSSL.org manpages

Author: ngie-eign

programming/python/openssl_manpage_scraper/fetch_openssl_manpages.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+bindir=$(dirname "$0")
+versions="1.0.2 1.1.1 3.0 3.1"
+for ver in $versions; do
+	d="$ver"
+	mkdir -p "$d"
+	"$bindir/openssl_manpage_libcall_scrape.py" https://www.openssl.org/docs/man${ver}/man3/ | \
+	    xargs -n 10 -P 5 "$bindir/../concurrent_fetch.py" --output-dir="$d" --verbose --retries=3
+done

programming/python/openssl_manpage_scraper/openssl_manpage_libcall_scrape.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+"""Scrape library manpage listing from OpenSSL.org.
+"""
+
+import argparse
+import urllib.parse
+from typing import Optional
+
+import bs4
+import requests
+
+
+BS4_PARSER = "html5lib"
+
+
+def parse_args(argv : Optional[str] = None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("url")
+
+    return parser.parse_args(args=argv)
+
+
+def get_manpage_html_text(url: str) -> str:
+    resp = requests.get(url)
+    resp.raise_for_status()
+    return resp.text
+
+
+def extract_libcall_links_from_page(text: str) -> str:
+    soup = bs4.BeautifulSoup(text, BS4_PARSER)
+    return [
+        link["href"] for link in soup.body.article.table.tbody.find_all("a", href=True)
+    ]
+
+
+def main(argv : Optional[str] = None) -> None:
+    args = parse_args(argv)
+    text = get_manpage_html_text(args.url)
+    parsed_links = extract_libcall_links_from_page(text)
+    for link in parsed_links:
+        print(urllib.parse.urljoin(args.url, link))
+
+
+if __name__ == "__main__":
+    main()

programming/python/openssl_manpage_scraper/parse_synopsis_from_openssl_html_manpage.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+"""Scrape library manpage output from OpenSSL.org.
+"""
+
+import argparse
+import html
+import logging
+import os
+import pathlib
+import re
+import subprocess
+from typing import Optional
+
+import bs4
+
+
+INCLUDE_RE = re.compile(r"#include <openssl/")
+FLATTEN_DECLARATIONS_RE = re.compile(",\n", re.M)
+BS4_PARSER = "html5lib"
+
+
+def parse_args(argv : Optional[str] = None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("url")
+    parser.add_argument("output", type=argparse.FileType("w"))
+
+    return parser.parse_args(args=argv)
+
+
+def fetch_manpage_html_text(url: str) -> str:
+    import requests
+    resp = requests.get(url)
+    resp.raise_for_status()
+    return resp.text
+
+
+def extract_synopsis_from_manpage_html(text: str) -> str:
+    soup = bs4.BeautifulSoup(text, BS4_PARSER)
+    for elem in soup.body.find_all("pre"):
+        code_elems = elem.find_all("code")
+        if not code_elems or len(code_elems) != 1:
+            continue
+        synopsis_text = html.unescape(code_elems[0].text.lstrip())
+        if INCLUDE_RE.match(synopsis_text) is None:
+            synopsis_text = INCLUDE_RE.sub("", synopsis_text)
+        return FLATTEN_DECLARATIONS_RE.sub(", ", synopsis_text)
+
+
+BINDIR = os.path.dirname(__file__)
+
+def main(argv : Optional[str] = None) -> None:
+    args = parse_args(argv)
+
+    logging.basicConfig()
+    logging.getLogger().setLevel(logging.DEBUG)
+    logging.debug("Will parse synopsis from %s", args.url)
+
+    url_path = pathlib.Path(args.url)
+    try:
+        text = (
+            url_path.read_text() if url_path.exists()
+            else fetch_manpage_html_text(args.url)
+        )
+        synopsis = extract_synopsis_from_manpage_html(text)
+        if not synopsis:
+            raise AssertionError(f"{args.url} has no synopsis")
+        with args.output as output_fp:
+            output_fp.write(synopsis)
+        subprocess.check_call(["clang-format", "-i", f"--style=file:{BINDIR}/.clang_format", output_fp.name])
+    except AssertionError as exc:
+        print(exc)
+        os.unlink(args.output.name)
+    except Exception:
+        logging.exception("")
+        os.unlink(args.output.name)
+
+    logging.debug("Parsed synopsis from %s into %s", args.url, args.output.name)
+
+
+if __name__ == "__main__":
+    main()

programming/python/openssl_manpage_scraper/parse_synopsis_from_openssl_manpages.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+set -eu
+
+max_workers=10
+dir_path=$(realpath "$1")
+set -- $(find "$dir_path" -type f -name \*.html)
+while [ $# -gt 0 ]; do
+	allocated_workers=0
+	set +u
+	for i in $(seq 1 $max_workers); do
+		html_file=$1; shift
+		if [ $# -gt 0 ]; then
+			./parse_synopsis_from_openssl_html_manpage.py $html_file ${html_file%.html*}.txt &
+			: $(( allocated_workers += 1 ))
+		fi
+	done
+	for i in $(seq 1 $allocated_workers); do
+		wait
+	done
+done