Merge branch 'master' into adjust_pr1152

Author: erezsh

lark/lark.py

@@ -19,8 +19,8 @@
     from .parser_frontends import ParsingFrontend
 
 from .exceptions import ConfigurationError, assert_config, UnexpectedInput
-from .utils import Serialize, SerializeMemoizer, FS, isascii, logger, SerializeMemoizer
-from .load_grammar import load_grammar, FromPackageLoader, Grammar, verify_used_files, PackageResource
+from .utils import Serialize, SerializeMemoizer, FS, isascii, logger
+from .load_grammar import load_grammar, FromPackageLoader, Grammar, verify_used_files, PackageResource, md5_digest
 from .tree import Tree
 from .common import LexerConf, ParserConf, _ParserArgType, _LexerArgType
 
@@ -303,7 +303,7 @@ def __init__(self, grammar: 'Union[Grammar, str, IO[str]]', **options) -> None:
                 options_str = ''.join(k+str(v) for k, v in options.items() if k not in unhashable)
                 from . import __version__
                 s = grammar + options_str + __version__ + str(sys.version_info[:2])
-                cache_md5 = hashlib.md5(s.encode('utf8')).hexdigest()
+                cache_md5 = md5_digest(s)
 
                 if isinstance(self.options.cache, str):
                     cache_fn = self.options.cache

lark/load_grammar.py

@@ -1312,7 +1312,7 @@ def do_import(self, dotted_path: Tuple[str, ...], base_path: Optional[str], alia
             except IOError:
                 continue
             else:
-                h = hashlib.md5(text.encode('utf8')).hexdigest()
+                h = md5_digest(text)
                 if self.used_files.get(joined_path, h) != h:
                     raise RuntimeError("Grammar file was changed during importing")
                 self.used_files[joined_path] = h
@@ -1391,7 +1391,7 @@ def verify_used_files(file_hashes):
         if text is None: # We don't know how to load the path. ignore it.
             continue
 
-        current = hashlib.md5(text.encode()).hexdigest()
+        current = md5_digest(text)
         if old != current:
             logger.info("File %r changed, rebuilding Parser" % path)
             return False
@@ -1407,3 +1407,15 @@ def load_grammar(grammar, source, import_paths, global_keep_all_tokens):
     builder = GrammarBuilder(global_keep_all_tokens, import_paths)
     builder.load_grammar(grammar, source)
     return builder.build(), builder.used_files
+
+
+def md5_digest(s: str) -> str:
+    """Get the md5 digest of a string
+
+    Supports the `usedforsecurity` argument for Python 3.9+ to allow running on
+    a FIPS-enabled system.
+    """
+    if sys.version_info >= (3, 9):
+        return hashlib.md5(s.encode('utf8'), usedforsecurity=False).hexdigest()
+    else:
+        return hashlib.md5(s.encode('utf8')).hexdigest()