WIP: Add the ability to package more libraries

Author: romain-intel

metaflow/client/core.py

@@ -31,11 +31,12 @@
 from metaflow.includefile import IncludedFile
 from metaflow.metaflow_config import DEFAULT_METADATA, MAX_ATTEMPTS
 from metaflow.metaflow_environment import MetaflowEnvironment
+from metaflow.package.mfenv import MFEnv
 from metaflow.plugins import ENVIRONMENTS, METADATA_PROVIDERS
+from metaflow.special_files import SpecialFile
 from metaflow.unbounded_foreach import CONTROL_TASK_TAG
 from metaflow.util import cached_property, is_stringish, resolve_identity, to_unicode
 
-from ..info_file import INFO_FILE
 from .filecache import FileCache
 
 if TYPE_CHECKING:
@@ -823,10 +824,7 @@ def __init__(self, flow_name: str, code_package: str):
         )
         code_obj = BytesIO(blobdata)
         self._tar = tarfile.open(fileobj=code_obj, mode="r:gz")
-        # The JSON module in Python3 deals with Unicode. Tar gives bytes.
-        info_str = (
-            self._tar.extractfile(os.path.basename(INFO_FILE)).read().decode("utf-8")
-        )
+        info_str = MFEnv.get_archive_content(self._tar, SpecialFile.INFO_FILE)
         self._info = json.loads(info_str)
         self._flowspec = self._tar.extractfile(self._info["script"]).read()
 

metaflow/extension_support/__init__.py

@@ -12,7 +12,7 @@
 from itertools import chain
 from pathlib import Path
 
-from metaflow.info_file import read_info_file
+from metaflow.special_files import read_info_file
 
 
 #

metaflow/extension_support/_empty_file.py

@@ -1,2 +1,2 @@
-# This file serves as a __init__.py for metaflow_extensions when it is packaged
-# and needs to remain empty.
+# This file serves as a __init__.py for metaflow_extensions or metaflow
+# packages when they are packaged and needs to remain empty.

metaflow/info_file.py

@@ -445,7 +445,15 @@
 ###
 # Debug configuration
 ###
-DEBUG_OPTIONS = ["subcommand", "sidecar", "s3client", "tracing", "stubgen", "userconf"]
+DEBUG_OPTIONS = [
+    "subcommand",
+    "sidecar",
+    "s3client",
+    "tracing",
+    "stubgen",
+    "userconf",
+    "package",
+]
 
 for typ in DEBUG_OPTIONS:
     vars()["DEBUG_%s" % typ.upper()] = from_conf("DEBUG_%s" % typ.upper(), False)

metaflow/metaflow_config.py

@@ -11,7 +11,7 @@
 from os import path, name, environ, listdir
 
 from metaflow.extension_support import update_package_info
-from metaflow.info_file import CURRENT_DIRECTORY, read_info_file
+from metaflow.special_files import read_info_file
 
 
 # True/False correspond to the value `public`` in get_version