[PREVIEW-WIP] PERF: Improve startup time by 8% with lazy loading of wrapped libraries

Base/Python/CMakeLists.txt

@@ -7,6 +7,7 @@ set(Slicer_PYTHON_SCRIPTS
   slicer/testing
   slicer/util
   freesurfer
+  lazy
   mrml
   saferef
   teem

Base/Python/lazy.py

@@ -0,0 +1,107 @@
+import imp
+import json
+import os
+import sys
+import types
+
+not_loaded = 'not loaded'
+
+def library_loader(module_name):
+  #print("Loading %s" % module_name)
+  fp, pathname, description = imp.find_module(module_name)
+  module = imp.load_module(module_name, fp, pathname, description)
+  return module
+
+class LazyModule(types.ModuleType):
+  """Subclass of ModuleType that implements a custom __getattribute__ method
+  to allow lazy-loading of attributes from slicer sub-modules."""
+
+  def __init__(self, name):
+    types.ModuleType.__init__(self, name)
+    self.__lazy_attributes = {}
+    #print("__lazy_attributes: %s" % len(self.__lazy_attributes))
+
+  def _update_lazy_attributes(self, lazy_attributes):
+    self.__lazy_attributes.update(lazy_attributes)
+    for k in lazy_attributes:
+      setattr(self, k, not_loaded)
+
+  def __getattribute__(self, attr):
+    value = types.ModuleType.__getattribute__(self, attr)
+    #print("__getattribute__ %s" % (attr))
+    if value is not_loaded:
+      module_name = self.__lazy_attributes[attr]
+
+      module = library_loader(module_name)
+      namespace = module.__dict__
+
+      # Load into 'namespace' first, then self.__dict__ (via setattr) to
+      # prevent the warnings about overwriting the 'NotLoaded' values
+      # already in self.__dict__ we would get if we just update
+      # self.__dict__.
+      for k, v in namespace.items():
+        if not k.startswith('_'):
+          setattr(self, k, v)
+      value = namespace[attr]
+    return value
+
+def writeModuleAttributeFile(module_name, config_dir='.'):
+  try:
+    exec("import %s as module" % module_name)
+  except ImportError as details:
+    print("%s [skipped: failed to import: %s]" % (module_name, details))
+    return
+  attributes = []
+  for attr in dir(module):
+    if not attr.startswith('__'):
+      attributes.append(attr)
+  filename = os.path.join(config_dir, "%s.json" % module_name)
+  with open(filename, 'w') as output:
+    print("%s [done: %s]" % (module_name, filename))
+    output.write(json.dumps({"attributes":attributes}, indent=4))
+
+def updateLazyModule(module, input_module_names=[], config_dir=None):
+  if isinstance(module, basestring):
+    if module not in sys.modules:
+      print("updateLazyModule failed: Couldn't find %s module" % module)
+      return
+    module = sys.modules[module]
+  if not isinstance(module, LazyModule):
+    print("updateLazyModule failed: module '%s' is not a LazyModule" % module)
+    return
+  if isinstance(input_module_names, basestring):
+    input_module_names = [input_module_names]
+  if config_dir is None:
+    config_dir = os.path.dirname(module.__path__[0])
+  for input_module_name in input_module_names:
+    filename = os.path.join(config_dir, "%s.json" % input_module_name)
+    with open(filename) as input:
+      module_attributes = json.load(input)['attributes']
+      #print("Updating %s with %d attributes" % (filename, len(module_attributes)))
+      module._update_lazy_attributes({attribute: input_module_name for attribute in module_attributes})
+
+    #print("Updated %s module with %d attributes from %s" % (module, len(module._LazyModule__lazy_attributes), input_module_name))
+
+def createLazyModule(module_name, module_path, input_module_names=[], config_dir=None):
+
+  thisModule = sys.modules[module_name] if module_name in sys.modules else None
+
+  if isinstance(thisModule, LazyModule):
+    # Handle reload case where we've already done this once.
+    # If we made a new module every time, multiple reload()s would fail
+    # because the identity of sys.modules['itk'] would always be changing.
+    #print("slicer: Calling ctor of LazyModule")
+    thisModule.__init__(module_name)
+  else:
+    print("slicer: Creating new LazyModule")
+    thisModule = LazyModule(module_name)
+
+  # Set the __path__ attribute, which is required for this module to be used as a
+  # package
+  setattr(thisModule, '__path__', module_path)
+
+  sys.modules[module_name] = thisModule
+
+  updateLazyModule(thisModule, input_module_names, config_dir)
+
+  return thisModule

Base/Python/slicer/__init__.py

@@ -1,5 +1,9 @@
 """ This module sets up root logging and loads the Slicer library modules into its namespace."""
 
+import lazy
+thisModule = lazy.createLazyModule(__name__, __path__)
+del lazy
+
 #-----------------------------------------------------------------------------
 def _createModule(name, globals, docstring):
   import imp
@@ -14,14 +18,14 @@ def _createModule(name, globals, docstring):
 #-----------------------------------------------------------------------------
 # Create slicer.modules and slicer.moduleNames
 
-_createModule('slicer.modules', globals(),
+_createModule('slicer.modules', vars(thisModule),
 """This module provides an access to all instantiated Slicer modules.
 
 The module attributes are the lower-cased Slicer module names, the
 associated value is an instance of ``qSlicerAbstractCoreModule``.
 """)
 
-_createModule('slicer.moduleNames', globals(),
+_createModule('slicer.moduleNames', vars(thisModule),
 """This module provides an access to all instantiated Slicer module names.
 
 The module attributes are the Slicer modules names, the associated
@@ -36,15 +40,19 @@ def _createModule(name, globals, docstring):
 except ImportError:
   available_kits = []
 
+from .util import importModuleObjects
+
 for kit in available_kits:
    try:
-     exec "from %s import *" % (kit)
+     importModuleObjects(kit, thisModule)
+     #exec "from %s import *" % (kit)
    except ImportError as detail:
      print detail
 
 #-----------------------------------------------------------------------------
 # Cleanup: Removing things the user shouldn't have to see.
 
+del thisModule
 del _createModule
 del available_kits
 del kit

Base/Python/slicer/util.py

@@ -53,11 +53,11 @@ def sourceDir():
 # Custom Import
 #
 
-def importVTKClassesFromDirectory(directory, dest_module_name, filematch = '*'):
-  importClassesFromDirectory(directory, dest_module_name, 'vtkclass', filematch)
+def importVTKClassesFromDirectory(directory, dest_module_name, filematch = '*', lazy=False):
+  importClassesFromDirectory(directory, dest_module_name, 'vtkclass', filematch, lazy)
 
-def importQtClassesFromDirectory(directory, dest_module_name, filematch = '*'):
-  importClassesFromDirectory(directory, dest_module_name, 'PythonQtClassWrapper', filematch)
+def importQtClassesFromDirectory(directory, dest_module_name, filematch = '*', lazy=False):
+  importClassesFromDirectory(directory, dest_module_name, 'PythonQtClassWrapper', filematch, lazy)
 
 # To avoid globbing multiple times the same directory, successful
 # call to ``importClassesFromDirectory()`` will be indicated by
@@ -66,35 +66,39 @@ def importQtClassesFromDirectory(directory, dest_module_name, filematch = '*'):
 # Each entry is a tuple of form (directory, dest_module_name, type_name, filematch)
 __import_classes_cache = set()
 
-def importClassesFromDirectory(directory, dest_module_name, type_name, filematch = '*'):
+def importClassesFromDirectory(directory, dest_module_name, type_name, filematch = '*', lazy=False):
 
   # Create entry for __import_classes_cache
   cache_key = ",".join([directory, dest_module_name, type_name, filematch])
   # Check if function has already been called with this set of parameters
   if cache_key in __import_classes_cache:
     return
 
-  import glob, os, re, fnmatch
+  import glob, lazy, os, re, fnmatch
   re_filematch = re.compile(fnmatch.translate(filematch))
   for fname in glob.glob(os.path.join(directory, filematch)):
     if not re_filematch.match(os.path.basename(fname)):
       continue
-    try:
-      from_module_name = os.path.splitext(os.path.basename(fname))[0]
-      importModuleObjects(from_module_name, dest_module_name, type_name)
-    except ImportError as detail:
-      import sys
-      print(detail, file=sys.stderr)
+    from_module_name = os.path.splitext(os.path.basename(fname))[0]
+    if lazy:
+      lazy.updateLazyModule(dest_module_name, from_module_name, os.path.dirname(fname))
+    else:
+      try:
+        importModuleObjects(from_module_name, dest_module_name, type_name)
+      except ImportError as detail:
+        import sys
+        print(detail, file=sys.stderr)
 
   __import_classes_cache.add(cache_key)
 
-def importModuleObjects(from_module_name, dest_module_name, type_name):
+def importModuleObjects(from_module_name, dest_module, type_name='*'):
   """Import object of type 'type_name' from module identified
-  by 'from_module_name' into the module identified by 'dest_module_name'."""
+  by 'from_module_name' into the module identified by 'dest_module'."""
 
-  # Obtain a reference to the module identifed by 'dest_module_name'
+  # Obtain a reference to the module identifed by 'dest_module'
   import sys
-  dest_module = sys.modules[dest_module_name]
+  if isinstance(dest_module, basestring):
+    dest_module = sys.modules[dest_module]
 
   # Skip if module has already been loaded
   if from_module_name in sys.modules:
@@ -112,7 +116,7 @@ def importModuleObjects(from_module_name, dest_module_name, type_name):
     item = getattr(module, item_name)
 
     # Add the object to dest_module_globals_dict if any
-    if type(item).__name__ == type_name:
+    if type(item).__name__ == type_name or (type_name == '*' and not item_name.startswith('_')):
       setattr(dest_module, item_name, item)
 
 #

Base/QTGUI/qSlicerLoadableModule.cxx

@@ -74,13 +74,13 @@ bool qSlicerLoadableModule::importModulePythonExtensions(
   ctkScopedCurrentDir scopedCurrentDir(QFileInfo(modulePath).absolutePath());
   pythonManager->executeString(QString(
         "from slicer.util import importVTKClassesFromDirectory;"
-        "importVTKClassesFromDirectory('%1', 'slicer', filematch='vtkSlicer*ModuleLogicPython.*');"
-        "importVTKClassesFromDirectory('%1', 'slicer', filematch='vtkSlicer*ModuleMRMLPython.*');"
-        "importVTKClassesFromDirectory('%1', 'slicer', filematch='vtkSlicer*ModuleMRMLDisplayableManagerPython.*');"
+        "importVTKClassesFromDirectory('%1', 'slicer', filematch='vtkSlicer*ModuleLogicPython.*', lazy=True);"
+        "importVTKClassesFromDirectory('%1', 'slicer', filematch='vtkSlicer*ModuleMRMLPython.*', lazy=True);"
+        "importVTKClassesFromDirectory('%1', 'slicer', filematch='vtkSlicer*ModuleMRMLDisplayableManagerPython.*', lazy=True);"
         ).arg(scopedCurrentDir.currentPath()));
   pythonManager->executeString(QString(
         "from slicer.util import importQtClassesFromDirectory;"
-        "importQtClassesFromDirectory('%1', 'slicer', filematch='qSlicer*PythonQt.*');"
+        "importQtClassesFromDirectory('%1', 'slicer', filematch='qSlicer*PythonQt.*', lazy=True);"
         ).arg(scopedCurrentDir.currentPath()));
   return !pythonManager->pythonErrorOccured();
 #else

CMake/SlicerMacroBuildModuleQtLibrary.cmake

@@ -240,6 +240,22 @@ macro(SlicerMacroBuildModuleQtLibrary)
     if(NOT "${MODULEQTLIBRARY_FOLDER}" STREQUAL "")
       set_target_properties(${lib_name}PythonQt PROPERTIES FOLDER ${MODULEQTLIBRARY_FOLDER})
     endif()
+
+    # XXX Check if Slicer_LAUNCHER_EXECUTABLE available at during a clean build
+    # XXX Install .json file. Should be taking care of by ctkMacroCompilePythonScript
+
+    # Add target to generate module attributes file to allow lazy loading
+    set(module_name "${lib_name}PythonQt")
+    set(config_dir "${CMAKE_BINARY_DIR}/${Slicer_QTLOADABLEMODULES_LIB_DIR}/")
+    set(code "import sys; sys.path.append('${Slicer_SOURCE_DIR}/Base/Python/');")
+    set(code "${code}import lazy;")
+    set(code "${code}lazy.writeModuleAttributeFile('${module_name}', config_dir='${config_dir}')")
+    add_custom_command(TARGET ${module_name} POST_BUILD
+      COMMAND ${Slicer_LAUNCHER_EXECUTABLE} --no-splash -c "${code}"
+      COMMENT "Generating ${module_name}.json"
+      VERBATIM
+      )
+
   endif()
 
 endmacro()

CMake/SlicerMacroPythonWrapModuleVTKLibrary.cmake

@@ -81,4 +81,23 @@ macro(SlicerMacroPythonWrapModuleVTKLibrary)
     KIT_PYTHON_LIBRARIES ${PYTHONWRAPMODULEVTKLIBRARY_Wrapped_LIBRARIES}
     )
 
+  # XXX Check if Slicer_LAUNCHER_EXECUTABLE available at during a clean build
+  # XXX Install .json file. Should be taking care of by ctkMacroCompilePythonScript
+
+  # Get path to real executable
+  get_filename_component(python_bin_dir ${PYTHON_EXECUTABLE} PATH)
+  set(real_python_executable ${python_bin_dir}/python${CMAKE_EXECUTABLE_SUFFIX})
+
+  # Add target to generate module attributes file to allow lazy loading
+  set(module_name "${PYTHONWRAPMODULEVTKLIBRARY_NAME}Python")
+  set(config_dir "${CMAKE_BINARY_DIR}/${Slicer_QTLOADABLEMODULES_LIB_DIR}/")
+  set(code "import sys; sys.path.append('${Slicer_SOURCE_DIR}/Base/Python/');")
+  set(code "${code}import lazy;")
+  set(code "${code}lazy.writeModuleAttributeFile('${module_name}', config_dir='${config_dir}')")
+  add_custom_command(TARGET ${module_name} POST_BUILD
+    COMMAND ${Slicer_LAUNCHER_EXECUTABLE} --launch ${real_python_executable} -c "${code}"
+    COMMENT "Generating ${module_name}.json"
+    VERBATIM
+    )
+
 endmacro()