Merge pull request #4 from delcypher/master

Various Improvements

Author: travitch

README.md

@@ -10,9 +10,12 @@ bitcode file in an ELF section of the actual object file.
 
 When object files are linked together, the contents of non-special ELF
 sections are just concatenated (so we don't lose the locations of any
-of the constituent bitcode files).  This package contains an extra
-utility, extract-bc, to read the contents of this ELF section and link
-all of the bitcode into a single whole-program bitcode file.
+of the constituent bitcode files).
+
+This package contains an extra utility, extract-bc, to read the
+contents of this ELF section and link all of the bitcode into a single
+whole-program bitcode file. This utility can also be used on built
+native static libraries to generate LLVM bitcode archives.
 
 This two-phase build process is slower and more elaborate than normal,
 but in practice is necessary to be a drop-in replacement for gcc in
@@ -41,8 +44,18 @@ wrapper script:
 Once the environment is set up, just use wllvm and wllvm++ as your C
 and C++ compilers, respectively.
 
-Example
-=======
+In addition to the above environment variables the following can be optionally used:
+
+ * `LLVM_COMPILER_PATH` can be set to the absolute path to the folder that
+   contains the compiler and other LLVM tools such as `llvm-link` to be used.
+   This prevents searching for the compiler in your PATH environment variable.
+   This can be useful if you have different versions of clang on your system
+   and you want to easily switch compilers without tinkering with your PATH
+   variable.
+   Example `LLVM_COMPILER_PATH=/home/user/llvm_and_clang/Debug+Asserts/bin`.
+
+Example building bitcode module
+===============================
 
     export LLVM_COMPILER=dragonegg
     export LLVM_GCC_PREFIX=llvm-
@@ -56,3 +69,32 @@ Example
     # Produces pkg-config.bc
     extract-bc pkg-config
 
+Example building bitcode archive
+================================
+
+    export LLVM_COMPILER=clang
+    tar -xvf bullet-2.81-rev2613.tgz
+    mkdir bullet-bin
+    cd bullet-bin
+    CC=wllvm CXX=wllvm++ cmake ../bullet-2.81-rev2613/
+    make
+
+    # Produces src/LinearMath/libLinearMath.bca
+    extract-bc src/LinearMath/libLinearMath.a
+
+Debugging
+=========
+
+The WLLVM tools can show various levels of output to aid with debugging.
+To show this output set WLLVM_OUTPUT to one of the following levels:
+
+ * `CRITICAL`
+ * `ERROR`
+ * `WARNING`
+ * `INFO`
+ * `DEBUG`
+
+For example
+
+    export WLLVM_OUTPUT=DEBUG
+

driver/as

@@ -11,8 +11,10 @@
 # compiler).  We'll link this together at a later stage.
 
 import os
-import subprocess, sys
+import sys
 from utils import *
+from subprocess import *
+from popenwrapper import Popen
 
 
 class BCFilter(ArgumentListFilter):

driver/logconfig.py

@@ -0,0 +1,27 @@
+"""
+  This module is intended to be imported by command line tools so they can
+  configure the root logger so that other loggers used in other modules can
+  inherit the configuration.
+"""
+import logging
+import os
+import sys
+
+_loggingEnv = 'WLLVM_OUTPUT'
+_validLogLevels = ['CRITICAL','ERROR', 'WARNING', 'INFO', 'DEBUG']
+logging.basicConfig(level=logging.WARNING, format='%(levelname)s:%(message)s')
+if os.getenv(_loggingEnv):
+    level = os.getenv(_loggingEnv).upper()
+    if not level in _validLogLevels:
+        logging.error('"{0}" is not a valid value for {1} . Valid values are {2}'.format(
+                      level, _loggingEnv, _validLogLevels))
+        sys.exit(1)
+    else:
+        logging.getLogger().setLevel(getattr(logging, level))
+
+# Adjust the format if debugging
+if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
+  formatter = logging.Formatter('%(levelname)s::%(module)s.%(funcName)s() at %(filename)s:%(lineno)d ::%(message)s')
+  for h in logging.getLogger().handlers:
+    h.setFormatter(formatter)
+

driver/popenwrapper.py

@@ -0,0 +1,18 @@
+import os
+import subprocess
+import pprint
+import logging
+
+# This module provides a wrapper for subprocess.POpen
+# that can be used for debugging
+
+# Internal logger
+_logger = logging.getLogger(__name__)
+
+def Popen(*pargs, **kwargs):
+    _logger.debug("WLLVM Executing:\n" + pprint.pformat(pargs[0]))
+    try:
+        return subprocess.Popen(*pargs,**kwargs)
+    except OSError:
+        _logger.error("WWLVM Failed to execute:\n" + pprint.pformat(pargs[0]))
+        raise

driver/utils.py

@@ -1,16 +1,27 @@
 from subprocess import *
 import collections
+import pprint
+import logging
 import errno
 import os
 import re
 import sys
 import tempfile
-import subprocess
+from driver.popenwrapper import Popen
 
 fullSelfPath = os.path.realpath(__file__)
 prefix = os.path.dirname(fullSelfPath)
 driverDir = prefix
 
+# Environmental variable for path to compiler tools (clang/llvm-link etc..)
+llvmCompilerPathEnv = 'LLVM_COMPILER_PATH'
+
+# This is the ELF section name inserted into binaries
+elfSectionName='.llvm_bc'
+
+# Internal logger
+_logger = logging.getLogger(__name__)
+
 # This class applies filters to GCC argument lists.  It has a few
 # default arguments that it records, but does not modify the argument
 # list at all.  It can be subclassed to change this behavior.
@@ -106,7 +117,7 @@ def __init__(self, inputList, exactMatches={}, patternMatches={}):
                 handler(self, currentItem, *flagArgs)
             else:
                 matched = False
-                for pattern, (arity, handler) in argPatterns.iteritems():
+                for pattern, (arity, handler) in argPatterns.items():
                     if re.match(pattern, currentItem):
                         flagArgs = self._shiftArgs(arity)
                         handler(self, currentItem, *flagArgs)
@@ -180,6 +191,39 @@ def __init__(self, arglist):
     def outputFileCallback(self, flag, filename):
         self.outputFilename = filename
 
+
+# Static class that allows the type of a file to be checked.
+class FileType(object):
+  # Provides int -> str map
+  revMap = { }
+
+  @classmethod
+  def getFileType(cls, fileName):
+      # This is a hacky way of determining
+      # the type of file we are looking at.
+      # Maybe we should use python-magic instead?
+
+      fileP = Popen(['file',fileName], stdout=PIPE)
+      output = fileP.communicate()[0]
+      output = output.decode()
+      if 'ELF' in output and 'executable' in output:
+          return cls.EXECUTABLE
+      elif 'current ar archive' in output:
+          return cls.ARCHIVE
+      elif 'ELF' in output and 'relocatable' in output:
+          return cls.OBJECT
+      else:
+          return cls.UNKNOWN
+
+  @classmethod
+  def init(cls):
+      for (index, name) in enumerate(('UNKNOWN', 'EXECUTABLE', 'OBJECT', 'ARCHIVE')):
+          setattr(cls, name, index)
+          cls.revMap[index] = name
+
+# Initialise FileType static class
+FileType.init()
+
 def attachBitcodePathToObject(bcPath, outFileName):
     # Don't try to attach a bitcode path to a binary.  Unfortunately
     # that won't work.
@@ -189,9 +233,11 @@ def attachBitcodePathToObject(bcPath, outFileName):
 
     # Now just build a temporary text file with the full path to the
     # bitcode file that we'll write into the object file.
-    f = tempfile.NamedTemporaryFile(mode='rw+b', delete=False)
-    f.write(os.path.abspath(bcPath))
-    f.write('\n')
+    f = tempfile.NamedTemporaryFile(mode='w+b', delete=False)
+    absBcPath = os.path.abspath(bcPath)
+    f.write(absBcPath.encode())
+    f.write('\n'.encode())
+    _logger.debug(pprint.pformat('Wrote "{0}" to file "{1}"'.format(absBcPath, f.name)))
 
     # Ensure buffers are flushed so that objcopy doesn't read an empty
     # file
@@ -200,12 +246,12 @@ def attachBitcodePathToObject(bcPath, outFileName):
     f.close()
 
     # Now write our .llvm_bc section
-    objcopyCmd = ['objcopy', '--add-section', '.llvm_bc={0}'.format(f.name), outFileName]
+    objcopyCmd = ['objcopy', '--add-section', '{0}={1}'.format(elfSectionName, f.name), outFileName]
     orc = 0
 
     try:
         if os.path.getsize(outFileName) > 0:
-            objProc = subprocess.Popen(objcopyCmd)
+            objProc = Popen(objcopyCmd)
             orc = objProc.wait()
     except OSError:
         # configure loves to immediately delete things, causing issues for
@@ -216,28 +262,44 @@ def attachBitcodePathToObject(bcPath, outFileName):
     os.remove(f.name)
 
     if orc != 0:
-        print objcopyCmd
-        print('objcopy failed with {0}'.format(orc))
+        _logger.error('objcopy failed with {0}'.format(orc))
         sys.exit(-1)
 
 class BuilderBase(object):
-    def __init__(self, cmd, isCxx):
+    def __init__(self, cmd, isCxx, prefixPath=None):
         self.cmd = cmd
         self.isCxx = isCxx
 
+        # Used as prefix path for compiler
+        if prefixPath:
+          self.prefixPath = prefixPath
+
+          # Ensure prefixPath has trailing slash
+          if self.prefixPath[-1] != os.path.sep:
+            self.prefixPath = self.prefixPath + os.path.sep
+
+          # Check prefix path exists
+          if not os.path.exists(self.prefixPath):
+            errorMsg='Path to compiler "{0}" does not exist'.format(self.prefixPath)
+            _logger.error(errorMsg)
+            raise Exception(errorMsg)
+
+        else:
+          self.prefixPath = ''
+
 class ClangBuilder(BuilderBase):
-    def __init__(self, cmd, isCxx):
-        super(ClangBuilder, self).__init__(cmd, isCxx)
+    def __init__(self, cmd, isCxx, prefixPath=None):
+        super(ClangBuilder, self).__init__(cmd, isCxx, prefixPath)
 
     def getBitcodeCompiler(self):
         cc = self.getCompiler()
         return cc + ['-emit-llvm']
 
     def getCompiler(self):
         if self.isCxx:
-            return ['clang++']
+            return ['{0}clang++'.format(self.prefixPath)]
         else:
-            return ['clang']
+            return ['{0}clang'.format(self.prefixPath)]
 
     def getBitcodeArglistFilter(self):
         return ClangBitcodeArgumentListFilter(self.cmd)
@@ -259,8 +321,8 @@ def attachBitcode(self, argFilter):
         attachBitcodePathToObject(bcname, outFile)
 
 class DragoneggBuilder(BuilderBase):
-    def __init__(self, cmd, isCxx):
-        super(DragoneggBuilder, self).__init__(cmd, isCxx)
+    def __init__(self, cmd, isCxx, prefixPath=None):
+        super(DragoneggBuilder, self).__init__(cmd, isCxx, prefixPath)
 
     def getBitcodeCompiler(self):
         pth = os.getenv('LLVM_DRAGONEGG_PLUGIN')
@@ -274,9 +336,9 @@ def getCompiler(self):
             pfx = os.getenv('LLVM_GCC_PREFIX')
 
         if self.isCxx:
-            return ['{0}g++'.format(pfx)]
+            return ['{0}{1}g++'.format(self.prefixPath, pfx)]
         else:
-            return ['{0}gcc'.format(pfx)]
+            return ['{0}{1}gcc'.format(self.prefixPath, pfx)]
 
     def getBitcodeArglistFilter(self):
         return ArgumentListFilter(self.cmd)
@@ -291,13 +353,25 @@ def extraBitcodeArgs(self, argFilter):
 
 
 def getBuilder(cmd, isCxx):
-    cstring = os.getenv('LLVM_COMPILER')
+    compilerEnv = 'LLVM_COMPILER'
+    cstring = os.getenv(compilerEnv)
+    pathPrefix = os.getenv(llvmCompilerPathEnv) # Optional
+    _logger.info('WLLVM compiler using {0}'.format(cstring))
+    if pathPrefix:
+      _logger.info('WLLVM compiler path prefix "{0}"'.format(pathPrefix))
+
     if cstring == 'clang':
-        return ClangBuilder(cmd, isCxx)
+        return ClangBuilder(cmd, isCxx, pathPrefix)
     elif cstring == 'dragonegg':
-        return DragoneggBuilder(cmd, isCxx)
+        return DragoneggBuilder(cmd, isCxx, pathPrefix)
+    elif cstring == None:
+        errorMsg = ' No compiler set. Please set environment variable ' + compilerEnv
+        _logger.critical(errorMsg)
+        raise Exception(errorMsg)
     else:
-        raise Exception('Invalid compiler type: ' + cstring)
+        errorMsg= compilerEnv + '=' + str(cstring) + ' : Invalid compiler type'
+        _logger.critical(errorMsg)
+        raise Exception(errorMsg)
 
 def buildObject(builder):
     objCompiler = builder.getCompiler()
@@ -330,3 +404,4 @@ def buildAndAttachBitcode(builder):
     if rc == 0:
         builder.attachBitcode(af)
     sys.exit(rc)
+