Merge pull request travitch#58 from SRI-CSL/master

--emit-llvm, bitcode store, and minor bug fixes.

Author: ianamason

README.md

@@ -40,7 +40,7 @@ As of August 2016 WLLVM is now a pip package. You can just do:
 
     pip install wllvm
 
-or 
+or
 
     sudo pip install wllvm
 
@@ -222,8 +222,19 @@ Building a bitcode archive then extracting the bitcode
     llvm-ar x libjansson.bca
     ls -la
 
-    
 
+Preserving bitcode files in a store
+--------------------------------
+
+Sometimes it can be useful to preserve the bitcode files produced in a
+build, either to prevent deletion or to retrieve it later. If the
+environment variable `WLLVM_BC_STORE` is set to the absolute path of
+an existing directory,
+then WLLVM will copy the produced bitcode file into that directory.
+The name of the copied bitcode file is the hash of the path to the
+original bitcode file.  For convenience, when using both the manifest
+feature of `extract-bc` and the store, the manifest will contain both
+the original path, and the store path.
 
 
 Debugging

setup.py

@@ -20,7 +20,7 @@
     description='Whole Program LLVM',
     long_description=long_description,
     url='https://github.com/SRI-CSL/whole-program-llvm',
-    author='Ian A. Mason, Tristan Ravitch, Dan Liew, Bruno Dutertre, Benjamin Schubert, Berkeley Churchill, Marko Dimjasevic, Will Dietz, Fabian Mager, Ben Liblit, Andrew Santosa.',
+    author='Ian A. Mason, Tristan Ravitch, Dan Liew, Bruno Dutertre, Benjamin Schubert, Berkeley Churchill, Marko Dimjasevic, Will Dietz, Fabian Mager, Ben Liblit, Andrew Santosa, Tomas Kalibera, Loic  Gelle.',
     author_email='iam@csl.sri.com',
 
 

test/test_files/Makefile

@@ -1,5 +1,6 @@
 #iam: please leave this Makefile; it is useful for debugging when things break.
 
+all: one archive
 
 zero:
 	${CXX} hello.cc -o hello

wllvm/arglistfilter.py

@@ -264,7 +264,7 @@ def __init__(self, inputList, exactMatches={}, patternMatches={}):
         self._inputArgs = collections.deque(inputList)
 
         #iam: parse the cmd line, bailing if we discover that there will be no second phase.
-        while (len(self._inputArgs) > 0   and
+        while (self._inputArgs   and
                not (self.isAssembly or
                     self.isAssembleOnly or
                     self.isPreprocessOnly)):
@@ -397,8 +397,7 @@ def getOutputFilename(self):
             (_, base) = os.path.split(self.inputFiles[0])
             (root, _) = os.path.splitext(base)
             return '{0}.o'.format(root)
-        else:
-            return 'a.out'
+        return 'a.out'
 
     # iam: returns a pair [objectFilename, bitcodeFilename] i.e .o and .bc.
     # the hidden flag determines whether the objectFile is hidden like the

wllvm/checker.py

@@ -13,6 +13,7 @@
 import subprocess as sp
 import errno
 
+from .version import wllvm_version
 
 explain_LLVM_COMPILER = """
 
@@ -90,6 +91,7 @@ def check(self):
         1. Check that the OS is supported.
         2. Checks that the compiler settings make sense.
         3. Checks that the needed LLVM utilities exists.
+        4. Check that the store, if set, exists.
         """
 
         if not self.checkOS():
@@ -100,6 +102,7 @@ def check(self):
 
         if success:
             self.checkAuxiliaries()
+            self.checkStore()
 
         return 0 if success else 1
 
@@ -115,12 +118,12 @@ def checkOS(self):
     def checkSwitch(self):
         """Checks the correctness of the LLVM_COMPILER env var."""
         compiler_type = os.getenv('LLVM_COMPILER')
+        vmsg = 'We are wllvm version {0} and'.format(wllvm_version)
         if compiler_type == 'clang':
-            return (1, '\nGood, we are using clang.\n')
+            return (1, '\n{0} we are using clang.\n'.format(vmsg))
         elif compiler_type == 'dragonegg':
-            return (2, '\nOK, we are using dragonegg.\n')
-        else:
-            return (0, explain_LLVM_COMPILER)
+            return (2, '\n{0} we are using dragonegg.\n'.format(vmsg))
+        return (0, explain_LLVM_COMPILER)
 
 
     def checkClang(self):
@@ -182,9 +185,8 @@ def checkCompiler(self):
         elif code == 2:
             print(comment)
             return self.checkDragonegg()
-        else:
-            print('Insane')
-            return False
+        print('Insane')
+        return False
 
 
 
@@ -196,12 +198,12 @@ def checkCompilers(self, cc, cxx):
         if not ccOk:
             print('The C compiler {0} was not found or not executable.\nBetter not try using wllvm!\n'.format(cc))
         else:
-            print('The C compiler {0} is:\n\n{1}\n'.format(cc, extractLine(ccVersion, 0)))
+            print('The C compiler {0} is:\n\n\t{1}\n'.format(cc, extractLine(ccVersion, 0)))
 
         if not cxxOk:
             print('The CXX compiler {0} was not found or not executable.\nBetter not try using wllvm++!\n'.format(cxx))
         else:
-            print('The C++ compiler {0} is:\n\n{1}\n'.format(cxx, extractLine(cxxVersion, 0)))
+            print('The C++ compiler {0} is:\n\n\t{1}\n'.format(cxx, extractLine(cxxVersion, 0)))
 
         if not ccOk or  not cxxOk:
             print(explain_LLVM_COMPILER_PATH)
@@ -227,8 +229,7 @@ def checkExecutable(self, exe, version_switch='-v'):
                 return (False, '{0} not executable'.format(exe))
             elif e.errno == errno.ENOENT:
                 return (False, '{0} not found'.format(exe))
-            else:
-                return (False, '{0} not sure why, errno is {1}'.format(exe, e.errno))
+            return (False, '{0} not sure why, errno is {1}'.format(exe, e.errno))
         else:
             return (True, compilerOutput)
 
@@ -256,21 +257,30 @@ def checkAuxiliaries(self):
             print('The bitcode linker {0} was not found or not executable.\nBetter not try using extract-bc!\n'.format(link))
             print(explain_LLVM_LINK_NAME)
         else:
-            print('The bitcode linker {0} is:\n\n{1}\n'.format(link, extractLine(linkVersion, 1)))
+            print('The bitcode linker {0} is:\n\n\t{1}\n'.format(link, extractLine(linkVersion, 1)))
 
         if not arOk:
             print('The bitcode archiver {0} was not found or not executable.\nBetter not try using extract-bc!\n'.format(ar))
             print(explain_LLVM_AR_NAME)
         else:
-            print('The bitcode archiver {0} is:\n\n{1}\n'.format(ar, extractLine(arVersion, 1)))
+            print('The bitcode archiver {0} is:\n\n\t{1}\n'.format(ar, extractLine(arVersion, 1)))
 
 
+    def checkStore(self):
+        """Checks that the bitcode store, if set, makes sense."""
+        store_dir = os.getenv('WLLVM_BC_STORE')
+        if store_dir:
+            if os.path.exists(store_dir) and os.path.isdir(store_dir) and os.path.isabs(store_dir):
+                print('Using the bitcode store:\n\n\t{0}\n\n'.format(store_dir))
+            else:
+                print('The bitcode store:\n\n\t{0}\n\nis either not absolute, does not exist, or is not a directory.\n\n'.format(store_dir))
+        else:
+            print('Not using a bitcode store.\n\n')
+
 
 def extractLine(version, n):
     if not version:
         return version
     lines = version.split('\n')
-    if n < len(lines):
-        return lines[n]
-    else:
-        return lines[-1]
+    line = lines[n] if n < len(lines) else lines[-1]
+    return line.strip() if line else line

wllvm/compilers.py

@@ -5,7 +5,9 @@
 import os
 import sys
 import tempfile
+import hashlib
 
+from shutil import copyfile
 from .filetype import FileType
 from .popenwrapper import Popen
 from .arglistfilter import ArgumentListFilter
@@ -120,6 +122,13 @@ def attachBitcodePathToObject(bcPath, outFileName):
         _logger.error('objcopy failed with %s', orc)
         sys.exit(-1)
 
+    # loicg: If the environment variable WLLVM_BC_STORE is set, copy the bitcode
+    # file to that location, using a hash of the original bitcode path as a name
+    storeEnv = os.getenv('WLLVM_BC_STORE')
+    if storeEnv:
+        hashName = hashlib.sha256(absBcPath).hexdigest()
+        copyfile(absBcPath, os.path.join(storeEnv, hashName))
+
 class BuilderBase(object):
     def __init__(self, cmd, isCxx, prefixPath=None):
         self.cmd = cmd
@@ -140,16 +149,7 @@ def __init__(self, cmd, isCxx, prefixPath=None):
         else:
             self.prefixPath = ''
 
-    #clang and dragonegg share the same taste in bitcode filenames.
-    def getBitcodeFileName(self, argFilter):
-        (dirs, baseFile) = os.path.split(argFilter.getOutputFilename())
-        bcfilename = os.path.join(dirs, '.{0}.bc'.format(baseFile))
-        return bcfilename
-
 class ClangBuilder(BuilderBase):
-    def __init__(self, cmd, isCxx, prefixPath=None):
-        super(ClangBuilder, self).__init__(cmd, isCxx, prefixPath)
-
     def getBitcodeCompiler(self):
         cc = self.getCompiler()
         return cc + ['-emit-llvm']
@@ -159,31 +159,16 @@ def getCompiler(self):
             cxx = os.getenv('LLVM_CXX_NAME')
             if cxx:
                 return ['{0}{1}'.format(self.prefixPath, cxx)]
-            else:
-                return ['{0}clang++'.format(self.prefixPath)]
-        else:
-            cc = os.getenv('LLVM_CC_NAME')
-            if cc:
-                return ['{0}{1}'.format(self.prefixPath, cc)]
-            else:
-                return ['{0}clang'.format(self.prefixPath)]
+            return ['{0}clang++'.format(self.prefixPath)]
+        cc = os.getenv('LLVM_CC_NAME')
+        if cc:
+            return ['{0}{1}'.format(self.prefixPath, cc)]
+        return ['{0}clang'.format(self.prefixPath)]
 
     def getBitcodeArglistFilter(self):
         return ClangBitcodeArgumentListFilter(self.cmd)
 
-    def extraBitcodeArgs(self, argFilter):
-        bcPath = self.getBitcodeFileName(argFilter)
-        return ['-o', bcPath]
-
-    def attachBitcode(self, argFilter):
-        bcname = self.getBitcodeFileName(argFilter)
-        outFile = argFilter.getOutputFilename()
-        attachBitcodePathToObject(bcname, outFile)
-
 class DragoneggBuilder(BuilderBase):
-    def __init__(self, cmd, isCxx, prefixPath=None):
-        super(DragoneggBuilder, self).__init__(cmd, isCxx, prefixPath)
-
     def getBitcodeCompiler(self):
         pth = os.getenv('LLVM_DRAGONEGG_PLUGIN')
         cc = self.getCompiler()
@@ -201,21 +186,11 @@ def getCompiler(self):
 
         if self.isCxx:
             return ['{0}{1}g++'.format(self.prefixPath, pfx)]
-        else:
-            return ['{0}{1}gcc'.format(self.prefixPath, pfx)]
+        return ['{0}{1}gcc'.format(self.prefixPath, pfx)]
 
     def getBitcodeArglistFilter(self):
         return ArgumentListFilter(self.cmd)
 
-    # Don't need to do anything since the -B flag in the bitcode
-    # compiler and the assembly stub handles it
-    def attachBitcode(self, argFilter):
-        pass
-
-    def extraBitcodeArgs(self, _):
-        return []
-
-
 def getBuilder(cmd, isCxx):
     compilerEnv = 'LLVM_COMPILER'
     cstring = os.getenv(compilerEnv)
@@ -251,7 +226,7 @@ def buildAndAttachBitcode(builder):
 
     af = builder.getBitcodeArglistFilter()
 
-    if len(af.inputFiles) == 0 or af.isEmitLLVM or af.isAssembly or af.isAssembleOnly or (af.isDependencyOnly and not af.isCompileOnly) or af.isPreprocessOnly:
+    if not af.inputFiles or af.isEmitLLVM or af.isAssembly or af.isAssembleOnly or (af.isDependencyOnly and not af.isCompileOnly) or af.isPreprocessOnly:
         _logger.debug('No work to do')
         _logger.debug(af.__dict__)
         return
@@ -271,9 +246,6 @@ def buildAndAttachBitcode(builder):
         # maybe python-magic is in our future ...
         srcFile = af.inputFiles[0]
         (objFile, bcFile) = af.getArtifactNames(srcFile, hidden)
-        if af.outputFilename is not None:
-            objFile = af.outputFilename
-            bcFile = builder.getBitcodeFileName(af)
         buildBitcodeFile(builder, srcFile, bcFile)
         attachBitcodePathToObject(bcFile, objFile)
 

wllvm/extraction.py

@@ -8,6 +8,7 @@
 import tempfile
 import shutil
 import argparse
+import hashlib
 
 from .popenwrapper import Popen
 
@@ -149,11 +150,38 @@ def extract_section_linux(inputFile):
     return contents
 
 
+def getStorePath(bcPath):
+    storeEnv = os.getenv('WLLVM_BC_STORE')
+    if storeEnv:
+        hashName = hashlib.sha256(bcPath).hexdigest()
+        hashPath = os.path.join(storeEnv, hashName)
+        if os.path.isfile(hashPath):
+            return hashPath
+    return None
+
+
+def getBitcodePath(bcPath):
+    """Tries to resolve the whereabouts of the bitcode.
+
+    First, ihecks if the given path points to an existing bitcode file.
+    If it does not, it tries to look for the bitcode file in the store directory given
+    by the environment variable WLLVM_BC_STORE.
+    """
+
+    if not bcPath or os.path.isfile(bcPath):
+        return bcPath
+
+    storePath = getStorePath(bcPath)
+    if storePath:
+        return storePath
+    return bcPath
+
 def linkFiles(pArgs, fileNames):
     linkCmd = [pArgs.llvmLinker, '-v'] if pArgs.verboseFlag else [pArgs.llvmLinker]
 
-    linkCmd.extend(['-o', pArgs.outputFile])
+    linkCmd.append('-o={0}'.format(pArgs.outputFile))
 
+    fileNames = map(getBitcodePath, fileNames)
     linkCmd.extend([x for x in fileNames if x != ''])
     _logger.info('Writing output to %s', pArgs.outputFile)
     try:
@@ -216,10 +244,7 @@ def handleExecutable(pArgs):
         return 1
 
     if pArgs.manifestFlag:
-        manifestFile = '{0}.llvm.manifest'.format(pArgs.inputFile)
-        with open(manifestFile, 'w') as output:
-            for f in fileNames:
-                output.write('{0}\n'.format(f))
+        writeManifest('{0}.llvm.manifest'.format(pArgs.inputFile), fileNames)
 
     if pArgs.outputFile is None:
         pArgs.outputFile = pArgs.inputFile + '.' + moduleExtension
@@ -289,10 +314,7 @@ def handleArchive(pArgs):
 
     #write the manifest file if asked for
     if pArgs.manifestFlag:
-        manifestFile = '{0}.llvm.manifest'.format(pArgs.inputFile)
-        with open(manifestFile, 'w') as output:
-            for f in bitCodeFiles:
-                output.write('{0}\n'.format(f))
+        writeManifest('{0}.llvm.manifest'.format(pArgs.inputFile), bitCodeFiles)
 
     # Build bitcode archive
     os.chdir(originalDir)
@@ -326,6 +348,16 @@ def buildArchive(pArgs, bitCodeFiles):
         return archiveFiles(pArgs, bitCodeFiles)
 
 
+def writeManifest(manifestFile, bitCodeFiles):
+    with open(manifestFile, 'w') as output:
+        for f in bitCodeFiles:
+            output.write('{0}\n'.format(f))
+            sf = getStorePath(f)
+            if sf:
+                output.write('{0}\n'.format(sf))
+
+
+
 class ExtractedArgs(object):
 
     def __init__(self):