Merge pull request travitch#78 from SRI-CSL/master

SRI fork updates

Author: ianamason

README.md

@@ -242,9 +242,9 @@ Cross-Compilation
 
 To support cross-compilation WLLVM supports the `-target` triple used by clang.
 More information can be found
-[here.](https://clang.llvm.org/docs/CrossCompilation.html#target-triple).
+[here.](https://clang.llvm.org/docs/CrossCompilation.html#target-triple)
 
-Additionall, WLLVM leverages `objcopy` for some of its heavy lifting. When
+Additionally, WLLVM leverages `objcopy` for some of its heavy lifting. When
 cross-compiling you must ensure to use the appropriate `objcopy` for the target
 architecture. The `BINUTILS_TARGET_PREFIX` environment variable can be used to
 set the objcopy of choice, for example, `arm-linux-gnueabihf`.

doc/tutorial-ubuntu-16.04.md

@@ -75,7 +75,7 @@ Extract the bitcode.
 -rw-r--r-- 1 vagrant vagrant 1119584 Aug  4 20:02 httpd.bc
 ```
 
-## Step 6.  
+## Step 6.
 
 Turn the bitcode into a second executable binary. (optional -- just for fun and sanity checking)
 

setup.py

@@ -20,7 +20,7 @@
     description='Whole Program LLVM',
     long_description=long_description,
     url='https://github.com/SRI-CSL/whole-program-llvm',
-    author='Ian A. Mason, Tristan Ravitch, Dan Liew, Bruno Dutertre, Benjamin Schubert, Berkeley Churchill, Marko Dimjasevic, Will Dietz, Fabian Mager, Ben Liblit, Andrew Santosa, Tomas Kalibera, Loic  Gelle, Joshua Cranmer, Alexander Bakst.',
+    author='Ian A. Mason, Tristan Ravitch, Dan Liew, Bruno Dutertre, Benjamin Schubert, Berkeley Churchill, Marko Dimjasevic, Will Dietz, Fabian Mager, Ben Liblit, Andrew Santosa, Tomas Kalibera, Loic  Gelle, Joshua Cranmer, Alexander Bakst, Miguel Arroyo.',
     author_email='iam@csl.sri.com',
 
 

test/test_files/Makefile

@@ -61,7 +61,7 @@ link_with_archive:: archive
 	$(CC) main.o libfoo.a -o main.arch
 
 clean:
-	rm -f *.o main main.arch .*.o.bc .*.o *.bc .*.bc a.out *.s *.i hello *.a *.bca *.dylib *.manifest
+	rm -f *.o main main.arch .*.o.bc .*.o *.bc .*.bc a.out *.s *.i hello *.a *.bca *.dylib *.manifest *.ll
 
 mystery:
 	otool -X -s __WLLVM __llvm_bc main > main.otool

wllvm/arglistfilter.py

@@ -39,7 +39,7 @@ def __init__(self, inputList, exactMatches={}, patternMatches={}):
             '-S' : (0, ArgumentListFilter.assembleOnlyCallback),
 
             '--verbose' : (0, ArgumentListFilter.verboseFlagCallback),
-            '--param' : (1, ArgumentListFilter.defaultBinaryCallback),
+            '--param' : (1, ArgumentListFilter.compileBinaryCallback),
             '-aux-info' : (1, ArgumentListFilter.defaultBinaryCallback),
 
             #iam: presumably the len(inputFiles) == 0 in this case
@@ -103,6 +103,8 @@ def __init__(self, inputList, exactMatches={}, patternMatches={}):
             '-D' : (1, ArgumentListFilter.compileBinaryCallback),
             '-U' : (1, ArgumentListFilter.compileBinaryCallback),
 
+            '-arch' : (1, ArgumentListFilter.compileBinaryCallback),  #iam: openssl
+
             # Dependency generation
             '-M'  : (0, ArgumentListFilter.dependencyOnlyCallback),
             '-MM' : (0, ArgumentListFilter.dependencyOnlyCallback),
@@ -223,7 +225,8 @@ def __init__(self, inputList, exactMatches={}, patternMatches={}):
             # Update: found a fix for problem 1: add flag -keep_private_externs when
             # calling ld -r.
             #
-            '-Wl,-dead_strip' :  (0, ArgumentListFilter.darwinWarningLinkUnaryCallback),
+            '-Wl,-dead_strip' :  (0, ArgumentListFilter.warningLinkUnaryCallback),
+            '-dead_strip' :  (0, ArgumentListFilter.warningLinkUnaryCallback),
             '-Oz' : (0, ArgumentListFilter.compileUnaryCallback),   #did not find this in the GCC options.
             '-mno-global-merge' : (0, ArgumentListFilter.compileUnaryCallback),  #clang (do not merge globals)
 
@@ -254,6 +257,7 @@ def __init__(self, inputList, exactMatches={}, patternMatches={}):
             r'^-U.+$' : (0, ArgumentListFilter.compileUnaryCallback),
             r'^-Wl,.+$' : (0, ArgumentListFilter.linkUnaryCallback),
             r'^-W(?!l,).*$' : (0, ArgumentListFilter.compileUnaryCallback),
+            r'^-fsanitize=.+$' : (0, ArgumentListFilter.compileLinkUnaryCallback),
             r'^-f.+$' : (0, ArgumentListFilter.compileUnaryCallback),
             r'^-rtlib=.+$' : (0, ArgumentListFilter.linkUnaryCallback),
             r'^-std=.+$' : (0, ArgumentListFilter.compileUnaryCallback),
@@ -289,6 +293,8 @@ def __init__(self, inputList, exactMatches={}, patternMatches={}):
         #iam: try and split the args into linker and compiler switches
         self.compileArgs = []
         self.linkArgs = []
+        # currently only dead_strip belongs here; but I guess there could be more.
+        self.forbiddenArgs = []
 
 
         self.isVerbose = False
@@ -423,13 +429,10 @@ def compileUnaryCallback(self, flag):
         _logger.debug('compileUnaryCallback: %s', flag)
         self.compileArgs.append(flag)
 
-    def darwinWarningLinkUnaryCallback(self, flag):
-        _logger.debug('darwinWarningLinkUnaryCallback: %s', flag)
-        if sys.platform.startswith('darwin'):
-            _logger.warning('The flag "%s" cannot be used with this tool', flag)
-            sys.exit(1)
-        else:
-            self.linkArgs.append(flag)
+    def warningLinkUnaryCallback(self, flag):
+        _logger.debug('warningLinkUnaryCallback: %s', flag)
+        _logger.warning('The flag "%s" cannot be used with this tool; we are ignoring it', flag)
+        self.forbiddenArgs.append(flag)
 
     def defaultBinaryCallback(self, flag, arg):
         _logger.warning('Ignoring compiler arg pair: "%s %s"', flag, arg)

wllvm/as.py

@@ -62,7 +62,7 @@ def outFileCallback(self, flag, name):
         self.outFileName = name
 
 def main():
-    """ Entry point to the assebler 'as' in the dragonegg realm.
+    """ Entry point to the assembler 'as' in the dragonegg realm.
     """
     argFilter = BCFilter(sys.argv[1:])
     # Since this is just the assembler, there should only ever be one file

wllvm/compilers.py

@@ -31,6 +31,7 @@ def wcompile(mode):
     try:
         cmd = list(sys.argv)
         cmd = cmd[1:]
+
         builder = getBuilder(cmd, mode)
 
         af = builder.getBitcodeArglistFilter()
@@ -188,6 +189,16 @@ def __init__(self, cmd, mode, prefixPath=None):
         else:
             self.prefixPath = ''
 
+    def getCommand(self):
+        if self.af is not None:
+            # need to remove things like "-dead_strip"
+            forbidden = self.af.forbiddenArgs
+            if forbidden:
+                for baddy in forbidden:
+                    self.cmd.remove(baddy)
+        return self.cmd
+
+
 class ClangBuilder(BuilderBase):
     def getBitcodeCompiler(self):
         cc = self.getCompiler()
@@ -264,7 +275,7 @@ def getBuilder(cmd, mode):
 
 def buildObject(builder):
     objCompiler = builder.getCompiler()
-    objCompiler.extend(builder.cmd)
+    objCompiler.extend(builder.getCommand())
     proc = Popen(objCompiler)
     rc = proc.wait()
     _logger.debug('buildObject rc = %d', rc)

wllvm/extraction.py

@@ -20,7 +20,8 @@
 
 from .filetype import FileType
 
-from .logconfig import logConfig
+from .logconfig import logConfig, informUser
+
 
 
 _logger = logConfig(__name__)
@@ -236,7 +237,7 @@ def archiveFiles(pArgs, fileNames):
             break
 
     if retCode == 0:
-        _logger.info('Generated LLVM bitcode archive %s', pArgs.outputFile)
+        informUser('Generated LLVM bitcode archive {0}\n'.format(pArgs.outputFile))
     else:
         _logger.error('Failed to generate LLVM bitcode archive')
 
@@ -342,12 +343,86 @@ def extractFile(archive, filename, instance):
 
 
 
+def handleArchiveDarwin(pArgs):
+    originalDir = os.getcwd() # This will be the destination
+
+    pArgs.arCmd.append(pArgs.inputFile)
+
+    # Make temporary directory to extract objects to
+    tempDir = ''
+    bitCodeFiles = []
+
+    try:
+
+
+        tempDir = tempfile.mkdtemp(suffix='wllvm')
+        os.chdir(tempDir)
+
+        # Extract objects from archive
+        try:
+            arP = Popen(pArgs.arCmd)
+        except OSError as e:
+            if e.errno == 2:
+                errorMsg = 'Your ar does not seem to be easy to find.\n'
+            else:
+                errorMsg = 'OS error({0}): {1}'.format(e.errno, e.strerror)
+            _logger.error(errorMsg)
+            raise Exception(errorMsg)
+
+        arPE = arP.wait()
+
+        if arPE != 0:
+            errorMsg = 'Failed to execute archiver with command {0}'.format(pArgs.arCmd)
+            _logger.error(errorMsg)
+            raise Exception(errorMsg)
+
+        _logger.debug(2)
+
+        # Iterate over objects and examine their bitcode inserts
+        for (root, _, files) in os.walk(tempDir):
+            _logger.debug('Exploring "%s"', root)
+            for f in files:
+                fPath = os.path.join(root, f)
+                if FileType.getFileType(fPath) == pArgs.fileType:
+
+                    # Extract bitcode locations from object
+                    contents = pArgs.extractor(fPath)
+
+                    for bcFile in contents:
+                        if bcFile != '':
+                            if not os.path.exists(bcFile):
+                                _logger.warning('%s lists bitcode library "%s" but it could not be found', f, bcFile)
+                            else:
+                                bitCodeFiles.append(bcFile)
+                else:
+                    _logger.info('Ignoring file "%s" in archive', f)
+
+        _logger.info('Found the following bitcode file names to build bitcode archive:\n%s', pprint.pformat(bitCodeFiles))
+
+    finally:
+        # Delete the temporary folder
+        _logger.debug('Deleting temporary folder "%s"', tempDir)
+        shutil.rmtree(tempDir)
+
+    #write the manifest file if asked for
+    if pArgs.manifestFlag:
+        manifestFile = '{0}.llvm.manifest'.format(pArgs.inputFile)
+        with open(manifestFile, 'w') as output:
+            for f in bitCodeFiles:
+                output.write('{0}\n'.format(f))
+
+    # Build bitcode archive
+    os.chdir(originalDir)
+
+    return buildArchive(pArgs, bitCodeFiles)
+
+
 
 #iam: 5/1/2018
-def handleArchive(pArgs):
-    """ handleArchive processes a archive, and creates either a bitcode archive, or a module, depending on the flags used.
+def handleArchiveLinux(pArgs):
+    """ handleArchiveLinux processes a archive, and creates either a bitcode archive, or a module, depending on the flags used.
 
-    Archives are strange beasts. handleArchive processes the archive by:
+    Archives on Linux are strange beasts. handleArchive processes the archive by:
 
       1. first creating a table of contents of the archive, which maps file names (in the archive) to the number of
     times a file with that name is stored in the archive.
@@ -430,8 +505,7 @@ def buildArchive(pArgs, bitCodeFiles):
             pArgs.outputFile = pArgs.inputFile
             pArgs.outputFile += '.' + moduleExtension
 
-        _logger.info('Writing output to %s', pArgs.outputFile)
-
+        informUser('Writing output to {0}\n'.format(pArgs.outputFile))
         return linkFiles(pArgs, bitCodeFiles)
 
     else:
@@ -446,8 +520,7 @@ def buildArchive(pArgs, bitCodeFiles):
             else:
                 pArgs.outputFile = pArgs.inputFile + bcaExtension
 
-        _logger.info('Writing output to %s', pArgs.outputFile)
-
+        informUser('Writing output to {0}\n'.format(pArgs.outputFile))
         return archiveFiles(pArgs, bitCodeFiles)
 
 
@@ -567,7 +640,7 @@ def process_file_unix(pArgs):
         _logger.info('Generating LLVM Bitcode module')
         retval = handleExecutable(pArgs)
     elif ft == FileType.ARCHIVE:
-        retval = handleArchive(pArgs)
+        retval = handleArchiveLinux(pArgs)
     elif ft == FileType.THIN_ARCHIVE:
         retval = handleThinArchive(pArgs)
     else:
@@ -589,7 +662,10 @@ def process_file_darwin(pArgs):
         _logger.info('Generating LLVM Bitcode module')
         retval = handleExecutable(pArgs)
     elif ft == FileType.ARCHIVE:
-        retval = handleArchive(pArgs)
+        _logger.info('Handling archive')
+        retval = handleArchiveDarwin(pArgs)
+
+
     else:
         _logger.error('File "%s" of type %s cannot be used', pArgs.inputFile, FileType.revMap[ft])
     return retval

wllvm/logconfig.py

@@ -50,3 +50,7 @@ def loggingConfiguration():
     destination = os.getenv(_loggingDestination)
     level = os.getenv(_loggingEnvLevel_new)
     return (destination, level)
+
+
+def informUser(msg):
+    sys.stderr.write(msg)

wllvm/version.py

@@ -60,11 +60,17 @@
 
 1.1.5    - 3/14/2018 fixes suggested by Alexander Bakst
 
-1.1.6    - 4/24/2018 fixes suggested by building the Linux kernel and trying to harmonize with gllvm.
+1.2.0    - 4/24/2018 fixes suggested by building the Linux kernel and trying to harmonize with gllvm.
            4/28/2018 can handle thin archives, can sort bitcode input to llvm-{ar, link} and manifest via the -s switch.
            5/1/2018 can handle archives correctly (deal with multiple files with the same name in the archive).
 
+1.2.1    - 5/13/2018  -fsanitize=  now recognized as a compile AND link flag (mothers day edition)
+
+1.2.2    - 6/1/2018 lots of minor fixes from building big projects (and their dependencies) like tor
+
+1.2.3    - 4/15/2019 The tax day version. Almost a years worth of tweaks from building large things like the Linux kernel.
+
 """
 
-wllvm_version = '1.1.6'
-wllvm_date = 'May 1st 2018'
+wllvm_version = '1.2.3'
+wllvm_date = 'April 15 2019'