Merge pull request #26 from SRI-CSL/master

Dragonegg now compiles apache, AND is part of the travis_ci

Author: travitch

.travis.yml

@@ -8,14 +8,14 @@ python:
 install:
   - sudo apt-get update
   - sudo apt-get install llvm-3.4 clang-3.4 libapr1-dev libaprutil1-dev
-  - sudo apt-get install dragonegg llvm-3.3 llvm-gcc
+  - sudo apt-get install dragonegg llvm-3.0 llvm-gcc
   - export WLLVM_HOME=`pwd`
 
 # command to run tests
 script:
   # build apache with clang (i.e. httpd-2.4.12) 
   - ${WLLVM_HOME}/.travis/apache_clang.sh
-  # when it works we can uncomment this puppy out
-  # - ${WLLVM_HOME}/.travis/apache_dragonegg.sh
+  # build apache with gcc and dragonegg (i.e. httpd-2.4.12) 
+  - ${WLLVM_HOME}/.travis/apache_dragonegg.sh
 
 

.travis/apache_dragonegg.sh

@@ -2,6 +2,7 @@
 # Make sure we exit if there is a failure
 set -e
 
+export dragonegg_disable_version_check=true
 
 export PATH=/usr/lib/llvm-3.0/bin:${WLLVM_HOME}:${PATH}
 export LLVM_COMPILER=dragonegg

NOTES.txt

@@ -1,51 +1,53 @@
 
-[![Build Status](https://travis-ci.org/travitch/whole-program-llvm.svg?branch=master)](https://travis-ci.org/travitch/whole-program-llvm)
+[![Build Status](https://travis-ci.org/SRI-CSL/whole-program-llvm.svg?branch=master)](https://travis-ci.org/SRI-CSL/whole-program-llvm)
 
 
 Introduction
 ============
 
-This is a small python-based wrapper around a GCC-compatible compiler
-to make it easy to build whole-program (or whole-library) LLVM bitcode
-files.  The idea is that it first invokes the compiler as normal to
-build a real object file.  It then invokes a bitcode compiler to
-generate the corresponding bitcode, recording the location of the
-bitcode file in an ELF section of the actual object file.
-
-When object files are linked together, the contents of non-special ELF
-sections are just concatenated (so we don't lose the locations of any
-of the constituent bitcode files).
-
-This package contains an extra utility, extract-bc, to read the
-contents of this ELF section and link all of the bitcode into a single
-whole-program bitcode file. This utility can also be used on built
-native static libraries to generate LLVM bitcode archives.
-
-This two-phase build process is slower and more elaborate than normal,
-but in practice is necessary to be a drop-in replacement for gcc in
-any build system.  Approaches using the LTO framework in gcc and the
-gold linker plugin work for many cases, but fail in the presence of
-static libraries in builds.  This approach has the distinct advantage
-of generating working binaries, in case some part of a build process
-actually requires that.
-
-Currently, this package only works using clang or the dragonegg plugin
-with gcc 4.5 (with the required patch for dragonegg).
+This project, WLLVM, provides tools for building whole-program (or
+whole-library) LLVM bitcode files from an unmodified C or C++
+source package. It currently runs on `*nix` platforms such as Linux,
+FreeBSD, and Mac OS X.
+
+WLLVM provides python-based compiler wrappers that work in two
+steps. The wrappers first invoke the compiler as normal. Then, for
+each object file, they call a bitcode compiler to produce LLVM
+bitcode. The wrappers also store the location of the generated bitcode
+file in a dedicated section of the object file.  When object files are
+linked together, the contents of the dedicated sections are
+concatenated (so we don't lose the locations of any of the constituent
+bitcode files). After the build completes, one can use an WLLVM
+utility to read the contents of the dedicated section and link all of
+the bitcode into a single whole-program bitcode file. This utility
+works for both executable and native libraries.
+
+Currently, WLLVM works with either clang or the gcc dragonegg plugin.
+
+This two-phase build process is necessary to be a drop-in replacement
+for gcc or g++ in any build system.  Using the LTO framework in gcc
+and the gold linker plugin works in many cases, but fails in the
+presence of static libraries in builds.  WLLVM's approach has the
+distinct advantage of generating working binaries, in case some part
+of a build process requires that.
+
 
 Usage
 =====
 
-There are three environment variables that must be set to use this
-wrapper script:
+WLLVM includes two python executables: `wllvm` for compiling C code
+and `wllvm++` for C++, and an auxiliary tool `extract-bc`.
+
+Three environment variables must be set to use these wrappers:
 
- * `LLVM_COMPILER` should be set to 'dragonegg' or 'clang'.
+ * `LLVM_COMPILER` should be set to either 'dragonegg' or 'clang'.
  * `LLVM_GCC_PREFIX` should be set to the prefix for the version of gcc that should
    be used with dragonegg.  This can be empty if there is no prefix.  This variable is
    not used if `$LLVM_COMPILER == clang`.
  * `LLVM_DRAGONEGG_PLUGIN` should be the full path to the dragonegg plugin.  This
    variable is not used if `$LLVM_COMPILER == clang`.
 
-Once the environment is set up, just use wllvm and wllvm++ as your C
+Once the environment is set up, just use `wllvm` and `wllvm++` as your C
 and C++ compilers, respectively.
 
 In addition to the above environment variables the following can be optionally used:
@@ -58,13 +60,28 @@ In addition to the above environment variables the following can be optionally u
    variable.
    Example `LLVM_COMPILER_PATH=/home/user/llvm_and_clang/Debug+Asserts/bin`.
 
-*  `WLLVM_CONFIGURE_ONLY` can be set to anything, when set `wllvm` and `wllvm++`
-   will not carry out the second phase that involves the production of bitcode.
-   This may prevent configuration errors being cause by the unexpected production
-   of the hidden bitcode files.
+* `WLLVM_CONFIGURE_ONLY` can be set to anything. If it is set, `wllvm`
+   and `wllvm++` behave like a normal C or C++ compiler. They do not
+   the produce bitcode.  Setting `WLLVM_CONFIGURE_ONLY` may prevent
+   configuration errors caused by the unexpected production of hidden
+   bitcode files.
+
+
+Building a bitcode module with clang
+====================================
+
+    export LLVM_COMPILER=clang
 
-Example building bitcode module
-===============================
+    tar xf pkg-config-0.26.tar.gz
+    cd pkg-config-0.26
+    CC=wllvm ./configure
+    make
+
+    # Produces pkg-config.bc
+    extract-bc pkg-config
+
+Building a bitcode module with dragonegg
+========================================
 
     export LLVM_COMPILER=dragonegg
     export LLVM_GCC_PREFIX=llvm-
@@ -78,8 +95,9 @@ Example building bitcode module
     # Produces pkg-config.bc
     extract-bc pkg-config
 
-Example building bitcode archive
-================================
+
+Building bitcode archive
+========================
 
     export LLVM_COMPILER=clang
     tar -xvf bullet-2.81-rev2613.tgz
@@ -91,20 +109,21 @@ Example building bitcode archive
     # Produces src/LinearMath/libLinearMath.bca
     extract-bc src/LinearMath/libLinearMath.a
 
-Example building an Operating System
-================================
 
-To see how to build freeBSD 10.0 from scratch check out the guide 
-[here.](../master/README-freeBSD.md)
+Building an Operating System
+============================
 
+To see how to build freeBSD 10.0 from scratch check out this 
+[guide.](../master/README-freeBSD.md)
 
-Example configuring without building bitcode
+
+Configuring without building bitcode
 ================================
 
 
     WLLVM_CONFIGURE_ONLY=1 CC=wllvm ./configure
     CC=wllvm make
-    
+
 
 Debugging
 =========

README.md

@@ -53,18 +53,15 @@ except AttributeError as e:
     logging.error('Output file argument not found.\nException message: ' + str(e))
     sys.exit(1)
 
-bcfilename = '.{0}.bc'.format(filename)
-bcPath = os.path.join(dirs, bcfilename)
-fakeAssembler = [llvmAssembler, infile, '-o', bcPath]
+fakeAssembler = [llvmAssembler, infile, '-o', argFilter.outFileName]
+
 asmProc = Popen(fakeAssembler)
 realRet = asmProc.wait()
 
 if realRet != 0:
     logging.error('llvm-as failed')
     sys.exit(realRet)
 
-attachBitcodePathToObject(bcPath, argFilter.outFileName)
-
 sys.exit(realRet)
 
 

driver/as

@@ -32,8 +32,8 @@
 # Internal logger
 _logger = logging.getLogger(__name__)
 
-# Flag for debugging
-DEBUG = False
+# Flag for dumping
+DUMPING = False
 
 
 # This class applies filters to GCC argument lists.  It has a few
@@ -179,11 +179,15 @@ def __init__(self, inputList, exactMatches={}, patternMatches={}):
             '-static' : (0, ArgumentListFilter.linkUnaryCallback),
             '-nostdlib' : (0, ArgumentListFilter.linkUnaryCallback),
             '-nodefaultlibs' : (0, ArgumentListFilter.linkUnaryCallback),
+            '-rdynamic' : (0, ArgumentListFilter.linkUnaryCallback),
             # darwin flags
             '-dynamiclib' : (0, ArgumentListFilter.linkUnaryCallback),
             '-current_version' : (1, ArgumentListFilter.linkBinaryCallback),
             '-compatibility_version' : (1, ArgumentListFilter.linkBinaryCallback),
 
+            # dragonegg mystery argument
+            '--64' : (0, ArgumentListFilter.compileUnaryCallback),
+
             #
             # BD: need to warn the darwin user that these flags will rain on their parade
             # (the Darwin ld is a bit single minded)
@@ -278,7 +282,7 @@ def __init__(self, inputList, exactMatches={}, patternMatches={}):
                     _logger.warning('Did not recognize the compiler flag "{0}"'.format(currentItem))
                     self.compileUnaryCallback(currentItem)
 
-        if DEBUG:
+        if DUMPING:
             self.dump()
 
     def _shiftArgs(self, nargs):
@@ -296,7 +300,7 @@ def abortUnaryCallback(self, flag):
     def inputFileCallback(self, infile):
         _logger.debug('Input file: ' + infile)
         self.inputFiles.append(infile)
-        if re.search('\\.(s|S)', infile):
+        if re.search('\\.(s|S)$', infile):
             self.isAssembly = True
 
     def outputFileCallback(self, flag, filename):
@@ -364,7 +368,7 @@ def getOutputFilename(self):
 
     # iam: returns a pair [objectFilename, bitcodeFilename] i.e .o and .bc.
     # the hidden flag determines whether the objectFile is hidden like the
-    # bitcodeFile is (starts with a '.'), use the DEBUG flag to get a sense
+    # bitcodeFile is (starts with a '.'), use the logging level & DUMPING flag to get a sense
     # of what is being written out.
     def getArtifactNames(self, srcFile, hidden=False):
         (srcpath, srcbase) = os.path.split(srcFile)
@@ -381,15 +385,15 @@ def getArtifactNames(self, srcFile, hidden=False):
 
     #iam: for printing our partitioning of the args
     def dump(self):
-        print("compileArgs: ", self.compileArgs)
-        print("inputFiles: ", self.inputFiles)
-        print("linkArgs: ", self.linkArgs)
-        print("objectFiles: ", self.objectFiles)
-        print("outputFilename: ", self.outputFilename)
+        _logger.debug('compileArgs: {0}'.format(self.compileArgs))
+        _logger.debug('inputFiles: {0}'.format(self.inputFiles))
+        _logger.debug('linkArgs: {0}'.format(self.linkArgs))
+        _logger.debug('objectFiles: {0}'.format(self.objectFiles))
+        _logger.debug('outputFilename: {0}'.format(self.outputFilename))
         for srcFile in self.inputFiles:
-            print("srcFile: ", srcFile)
+            _logger.debug('srcFile: {0}'.format(srcFile))
             (objFile, bcFile) = self.getArtifactNames(srcFile)
-            print("{0} ===> ({1}, {2})".format(srcFile, objFile, bcFile))
+            _logger.debug('{0} ===> ({1}, {2})'.format(srcFile, objFile, bcFile))
 
 
 
@@ -455,6 +459,7 @@ def attachBitcodePathToObject(bcPath, outFileName):
     # Don't try to attach a bitcode path to a binary.  Unfortunately
     # that won't work.
     (root, ext) = os.path.splitext(outFileName)
+    _logger.debug('attachBitcodePathToObject: {0}  ===> {1} [ext = {2}]\n'.format(bcPath, outFileName, ext))
     #iam: this also looks very dodgey; we need a more reliable way to do this:
     if ext not in ('.o', '.lo', '.os', '.So', '.po'):
         _logger.warning('Cannot attach bitcode path to "{0} of type {1}"'.format(outFileName, FileType.getFileType(outFileName)))
@@ -520,6 +525,12 @@ def __init__(self, cmd, isCxx, prefixPath=None):
         else:
           self.prefixPath = ''
 
+    #clang and drogonegg share the same taste in bitcode filenames.
+    def getBitcodeFileName(self, argFilter):
+        (dirs, baseFile) = os.path.split(argFilter.getOutputFilename())
+        bcfilename = os.path.join(dirs, '.{0}.bc'.format(baseFile))
+        return bcfilename
+
 class ClangBuilder(BuilderBase):
     def __init__(self, cmd, isCxx, prefixPath=None):
         super(ClangBuilder, self).__init__(cmd, isCxx, prefixPath)
@@ -537,11 +548,6 @@ def getCompiler(self):
     def getBitcodeArglistFilter(self):
         return ClangBitcodeArgumentListFilter(self.cmd)
 
-    def getBitcodeFileName(self, argFilter):
-        (dirs, baseFile) = os.path.split(argFilter.getOutputFilename())
-        bcfilename = os.path.join(dirs, '.{0}.bc'.format(baseFile))
-        return bcfilename
-
     def extraBitcodeArgs(self, argFilter):
         bcPath = self.getBitcodeFileName(argFilter)
         return ['-o', bcPath]
@@ -562,8 +568,9 @@ def getBitcodeCompiler(self):
         # We use '-B' to tell gcc where to look for an assembler.
         # When we build LLVM bitcode we do not want to use the GNU assembler,
         # instead we want gcc to use our own assembler (see driver/as).
-        return cc + ['-B', driverDir, '-fplugin={0}'.format(pth),
-                     '-fplugin-arg-dragonegg-emit-ir']
+        cmd = cc + ['-B', driverDir, '-fplugin={0}'.format(pth), '-fplugin-arg-dragonegg-emit-ir']
+        _logger.debug(cmd)
+        return cmd
 
     def getCompiler(self):
         pfx = ''
@@ -687,6 +694,7 @@ def buildBitcodeFile(builder, srcFile, bcFile):
     bcc.extend(af.compileArgs)
     bcc.extend(['-c', srcFile])
     bcc.extend(['-o', bcFile])
+    _logger.debug('buildBitcodeFile: {0}\n'.format(bcc))
     proc = Popen(bcc)
     rc = proc.wait()
     if rc != 0:
@@ -699,6 +707,7 @@ def buildObjectFile(builder, srcFile, objFile):
     cc.extend(af.compileArgs)
     cc.append(srcFile)
     cc.extend(['-c', '-o',  objFile])
+    _logger.debug('buildObjectFile: {0}\n'.format(cc))
     proc = Popen(cc)
     rc = proc.wait()
     if rc != 0:

driver/utils.py

@@ -344,7 +344,7 @@ def process_file_unix(inputFile, outputFile, llvmLinker, llvmArchiver):
     arCmd   =  ['ar', 'xv'] if verboseFlag else ['ar', 'x']
     ofileType = FileType.ELF_OBJECT
 
-    if ft == FileType.ELF_EXECUTABLE or ft == FileType.ELF_SHARED:
+    if ft == FileType.ELF_EXECUTABLE or ft == FileType.ELF_SHARED or ft == FileType.ELF_OBJECT:
         logging.info('Generating LLVM Bitcode module')
         return handleExecutable(inputFile, outputFile, extractor, llvmLinker)
     elif ft == FileType.ARCHIVE:
@@ -364,7 +364,7 @@ def process_file_darwin(inputFile, outputFile, llvmLinker, llvmArchiver):
     arCmd   =  ['ar', '-x', '-v'] if verboseFlag else ['ar', '-x']
     ofileType = FileType.MACH_OBJECT
 
-    if ft == FileType.MACH_EXECUTABLE or ft == FileType.MACH_SHARED:
+    if ft == FileType.MACH_EXECUTABLE or ft == FileType.MACH_SHARED or ft == FileType.MACH_OBJECT:
         logging.info('Generating LLVM Bitcode module')
         return handleExecutable(inputFile, outputFile, extractor, llvmLinker)
     elif ft == FileType.ARCHIVE: