Merge pull request travitch#59 from SRI-CSL/master

To everything (churn, churn, churn)

Author: ianamason

.travis.yml

@@ -15,7 +15,7 @@ install:
   - sudo apt-get update
   # apache prerequisites
   - sudo apt-get install  -y libapr1-dev libaprutil1-dev
-  # for the clang build  
+  # for the clang build
   - sudo apt-get install  -y llvm-3.5 clang-3.5
   # dragonegg prereqs. dragonegg and llvm-gcc use llvm 3.3
   - sudo apt-get install  -y llvm-3.3 llvm-gcc-4.7
@@ -29,13 +29,14 @@ install:
 
 # command to run tests
 script:
-  # Run unittests
+#  # Run unittests
   - python -m unittest discover test/ || exit 1
   - export WLLVM_HOME=`pwd`
+  - ${WLLVM_HOME}/.travis/store.sh
   - export APACHE_VER=2.4.18
   # build apache with clang
   - ${WLLVM_HOME}/.travis/apache_clang.sh
   # build apache with gcc and dragonegg
   - ${WLLVM_HOME}/.travis/apache_dragonegg.sh
-  
-
+  # build musllvm with clang
+  - ${WLLVM_HOME}/.travis/musllvm.sh

.travis/musllvm.sh

@@ -0,0 +1,39 @@
+#!/bin/bash -x
+# Make sure we exit if there is a failure
+set -e
+
+
+export PATH=/usr/lib/llvm-3.5/bin:${PATH}
+export LLVM_COMPILER=clang
+export WLLVM_OUTPUT=WARNING
+
+wllvm-sanity-checker
+
+#setup the store so we test that feature as well
+export WLLVM_BC_STORE=/tmp/bc
+mkdir -p /tmp/bc
+
+git clone https://github.com/SRI-CSL/musllvm.git musllvm
+cd musllvm
+WLLVM_CONFIGURE_ONLY=1  CC=wllvm ./configure --target=LLVM --build=LLVM
+make
+extract-bc --bitcode ./lib/libc.a
+
+if [ -s "./lib/libc.a.bc" ]
+then
+    echo "libc.a.bc exists."
+else
+    exit 1
+fi
+
+#now lets makes sure the store has the bitcode too.
+mv ./lib/libc.a .
+make clean
+extract-bc --bitcode ./libc.a
+
+if [ -s "./libc.a.bc" ]
+then
+    echo "libc.a.bc exists."
+else
+    exit 1
+fi

.travis/store.sh

@@ -0,0 +1,29 @@
+#!/bin/bash -x
+# Make sure we exit if there is a failure
+set -e
+
+
+export PATH=/usr/lib/llvm-3.5/bin:${PATH}
+export LLVM_COMPILER=clang
+export WLLVM_OUTPUT=WARNING
+
+wllvm-sanity-checker
+
+#setup the store so we test that feature as well
+export WLLVM_BC_STORE=/tmp/bc
+mkdir /tmp/bc
+
+cd ./test/test_files
+make clean
+CC=wllvm make one
+mv main ../..
+make clean
+cd ../..
+extract-bc main
+
+if [ -s "main.bc" ]
+then
+    echo "main.bc exists."
+else
+    exit 1
+fi

wllvm/arglistfilter.py

@@ -399,6 +399,11 @@ def getOutputFilename(self):
             return '{0}.o'.format(root)
         return 'a.out'
 
+    def getBitcodeFileName(self):
+        (dirs, baseFile) = os.path.split(self.getOutputFilename())
+        bcfilename = os.path.join(dirs, '.{0}.bc'.format(baseFile))
+        return bcfilename
+
     # iam: returns a pair [objectFilename, bitcodeFilename] i.e .o and .bc.
     # the hidden flag determines whether the objectFile is hidden like the
     # bitcodeFile is (starts with a '.'), use the logging level & DUMPING flag to get a sense
@@ -411,10 +416,7 @@ def getArtifactNames(self, srcFile, hidden=False):
         else:
             objbase = '{0}.o'.format(srcroot)
         bcbase = '.{0}.o.bc'.format(srcroot)
-        path = ''
-        if self.outputFilename is not None:
-            path = os.path.dirname(self.outputFilename)
-        return [os.path.join(path, objbase), os.path.join(path, bcbase)]
+        return [objbase, bcbase]
 
     #iam: for printing our partitioning of the args
     def dump(self):

wllvm/compilers.py

@@ -73,6 +73,9 @@ def outputFileCallback(self, flag, filename):
         self.outputFilename = filename
 
 
+def getHashedPathName(path):
+    return hashlib.sha256(path.encode('utf-8')).hexdigest() if path else None
+
 
 def attachBitcodePathToObject(bcPath, outFileName):
     # Don't try to attach a bitcode path to a binary.  Unfortunately
@@ -106,6 +109,13 @@ def attachBitcodePathToObject(bcPath, outFileName):
         objcopyCmd = ['objcopy', '--add-section', '{0}={1}'.format(elfSectionName, f.name), outFileName]
     orc = 0
 
+    # loicg: If the environment variable WLLVM_BC_STORE is set, copy the bitcode
+    # file to that location, using a hash of the original bitcode path as a name
+    storeEnv = os.getenv('WLLVM_BC_STORE')
+    if storeEnv:
+        hashName = getHashedPathName(absBcPath)
+        copyfile(absBcPath, os.path.join(storeEnv, hashName))
+
     try:
         if os.path.getsize(outFileName) > 0:
             objProc = Popen(objcopyCmd)
@@ -122,13 +132,6 @@ def attachBitcodePathToObject(bcPath, outFileName):
         _logger.error('objcopy failed with %s', orc)
         sys.exit(-1)
 
-    # loicg: If the environment variable WLLVM_BC_STORE is set, copy the bitcode
-    # file to that location, using a hash of the original bitcode path as a name
-    storeEnv = os.getenv('WLLVM_BC_STORE')
-    if storeEnv:
-        hashName = hashlib.sha256(absBcPath).hexdigest()
-        copyfile(absBcPath, os.path.join(storeEnv, hashName))
-
 class BuilderBase(object):
     def __init__(self, cmd, isCxx, prefixPath=None):
         self.cmd = cmd
@@ -246,6 +249,9 @@ def buildAndAttachBitcode(builder):
         # maybe python-magic is in our future ...
         srcFile = af.inputFiles[0]
         (objFile, bcFile) = af.getArtifactNames(srcFile, hidden)
+        if af.outputFilename is not None:
+            objFile = af.outputFilename
+            bcFile = af.getBitcodeFileName()
         buildBitcodeFile(builder, srcFile, bcFile)
         attachBitcodePathToObject(bcFile, objFile)
 

wllvm/extraction.py

@@ -16,6 +16,7 @@
 from .compilers import elfSectionName
 from .compilers import darwinSegmentName
 from .compilers import darwinSectionName
+from .compilers import getHashedPathName
 
 from .filetype import FileType
 
@@ -153,7 +154,7 @@ def extract_section_linux(inputFile):
 def getStorePath(bcPath):
     storeEnv = os.getenv('WLLVM_BC_STORE')
     if storeEnv:
-        hashName = hashlib.sha256(bcPath).hexdigest()
+        hashName = getHashedPathName(bcPath)
         hashPath = os.path.join(storeEnv, hashName)
         if os.path.isfile(hashPath):
             return hashPath
@@ -163,7 +164,7 @@ def getStorePath(bcPath):
 def getBitcodePath(bcPath):
     """Tries to resolve the whereabouts of the bitcode.
 
-    First, ihecks if the given path points to an existing bitcode file.
+    First, checks if the given path points to an existing bitcode file.
     If it does not, it tries to look for the bitcode file in the store directory given
     by the environment variable WLLVM_BC_STORE.
     """
@@ -298,6 +299,7 @@ def handleArchive(pArgs):
 
                     for bcFile in contents:
                         if bcFile != '':
+                            bcFile = getBitcodePath(bcFile)
                             if not os.path.exists(bcFile):
                                 _logger.warning('%s lists bitcode library "%s" but it could not be found', f, bcFile)
                             else:

wllvm/version.py

@@ -50,7 +50,10 @@
 
 1.1.0    - 4/21/2017  no new features on the horizon, no new bugs?
 
+1.1.1    - 4/25/2017 bugs introduced by the new fetures have hopefully been eradicated.
+
+1.1.2    - 4/26/2017 encoding issues with hashlib in the python 3 swarm.
 
 """
 
-wllvm_version = '1.1.0'
+wllvm_version = '1.1.2'