Merge pull request travitch#66 from SRI-CSL/master

Random minor improvements

Author: ianamason

Makefile

@@ -4,9 +4,7 @@ all:
 	@echo 'Here are the targets:'
 	@echo ''
 	@echo 'To develop          :    "make develop"'
-	@echo 'To test install     :    "make testinstall"'
 	@echo 'To install          :    "make install"'
-	@echo 'To test publish     :    "make testpublish"'
 	@echo 'To publish          :    "make publish"'
 
 	@echo 'To check clang      :    "make check_clang"'
@@ -17,11 +15,7 @@ all:
 	@echo ''
 	@echo 'e.g. on linux: PATH=/usr/lib/llvm-3.3/bin:... make check_dragonegg'
 	@echo ''
-	@echo 'To turn md 2 html   :    "make zippity"'
-	@echo ''
-	@echo 'then upload the zip file to https://pypi.python.org/pypi'
-	@echo ''
-	@echo 'To pylint                            :  "make lint"'
+	@echo 'To pylint           :  "make lint"'
 	@echo ''
 
 
@@ -38,16 +32,8 @@ dist: clean
 # INCREASE the version number in wllvm/version.py,
 # otherwise the server will give you an error.
 
-testpublish: dist
-	python setup.py register -r https://testpypi.python.org/pypi
-	python setup.py sdist upload -r https://testpypi.python.org/pypi
-
-testinstall:
-	pip install -i https://testpypi.python.org/pypi wllvm
-
 publish: dist
-	python setup.py register -r https://pypi.python.org/pypi
-	python setup.py sdist upload -r https://pypi.python.org/pypi
+	python setup.py sdist upload
 
 install:
 	pip install
@@ -58,11 +44,6 @@ check_clang:
 check_dragonegg:
 	cd test; python -m unittest -v test_base_driver test_dragonegg_driver
 
-zippity:
-	rm -rf doczip*; mkdir doczip;
-	cat README.md | pandoc -f markdown_github > doczip/index.html
-	zip -r -j doczip.zip doczip
-
 clean:
 	rm -f  wllvm/*.pyc wllvm/*~
 

README.md

@@ -31,7 +31,8 @@ presence of static libraries in builds.  WLLVM's approach has the
 distinct advantage of generating working binaries, in case some part
 of a build process requires that.
 
-WLLVM works with either clang or the gcc dragonegg plugin.
+WLLVM works with either clang or the gcc dragonegg plugin. If you are not interested in dragonegg support,
+and speed is an issue for you, you may want to try out [gllvm.](https://github.com/SRI-CSL/gllvm)
 
 Installation
 ------------
@@ -241,17 +242,25 @@ Debugging
 ---------
 
 The WLLVM tools can show various levels of output to aid with debugging.
-To show this output set WLLVM_OUTPUT to one of the following levels:
+To show this output set the `WLLVM_OUTPUT_LEVEL` environment
+variable to one of the following levels:
 
- * `CRITICAL`
  * `ERROR`
  * `WARNING`
  * `INFO`
  * `DEBUG`
 
-For example
+For example:
+```
+    export WLLVM_OUTPUT_LEVEL=DEBUG
+```
+Output will be directed to the standard error stream, unless you specify the
+path of a logfile via the `WLLVM_OUTPUT_FILE` environment variable.
 
-    export WLLVM_OUTPUT=DEBUG
+For example:
+```
+    export WLLVM_OUTPUT_FILE=/tmp/wllvm.log
+```
 
 
 Sanity Checking

setup.py

@@ -20,7 +20,7 @@
     description='Whole Program LLVM',
     long_description=long_description,
     url='https://github.com/SRI-CSL/whole-program-llvm',
-    author='Ian A. Mason, Tristan Ravitch, Dan Liew, Bruno Dutertre, Benjamin Schubert, Berkeley Churchill, Marko Dimjasevic, Will Dietz, Fabian Mager, Ben Liblit, Andrew Santosa, Tomas Kalibera, Loic  Gelle.',
+    author='Ian A. Mason, Tristan Ravitch, Dan Liew, Bruno Dutertre, Benjamin Schubert, Berkeley Churchill, Marko Dimjasevic, Will Dietz, Fabian Mager, Ben Liblit, Andrew Santosa, Tomas Kalibera, Loic  Gelle, Joshua Cranmer.',
     author_email='iam@csl.sri.com',
 
 

wllvm/arglistfilter.py

@@ -31,6 +31,8 @@ class ArgumentListFilter(object):
     def __init__(self, inputList, exactMatches={}, patternMatches={}):
         defaultArgExactMatches = {
 
+            '-' : (0, ArgumentListFilter.standardInCallback),
+
             '-o' : (1, ArgumentListFilter.outputFileCallback),
             '-c' : (0, ArgumentListFilter.compileOnlyCallback),
             '-E' : (0, ArgumentListFilter.preprocessOnlyCallback),
@@ -122,6 +124,14 @@ def __init__(self, inputList, exactMatches={}, patternMatches={}):
             # Language
             '-ansi' : (0, ArgumentListFilter.compileUnaryCallback),
             '-pedantic' : (0, ArgumentListFilter.compileUnaryCallback),
+            #iam: i notice that yices configure passes -xc so
+            # we should have a fall back pattern that captures the case
+            # when there is no space between the x and the langauge.
+            # for what its worth: the manual says the language can be one of
+            # c  objective-c  c++ c-header  cpp-output  c++-cpp-output
+            # assembler  assembler-with-cpp
+            # BD: care to comment on your configure?
+
             '-x' : (1, ArgumentListFilter.compileBinaryCallback),
 
             # Debug
@@ -235,6 +245,8 @@ def __init__(self, inputList, exactMatches={}, patternMatches={}):
             r'^--sysroot=.+$' :  (0, ArgumentListFilter.compileUnaryCallback),
             r'^-print-prog-name=.*$' : (0, ArgumentListFilter.compileUnaryCallback),
             r'^-print-file-name=.*$' : (0, ArgumentListFilter.compileUnaryCallback),
+            #iam: -xc from yices. why BD?
+            r'^-x.+$' : (0, ArgumentListFilter.compileUnaryCallback),
 
         }
 
@@ -256,7 +268,7 @@ def __init__(self, inputList, exactMatches={}, patternMatches={}):
         self.isAssembly = False
         self.isCompileOnly = False
         self.isEmitLLVM = False
-
+        self.isStandardIn = False
 
         argExactMatches = dict(defaultArgExactMatches)
         argExactMatches.update(exactMatches)
@@ -297,6 +309,16 @@ def __init__(self, inputList, exactMatches={}, patternMatches={}):
         if DUMPING:
             self.dump()
 
+
+    def skipBitcodeGeneration(self):
+        if os.environ.get('WLLVM_CONFIGURE_ONLY', False):
+            return True
+        if not self.inputFiles or self.isEmitLLVM or self.isAssembly or self.isAssembleOnly:
+            return True
+        if self.isPreprocessOnly or self.isStandardIn or (self.isDependencyOnly and not self.isCompileOnly):
+            return True
+        return False
+
     def _shiftArgs(self, nargs):
         ret = []
         while nargs > 0:
@@ -305,6 +327,11 @@ def _shiftArgs(self, nargs):
             nargs = nargs - 1
         return ret
 
+
+    def standardInCallback(self, flag):
+        _logger.debug('standardInCallback: %s', flag)
+        self.isStandardIn = True
+
     def abortUnaryCallback(self, flag):
         _logger.warning('Out of context experience: "%s" "%s"', str(self.inputList), flag)
         sys.exit(1)

wllvm/compilers.py

@@ -17,7 +17,6 @@
 # Internal logger
 _logger = logConfig(__name__)
 
-
 def wcompile(mode):
     """ The workhorse, called from wllvm and wllvm++.
     """
@@ -28,13 +27,29 @@ def wcompile(mode):
         cmd = list(sys.argv)
         cmd = cmd[1:]
         builder = getBuilder(cmd, mode)
+
+        af = builder.getBitcodeArglistFilter()
+
         rc = buildObject(builder)
 
-        if rc == 0 and not os.environ.get('WLLVM_CONFIGURE_ONLY', False):
-            buildAndAttachBitcode(builder)
+        # phase one compile failed. no point continuing
+        if rc != 0:
+            _logger.info('phase one failed: %s', str(sys.argv))
+            return rc
+
+        # no need to generate bitcode (e.g. configure only, assembly, ....)
+        if af.skipBitcodeGeneration():
+            _logger.info('No work to do')
+            _logger.debug(af.__dict__)
+            return rc
+
+        # phase two
+        buildAndAttachBitcode(builder, af)
+
     except Exception as e:
-        _logger.debug('%s: exception case: %s', mode, str(e))
+        _logger.warning('%s: exception case: %s', mode, str(e))
 
+    _logger.info('Calling %s returned %d', list(sys.argv), rc)
     return rc
 
 
@@ -134,6 +149,7 @@ def attachBitcodePathToObject(bcPath, outFileName):
 
 class BuilderBase(object):
     def __init__(self, cmd, mode, prefixPath=None):
+        self.af = None     #memoize the arglist filter
         self.cmd = cmd
         self.mode = mode
 
@@ -169,7 +185,9 @@ def getCompiler(self):
         return ['{0}{1}'.format(self.prefixPath, os.getenv(env) or prog)]
 
     def getBitcodeArglistFilter(self):
-        return ClangBitcodeArgumentListFilter(self.cmd)
+        if self.af is None:
+            self.af = ClangBitcodeArgumentListFilter(self.cmd)
+        return self.af
 
 class DragoneggBuilder(BuilderBase):
     def getBitcodeCompiler(self):
@@ -198,15 +216,17 @@ def getCompiler(self):
         return ['{0}{1}{2}'.format(self.prefixPath, pfx, mode)]
 
     def getBitcodeArglistFilter(self):
-        return ArgumentListFilter(self.cmd)
+        if self.af is None:
+            self.af = ArgumentListFilter(self.cmd)
+        return self.af
 
 def getBuilder(cmd, mode):
     compilerEnv = 'LLVM_COMPILER'
     cstring = os.getenv(compilerEnv)
     pathPrefix = os.getenv(llvmCompilerPathEnv) # Optional
-    _logger.info('WLLVM compiler using %s', cstring)
+    _logger.debug('WLLVM compiler using %s', cstring)
     if pathPrefix:
-        _logger.info('WLLVM compiler path prefix "%s"', pathPrefix)
+        _logger.debug('WLLVM compiler path prefix "%s"', pathPrefix)
 
     if cstring == 'clang':
         return ClangBuilder(cmd, mode, pathPrefix)
@@ -231,14 +251,7 @@ def buildObject(builder):
 
 
 # This command does not have the executable with it
-def buildAndAttachBitcode(builder):
-
-    af = builder.getBitcodeArglistFilter()
-
-    if not af.inputFiles or af.isEmitLLVM or af.isAssembly or af.isAssembleOnly or (af.isDependencyOnly and not af.isCompileOnly) or af.isPreprocessOnly:
-        _logger.debug('No work to do')
-        _logger.debug(af.__dict__)
-        return
+def buildAndAttachBitcode(builder, af):
 
     #iam: when we have multiple input files we'll have to keep track of their object files.
     newObjectFiles = []

wllvm/extraction.py

@@ -8,7 +8,6 @@
 import tempfile
 import shutil
 import argparse
-import hashlib
 
 from .popenwrapper import Popen
 

wllvm/logconfig.py

@@ -7,23 +7,33 @@
 import os
 import sys
 
-_loggingEnv = 'WLLVM_OUTPUT'
+# iam: 6/30/2017 decided to move to a gllvm style where we can set the level and the output file
+_loggingEnvLevel_old = 'WLLVM_OUTPUT'
+_loggingEnvLevel_new = 'WLLVM_OUTPUT_LEVEL'
 
-_validLogLevels = ['CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG']
+_loggingDestination = 'WLLVM_OUTPUT_FILE'
+
+_validLogLevels = ['ERROR', 'WARNING', 'INFO', 'DEBUG']
 
 def logConfig(name):
 
-    logging.basicConfig(level=logging.WARNING, format='%(levelname)s:%(message)s')
+    destination = os.getenv(_loggingDestination)
+
+    if destination:
+        logging.basicConfig(filename=destination, level=logging.WARNING, format='%(levelname)s:%(message)s')
+    else:
+        logging.basicConfig(level=logging.WARNING, format='%(levelname)s:%(message)s')
 
     retval = logging.getLogger(name)
 
-    level = os.getenv(_loggingEnv)
+    # ignore old setting
+    level = os.getenv(_loggingEnvLevel_new)
 
     if level:
         level = level.upper()
         if not level in _validLogLevels:
-            logging.error('"%s" is not a valid value for %s. Valid values are %s',
-                          level, _loggingEnv, _validLogLevels)
+            logging.error('"%s" is not a valid value for %s or %s. Valid values are %s',
+                          level, _loggingEnvLevel_old, _loggingEnvLevel_new, _validLogLevels)
             sys.exit(1)
         else:
             retval.setLevel(getattr(logging, level))

wllvm/version.py

@@ -54,6 +54,11 @@
 
 1.1.2    - 4/26/2017 encoding issues with hashlib in the python 3 swarm.
 
+1.1.3    - 5/20/2017 fortran support via flang (pull #60 over at travitch's place)
+
+1.1.4    - 7/24/2017 improvements motivated by gllvm and logic.
+
+
 """
 
-wllvm_version = '1.1.2'
+wllvm_version = '1.1.4'