merging pygments support from ui branch with x86 highlight formatter,

release parts of tests/ directory and add tox+travis configs

Author: bdcht

.gitignore

@@ -2,4 +2,7 @@ tags
 *.pyc
 *.swp
 build/
-tests/
+dist/
+*.egg-info
+tests/priv
+.tox/

.travis.yml

@@ -0,0 +1,15 @@
+language: python
+python: 2.7
+install:
+  - python setup.py install
+  - pip install tox
+  - apt-get install unzip
+  - wget -q -O /tmp/z3.zip https://github.com/Z3Prover/bin/blob/master/releases/z3-4.4.0-x64-ubuntu-14.04.zip
+  - mkdir z3
+  - unzip -q -d z3 -j /tmp/z3.zip
+  - export PYTHONPATH=$PYTHONPATH:$(pwd)/z3
+script:
+  - tox
+env:
+  - TOXENV=py27
+  - TOXENV=py27-full

README.rst

@@ -1,6 +1,10 @@
 =====
 Amoco
 =====
+
+.. image:: https://travis-ci.org/bdcht/amoco.svg?branch=release
+    :target: https://travis-ci.org/bdcht/amoco
+
 +-----------+-----------------------------------+
 | Status:   | Under Development                 |
 +-----------+-----------------------------------+
@@ -88,7 +92,7 @@ Amoco is tested on python 2.7 and depends on the following python packages:
 - grandalf_ used for building CFG (and eventually rendering it)
 - crysp_    used by the generic intruction decoder (``arch/core.py``)
 - z3_       used to simplify expressions and solve constraints
-- pygments_ (not in current release, planned for 2.4.2 release)
+- pygments_ used for pretty printing of assembly code and expressions
 - pyparsing_ for parsing instruction decoder formats
 - ply_ (optional), for parsing *GNU as* files
 - zodb_ (optional), provides persistence of amoco objects in a database
@@ -102,7 +106,7 @@ Below is a very simple example where basic blocks are build with linear sweep:
 .. sourcecode:: python
 
  >>> import amoco
- >>> p = amoco.system.loader.load_program('tests/samples/flow.elf')
+ >>> p = amoco.system.loader.load_program('tests/samples/x86/flow.elf')
  amoco.system.loader: INFO: Elf32 file detected
  amoco.system.loader: INFO: linux_x86 program created
  >>> p
@@ -1334,6 +1338,20 @@ Please see `LICENSE`_.
 Changelog
 =========
 
+- `v2.4.2`_
+
+  * merge support for pygments pretty printing methods (in ui.render module)
+  * add x86 hilighted syntax formatter (in arch.x86.formats)
+  * expose expression's pretty printing interface (exp.pp(), exp.toks())
+  * remove default config class fallback (ConfigParser is standard)
+  * merge some samples and tests ported to pytest package
+  * use setuptools, add tox.ini and travis-ci config
+  * fix some x86/x64 semantics
+  * improve sparc v8 formats
+  * add sparc coprocessor registers
+  * update README
+
+
 - `v2.4.1`_
 
   * add lbackward analysis and func.makemap() implementations
@@ -1412,6 +1430,7 @@ Changelog
 .. _ply: http://www.dabeaz.com/ply/
 .. _zodb: http://www.zodb.org
 .. _LICENSE: https://github.com/bdcht/amoco/blob/release/LICENSE
+.. _v2.4.2: https://github.com/bdcht/amoco/releases/tag/v2.4.2
 .. _v2.4.1: https://github.com/bdcht/amoco/releases/tag/v2.4.1
 .. _v2.4.0: https://github.com/bdcht/amoco/releases/tag/v2.4.0
 .. _v2.3.5: https://github.com/bdcht/amoco/releases/tag/v2.3.5

amoco/arch/core.py

@@ -458,13 +458,21 @@ def __init__(self,formats):
         self.formats = formats
         self.default = ('{i.mnemonic} ', lambda i: ', '.join(map(str,i.operands)))
 
-    def __call__(self,i):
-        s=[]
+    def getkey(self,i):
+        if i.mnemonic in self.formats: return i.mnemonic
+        if i.spec.hook.func_name in self.formats: return i.spec.hook.func_name
+        return None
+
+    def getparts(self,i):
         try:
             fmts = self.formats[i.mnemonic]
         except KeyError:
             fmts = self.formats.get(i.spec.hook.func_name,self.default)
-        for f in fmts:
+        return fmts
+
+    def __call__(self,i):
+        s=[]
+        for f in self.getparts(i):
             if hasattr(f,'format'):
                 # It is a string
                 s.append(f.format(i=i))

amoco/arch/sparc/env.py

@@ -18,6 +18,8 @@
 R = [reg('r%d'%x,32) for x in range(520)]
 # fpu registers:
 f = [reg('f%d'%x,32) for x in range(32)]
+# coprocessor registers:
+c = [reg('c%d'%x,32) for x in range(32)]
 
 # symbols for r0 ... r7:
 g0,g1,g2,g3,g4,g5,g6,g7 = (reg('g%d'%x,32) for x in range(8))

amoco/arch/x86/cpu_x86.py

@@ -7,7 +7,9 @@
 from amoco.arch.core import instruction, disassembler
 
 instruction.set_uarch(uarch)
-from amoco.arch.x86.formats import IA32_Intel
+
+from amoco.arch.x86.formats import *
+
 instruction.set_formatter(IA32_Intel)
 
 from amoco.arch.x86 import spec_ia32
@@ -17,3 +19,12 @@
 
 def PC():
     return eip
+
+def configure(**kargs):
+    from amoco.config import get_module_conf
+    conf = get_module_conf('x86')
+    conf.update(kargs)
+    if conf['highlight']:
+        instruction.set_formatter(IA32_Intel_highlighted)
+
+configure()

amoco/arch/x86/formats.py

@@ -69,3 +69,39 @@ def oprel(i):
 
 IA32_Intel = Formatter(IA32_Intel_formats)
 IA32_Intel.default = format_intel_default
+
+# highlighted formatter:
+#-----------------------
+from amoco.ui import render
+
+def IA32_Intel_tokenize(i):
+    toks = []
+    for f in IA32_Intel.getparts(i):
+        if   f is pfx:   toks.append((render.Token.Prefix,f(i)))
+        elif f is mnemo: toks.append((render.Token.Mnemonic,f(i)))
+        elif f is oprel:
+            s = f(i)
+            if s.startswith('*'):
+                t = render.Token.Address
+            else:
+                t = render.Token.Constant
+            toks.append((t,s))
+        elif f is opers:
+            for op in i.operands:
+                if   op._is_reg: toks.append((render.Token.Register,str(op)))
+                elif op._is_mem: toks.append((render.Token.Memory,deref(op)))
+                elif op._is_cst:
+                    if i.misc['imm_ref'] is not None:
+                        toks.append((render.Token.Address,str(i.misc['imm_ref'])))
+                    elif op.sf:
+                        toks.append((render.Token.Constant,'%+d'%op.value))
+                    else:
+                        toks.append((render.Token.Constant,str(op)))
+                toks.append((render.Token.Literal,', '))
+            if toks[-1][0] is render.Token.Literal: toks.pop()
+        else:
+            toks.append((render.Token.Comment,f(i)))
+    return toks
+
+def IA32_Intel_highlighted(null, i):
+    return render.highlight(IA32_Intel_tokenize(i))

amoco/cas/expressions.py

@@ -7,6 +7,8 @@
 from amoco.logger import Log
 logger = Log(__name__)
 
+from amoco.ui.render import Token,highlight
+
 # decorators:
 #------------
 
@@ -92,11 +94,12 @@ def setendian(cls,e):
     def length(self): # length value is in bytes
         return self.size/8
 
-    def bytes(self,sta=0,sto=None):
+    def bytes(self,sta=0,sto=None,endian=0):
         s = slice(sta,sto)
         l = self.length
         sta,sto,stp = s.indices(l)
-        if self._endian==-1:
+        endian |= self._endian
+        if endian==-1:
             sta,sto = l-sto,l-sta
         return self[sta*8:sto*8]
 
@@ -131,6 +134,12 @@ def __str__(self):
         if self._is_def is False: return '⊥%d'%self.size
         raise ValueError("void expression")
 
+    def toks(self,**kargs):
+        return [(Token.Literal,str(self))]
+
+    def pp(self,**kargs):
+        return highlight(self.toks(**kargs))
+
     def bit(self,i):
         i = i%self.size
         return self[i:i+1]
@@ -293,6 +302,9 @@ def __int__(self):
     def __str__(self):
         return '{:#x}'.format(self.value)
 
+    def toks(self,**kargs):
+        return [(Token.Constant,str(self))]
+
     def to_sym(self,ref):
         return sym(ref,self.v,self.size)
 
@@ -473,6 +485,9 @@ def __int__(self):
     def __str__(self):
         return '{:f}'.format(self.value)
 
+    def toks(self,**kargs):
+        return [(Token.Constant,str(self))]
+
     def eval(self,env): return cfp(self.value,self.size)
 
     def __neg__(self):
@@ -571,6 +586,9 @@ def __getitem__(self,i):
     def __str__(self):
         return self.ref
 
+    def toks(self,**kargs):
+        return [(Token.Register,str(self))]
+
     def eval(self,env):
         return env[self]
 
@@ -609,6 +627,10 @@ def __init__(self,refname,**kargs):
     def __str__(self):
         return '@%s'%self.ref
 
+    def toks(self,**kargs):
+        tk = Token.Tainted if '!' in self.ref else Token.Name
+        return [(tk,str(self))]
+
     def __setattr__(self,a,v):
         exp.__setattr__(self,a,v)
 
@@ -676,6 +698,17 @@ def __str__(self):
             cur += nv.size
         return s+' }'
 
+    def toks(self,**kargs):
+        tl = (Token.Literal,', ')
+        s  = [(Token.Literal,'{')]
+        for nv in self:
+            t = nv.toks(newline=False,**kargs)
+            s.extend(t)
+            s.append(tl)
+        if len(s)>1: s.pop()
+        s.append((Token.Literal,'}'))
+        return s
+
     def eval(self,env):
         res = comp(self.size)
         res.smask = self.smask[:]
@@ -833,6 +866,9 @@ def __str__(self):
         n = '$%d'%n if n>0 else ''
         return 'M%d%s%s'%(self.size,n,self.a)
 
+    def toks(self,**kargs):
+        return [(Token.Memory,str(self))]
+
     def eval(self,env):
         a = self.a.eval(env)
         m = env.use()
@@ -873,6 +909,9 @@ def __str__(self):
         d = '%+d'%self.disp if self.disp else ''
         return '%s(%s%s)'%(self.seg,self.base,d)
 
+    def toks(self,**kargs):
+        return [(Token.Address,str(self))]
+
     def simplify(self):
         self.base,offset = extract_offset(self.base)
         self.disp += offset
@@ -950,7 +989,12 @@ def __setattr__(self,a,v):
 
     def __str__(self):
         return self.ref or self.raw()
-    ##
+
+    def toks(self,**kargs):
+        if self._is_reg: return [(Token.Register,str(self))]
+        subpart = [(Token.Literal,'[%d:%d]'%(self.pos,self.pos+self.size))]
+        return self.x.toks(**kargs)+subpart
+
     def __hash__(self): return hash(self.raw())
 
     def eval(self,env):
@@ -1021,6 +1065,14 @@ def __init__(self,t,l,r):
     def __str__(self):
         return '(%s ? %s : %s)'%(str(self.tst),str(self.l),str(self.r))
 
+    def toks(self,**kargs):
+        ttest  = self.tst.toks(**kargs)
+        ttest.append((Token.Literal,' ? '))
+        ttrue  = self.l.toks(**kargs)
+        ttrue.append((Token.Literal,' : '))
+        tfalse = self.r.toks(**kargs)
+        return ttest+ttrue+tfalse
+
     # default verify method if smt module is not loaded.
     # here we check if tst or its negation exist in env.conds but we can
     # only rely on "syntaxic" features unless we have a solver.
@@ -1103,6 +1155,13 @@ def eval(self,env):
     def __str__(self):
         return '(%s%s%s)'%(str(self.l),self.op.symbol,str(self.r))
 
+    def toks(self,**kargs):
+        l = self.l.toks(**kargs)
+        l.insert(0,(Token.Literal,'('))
+        r = self.r.toks(**kargs)
+        r.append((Token.Literal,')'))
+        return l+[(Token.Literal,self.op.symbol)]+r
+
     def simplify(self):
         minus = (self.op.symbol=='-')
         l = self.l.simplify()
@@ -1168,6 +1227,11 @@ def l(self): return None
     def __str__(self):
         return '(%s%s)'%(self.op.symbol,str(self.r))
 
+    def toks(self,**kargs):
+        r = self.r.toks(**kargs)
+        r.append((Token.Literal,')'))
+        return [(Token.Literal,'(%s'%self.op.symbol)]+r
+
     def simplify(self):
         self.r = self.r.simplify()
         if not self.r._is_def: return top(self.size)

amoco/cas/mapper.py

@@ -52,6 +52,10 @@ def __len__(self):
     def __str__(self):
         return '\n'.join(["%s <- %s"%x for x in self])
 
+    def pp(self,**kargs):
+        lv = [(l.pp(**kargs),v.pp(**kargs)) for (l,v) in self]
+        return '\n'.join(["%s <- %s"%x for x in lv])
+
     def __getstate__(self):
         return (self.__map,self.csi)
     def __setstate__(self,state):