squash merge of develop branch:

 - improve itercfg method and lbackward CFG reconstruction
 - improve widening/fixpoint in computation of func's map
 - improve pretty printing methods
 - start merging ui.graphics packages

Author: bdcht

README.rst

@@ -1338,6 +1338,21 @@ Please see `LICENSE`_.
 Changelog
 =========
 
+- `v2.4.3`_
+
+  * add ui.graphics packages (emptied)
+  * add ui.views module with support for block/func/xfunc
+  * add ui.render.vltable class to pretty print tables
+  * improve instruction formatter class to access pp tokens
+  * cleaner itercfg and lbackward algorithms
+  * add vecw expression class to represent 'widened' vec expressions
+  * improve Memory write of vec expressions
+  * improve widening and fixpoint in func.makemap()
+  * add 'type' attribute (std/pc/flags/stack/other)
+  * define register type for x86 arch
+  * fix some x86/64 decoding/formating/semantics
+  * update travis config, fix pytest vs. Token.
+
 - `v2.4.2`_
 
   * merge support for pygments pretty printing methods (in ui.render module)
@@ -1351,7 +1366,6 @@ Changelog
   * add sparc coprocessor registers
   * update README
 
-
 - `v2.4.1`_
 
   * add lbackward analysis and func.makemap() implementations
@@ -1430,6 +1444,7 @@ Changelog
 .. _ply: http://www.dabeaz.com/ply/
 .. _zodb: http://www.zodb.org
 .. _LICENSE: https://github.com/bdcht/amoco/blob/release/LICENSE
+.. _v2.4.3: https://github.com/bdcht/amoco/releases/tag/v2.4.3
 .. _v2.4.2: https://github.com/bdcht/amoco/releases/tag/v2.4.2
 .. _v2.4.1: https://github.com/bdcht/amoco/releases/tag/v2.4.1
 .. _v2.4.0: https://github.com/bdcht/amoco/releases/tag/v2.4.0

amoco/arch/arm/v7/formats.py

@@ -4,22 +4,24 @@
 from .utils import *
 from amoco.arch.core import Formatter
 
+from amoco.ui.render import Token, TokenListJoin
+
 def mnemo(i):
     m = i.mnemonic
     if hasattr(i,'setflags') and i.setflags:
         m += 'S'
     if hasattr(i,'cond') and i.cond!=CONDITION_AL:
         m += '.%s'%CONDITION[i.cond][0]
-    return '%s'%(m.lower()).ljust(12)
+    return [(Token.Mnemonic,'%s'%(m.lower()).ljust(12))]
 
 def regs(i,limit=None):
     ops = i.operands
     if limit: ops = ops[:limit]
-    return ['{0}'.format(r) for r in ops]
+    return [(Token.Register,'{0}'.format(r)) for r in ops]
 
 def reglist(i,pos=-1):
     l = i.operands[pos]
-    return "{%s}"%(', '.join(['{0}'.format(r) for r in l]))
+    return [(Token.Register,"{%s}"%(', '.join(['{0}'.format(r) for r in l])))]
 
 def deref(i,pos=-2):
     assert len(i.operands)>2
@@ -37,19 +39,19 @@ def deref(i,pos=-2):
             loc = '[%s], %s'%(base, ostr)
     else:
         loc = '[%s], %s'%(base,ostr)
-    return [loc]
+    return [(Token.Memory,loc)]
 
 def label(i,pos=0):
     _pc = i.address
     if _pc is None: _pc=pc
     pcoffset = 4 if internals['isetstate']==0 else 2
     _pc = _pc + 2*pcoffset
     offset = i.operands[pos]
-    return '*'+str(_pc+offset)
+    return [(Token.Address,'*'+str(_pc+offset))]
 
 def setend(i):
     endian_specifier = 'BE' if i.set_bigend else 'LE'
-    return mnemo(i)+endian_specifier
+    return mnemo(i)+[(Token.Literal,endian_specifier)]
 
 def plx(i):
     m = mnemo(i)
@@ -59,23 +61,23 @@ def plx(i):
         ostr = '#%c%d'%(sign,offset.value)
     else:
         ostr = sign+str(offset)
-    loc = '[%s, %s]'%(base, ostr)
+    loc = [(Token.Memory,'[%s, %s]'%(base, ostr))]
     return m+loc
 
 def specreg(i):
     spec_reg = "%s_"%apsr
     if i.write_nzcvq: spec_reg += 'nzcvq'
     if i.write_g: spec_reg += 'g'
-    return '%s, %s'%(i.operands[0],spec_reg)
+    return [(Token.Register,'%s, %s'%(i.operands[0],spec_reg))]
 
-format_allregs = [lambda i: ', '.join(regs(i))]
+format_allregs = [lambda i: TokenListJoin(', ',regs(i))]
 format_default = [mnemo]+format_allregs
 format_sreg    = format_default
 format_label   = [mnemo, label]
-format_adr     = [mnemo, lambda i: '{0}, '.format(i.operands[0]), lambda i: label(i,1)]
+format_adr     = [mnemo, lambda i: regs(i,1), lambda i: label(i,1)]
 format_bits    = format_default
-format_reglist = [mnemo, (lambda i: ', '.join(regs(i,-1))), reglist]
-format_deref   = [mnemo, lambda i: ', '.join(regs(i,-2)+deref(i,-2))]
+format_reglist = [mnemo, (lambda i: TokenListJoin(', ',regs(i,-1))), reglist]
+format_deref   = [mnemo, lambda i: TokenListJoin(', ',regs(i,-2)+deref(i,-2))]
 format_plx     = [plx]
 format_msr     = [mnemo, specreg]
 format_setend  = [setend]

amoco/arch/core.py

@@ -14,6 +14,8 @@
 from amoco.logger import Log
 logger = Log(__name__)
 
+from amoco.ui.render import Token,highlight
+
 type_unpredictable     =  -1
 type_undefined         =   0
 type_data_processing   =   1
@@ -93,15 +95,19 @@ def __repr__(self):
     def set_formatter(cls,f):
         cls.formatter = f
 
-    #default formatter:
-    def formatter(self,i):
-        m = i.mnemonic
-        o = ','.join(map(str,i.operands))
-        return '%s %s'%(m,o)
+    @staticmethod
+    def formatter(i,toks=False):
+        t = (Token.Mnemonic,i.mnemonic)
+        t+= [(Token.Literal,op) for op in map(str,i.operands[0:1])]
+        t+= [(Token.Literal,', '+op) for op in map(str,i.operands[1:])]
+        return t if toks else highlight(t)
 
     def __str__(self):
         return self.formatter(i=self)
 
+    def toks(self):
+        return self.formatter(i=self,toks=True)
+
     def __getstate__(self):
         return (self.bytes,
                 self.type,
@@ -470,16 +476,17 @@ def getparts(self,i):
             fmts = self.formats.get(i.spec.hook.func_name,self.default)
         return fmts
 
-    def __call__(self,i):
+    def __call__(self,i,toks=False):
         s=[]
         for f in self.getparts(i):
             if hasattr(f,'format'):
-                # It is a string
-                s.append(f.format(i=i))
+                t = f.format(i=i)
             else:
-                # It is a function
-                s.append(f(i))
-        return ''.join(s)
+                t = f(i)
+            if isinstance(t,str):
+                t = [(Token.Literal,t)]
+            s.extend(t)
+        return s if toks else highlight(s)
 
 
 # ispec format parser:

amoco/arch/sparc/asm.py

@@ -551,18 +551,18 @@ def i_flush(ins,fmap):raise NotImplementedError
 
 @__pcnpc
 def i_FPop1(ins,fmap):
-    raise InstructionError
+    raise NotImplementedError
 @__pcnpc
 def i_FPop2(ins,fmap):
-    raise InstructionError
+    raise NotImplementedError
 
 @__pcnpc
 def i_CPop1(ins,fmap):
-    raise InstructionError
+    raise NotImplementedError
 @__pcnpc
 def i_CPop2(ins,fmap):
-    raise InstructionError
+    raise NotImplementedError
 
 @__pcnpc
 def i_unimp(ins,fmap):
-    raise InstructionError
+    raise NotImplementedError

amoco/arch/sparc/formats.py

@@ -204,7 +204,7 @@ def label(i):
 
 SPARC_V8_full = Formatter(SPARC_V8_full_formats)
 
-def SPARC_V8_synthetic(null,i):
+def SPARC_V8_synthetic(null,i,toks=False):
     s = SPARC_V8_full(i)
     return SPARC_Synthetic_renaming(s, i)
 

amoco/arch/x64/spec_ia32e.py

@@ -136,6 +136,9 @@ def ia32_strings(obj):
 # imm8:
 @ispec_ia32("16>[ {6a} ib(8) ]", mnemonic = "PUSH",   type=type_data_processing)
 @ispec_ia32("16>[ {cd} ib(8) ]", mnemonic = "INT",    type=type_control_flow)
+def ia32_imm8(obj,ib):
+    obj.operands = [env.cst(ib,8)]
+
 @ispec_ia32("16>[ {eb} ib(8) ]", mnemonic = "JMP",    type=type_control_flow)
 @ispec_ia32("16>[ {e2} ib(8) ]", mnemonic = "LOOP",   type=type_control_flow)
 @ispec_ia32("16>[ {e1} ib(8) ]", mnemonic = "LOOPE",  type=type_control_flow)
@@ -285,7 +288,8 @@ def ia32_imm_rel(obj,cc,cb):
 def ia32_imm_rel(obj,cc,data):
     obj.cond = CONDITION_CODES[cc]
     size = obj.misc['opdsz'] or 32
-    if size==16: raise InstructionError(obj)
+    if size==16 or data.size<size:
+        raise InstructionError(obj)
     imm = data[0:size]
     op1 = env.cst(imm.int(-1),size)
     op1.sf = True

amoco/arch/x86/cpu_x86.py

@@ -10,10 +10,7 @@
 
 from amoco.arch.x86.formats import *
 
-instruction.set_formatter(IA32_Intel)
-
 from amoco.arch.x86 import spec_ia32
-
 disassemble = disassembler([spec_ia32])
 disassemble.maxlen = 15
 
@@ -24,7 +21,10 @@ def configure(**kargs):
     from amoco.config import get_module_conf
     conf = get_module_conf('x86')
     conf.update(kargs)
-    if conf['highlight']:
-        instruction.set_formatter(IA32_Intel_highlighted)
+    # asm format:
+    if conf['format'] in ('AT&T','at&t','ATT','att'):
+        instruction.set_formatter(IA32_ATT)
+    else:
+        instruction.set_formatter(IA32_Intel)
 
 configure()

amoco/arch/x86/env.py

@@ -21,6 +21,10 @@
 eip    = reg('eip',32)     # instruction pointer in 32 bit mode
 eflags = reg('eflags',32)
 
+is_reg_pc(eip)
+is_reg_flags(eflags)
+is_reg_stack(esp)
+
 ax = slc(eax,0,16,'ax')
 bx = slc(ebx,0,16,'bx')
 cx = slc(ecx,0,16,'cx')
@@ -45,25 +49,27 @@
 af = slc(eflags,4,1,'af')   # aux carry flag
 zf = slc(eflags,6,1,'zf')   # zero flag
 sf = slc(eflags,7,1,'sf')   # sign flag
-tf = slc(eflags,8,1,'sf')   # trap flag
+tf = slc(eflags,8,1,'tf')   # trap flag
 df = slc(eflags,10,1,'df')  # direction flag
 of = slc(eflags,11,1,'of')  # overflow flag
 
-# segment registers & other mappings:
-cs = reg('cs',16)      # segment selector for the code segment
-ds = reg('ds',16)      # segment selector to a data segment
-ss = reg('ss',16)      # segment selector to the stack segment
-es = reg('es',16)      # (data)
-fs = reg('fs',16)      # (data)
-gs = reg('gs',16)      # (data)
+with is_reg_other:
+    # segment registers & other mappings:
+    cs = reg('cs',16)      # segment selector for the code segment
+    ds = reg('ds',16)      # segment selector to a data segment
+    ss = reg('ss',16)      # segment selector to the stack segment
+    es = reg('es',16)      # (data)
+    fs = reg('fs',16)      # (data)
+    gs = reg('gs',16)      # (data)
+
+    mmregs = [reg('mm%d'%n,64) for n in range(8)]
+
+    xmmregs = [reg('xmm%d'%n, 128) for n in range(16)]
 
 # fpu registers (80 bits holds double extended floats see Intel Vol1--4.4.2):
 def st(num):
-  return reg('st%d'%num,80)
+  return is_reg_other(reg('st%d'%num,80))
 
-mmregs = [reg('mm%d'%n,64) for n in range(8)]
-
-xmmregs = [reg('xmm%d'%n, 128) for n in range(16)]
 
 # return R/M register (see ModR/M Byte encoding) :
 def getreg(i,size=32):
@@ -74,13 +80,12 @@ def getreg(i,size=32):
             128 : xmmregs[:8],
             }[size][i]
 
-
 # system registers:
 
 # control regs:
 def cr(num):
-    return reg('cr%d'%num,32)
+    return is_reg_other(reg('cr%d'%num,32))
 
 # debug regs:
 def dr(num):
-    return reg('dr%d'%num,32)
+    return is_reg_other(reg('dr%d'%num,32))