Merge branch 'develop'

Author: dlenski

setup.py

@@ -1,7 +1,8 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
+from __future__ import print_function
 import sys, os, re, subprocess as sp
-from distutils.core import setup
+from setuptools import setup
 
 if not sys.version_info[0] == 3:
     sys.exit("Python 2.x is not supported; Python 3.x is required.")
@@ -14,19 +15,21 @@
 
 # Fetch version from git tags, and write to version.py.
 # Also, when git is not available (PyPi package), use stored version.py.
-version_py = os.path.join(os.path.dirname(__file__), 'version.py')
+version_py = os.path.join(os.path.dirname(__file__), 'tetherback', 'version.py')
 
 try:
-    version_git = sp.check_output(["git", "describe", "--tags"]).strip().decode('ascii')
-    final, dev, blob = re.match(r'v?((?:\d+\.)*\d+)(?:-(\d+)-(g[a-z0-9]+))?', version_git).groups()
-    version_pep = final+('.dev%s+%s'%(dev,blob) if dev else '')
-except sp.CalledProcessError:
+    version_git = sp.check_output(["git", "describe", "--tags", "--dirty=_dirty"]).strip().decode('ascii')
+    final, dev, blob, dirty = re.match(r'v?((?:\d+\.)*\d+)(?:-(\d+)-(g[a-z0-9]+))?(_dirty)?', version_git).groups()
+    version_pep = final+('.dev%s+%s'%(dev,blob) if dev else '')+(dirty if dirty else '')
+except:
+    d = {}
     with open(version_py, 'r') as fh:
-        version_pep = open(version_py).read().strip().split('=')[-1][1:-1]
+        exec(fh.read(), d)
+        version_pep = d['__version__']
 else:
     with open(version_py, 'w') as fh:
         print("# Do not edit this file, tetherback versioning is governed by git tags", file=fh)
-        print('__version__="%s"\n' % version_pep, file=fh)
+        print('__version__="%s"' % version_pep, file=fh)
 
 ########################################
 

tetherback/README.md

@@ -60,43 +60,55 @@ List of devices attached
   and `data.ext4.win`:
 
     ```bash
-    $ tetherback
-    Device reports TWRP kernel (3.4.0-bricked-hammerhead-twrp-g7b77eb4).
+    Found ADB version 1.0.32
+    Using default transfer method: adb exec-out pipe (--exec-out)
+    Device reports kernel 3.4.0-bricked-hammerhead-twrp-g7b77eb4
+    Device reports TWRP version 3.0.0-0
     Reading partition map for mmcblk0 (29 partitions)...
-      partition map: 100% Time: 0:00:03
-    Saving backup images in twrp-backup-2016-03-17--17-44-04/ ...
+      partition map: 100%
+    Reading partition map for mmcblk0rpmb (0 partitions)...
+      partition map: 100%
+    Saving backup images in ./twrp-backup-2016-07-03--14-53-29/ ...
     Saving partition boot (mmcblk0p19), 22 MiB uncompressed...
-      boot.emmc.win: 100% Time: 0:00:05   3.04 MB/s
+      boot.emmc.win: 100%   4.0 MiB/s  16.3 MiB
     Saving tarball of mmcblk0p25 (mounted at /system), 1024 MiB uncompressed...
-      system.ext4.win: 100% Time: 0:02:16   2.29 MB/s
+      system.ext4.win: 100%   2.5 MiB/s 299.7 MiB
     Saving tarball of mmcblk0p28 (mounted at /data), 13089 MiB uncompressed...
-      data.ext4.win: 100% Time: 0:05:38   2.60 MB/s
+      data.ext4.win: 100%   2.0 MiB/s 804.0 MiB
     ```
 
 * Make a "nandroid"-style backup over ADB. This saves gzipped images
   of the partitions labeled `boot`, `system`, and `userdata` (named
   `<label>.img.gz`):
 
     ```bash
-    $ tetherback -n
-    Device reports TWRP kernel (3.4.0-bricked-hammerhead-twrp-g7b77eb4).
+    $ tetherback -N
+    Found ADB version 1.0.32
+    Using default transfer method: adb exec-out pipe (--exec-out)
+    Device reports kernel 3.4.0-bricked-hammerhead-twrp-g7b77eb4
+    Device reports TWRP version 3.0.0-0
     Reading partition map for mmcblk0 (29 partitions)...
       partition map: 100% Time: 0:00:03
-    Saving backup images in nandroid-backup-2016-03-17--18-15-03/ ...
+    Reading partition map for mmcblk0rpmb (0 partitions)...
+      partition map: 100%
+    Saving backup images in nandroid-backup-2016-07-03--18-15-03/ ...
     Saving partition boot (mmcblk0p19), 22 MiB uncompressed...
-      mmcblk0p19: 100% Time: 0:00:05   3.07 MB/s
+      mmcblk0p19: 100%   3.07 MB/s  16.3 MiB
     Saving partition system (mmcblk0p25), 1024 MiB uncompressed...
-      mmcblk0p25: 100% Time: 0:03:05   1.76 MB/s
+      mmcblk0p25: 100%   1.76 MB/s  343.7 MiB
     Saving partition userdata (mmcblk0p28), 13089 MiB uncompressed...
-      mmcblk0p28: 100% Time: 0:40:04   1.80 MB/s
+      mmcblk0p28: 100%   1.80 MB/s  6.4 GiB
     ```
 
 ### Additional options
 
-* Extra partitions can be included (as raw images) with the `-X`/`--extra`
-  option; for example, `-X modemst1 -X modemst2` to backup the
+* Extra partitions can be included with the `-X`/`--extra` and `--extra-raw`
+  options; for example, `-X modemst1 -X modemst2` to backup the
   [Nexus 5 EFS partitions](http://forum.xda-developers.com/google-nexus-5/development/modem-nexus-5-flashable-modems-efs-t2514095).
 
+    * With `--extra-raw`, the extra partition will *always* be saved as a raw image, rather than as a tarball, even if it is a
+      mountable filesystem and tetherback is run in TWRP backup mode.
+
 * The partition map and backup plan will be printed with
   `-v`/`--verbose` (or use `-0`/`--dry-run` to **only** print it, and
   skip the actual backup). For example, the following partition map

tetherback/adb_stuff.py

@@ -1,38 +1,40 @@
 from sys import stderr
 import time
 
-def really_mount(adb, dev, node, mode='ro'):
-    for opts in (mode, 'remount,'+mode):
-        if adb.check_output(('shell','mount -o %s %s %s 2>/dev/null && echo ok' % (opts, dev, node))).strip():
-            break
+def find_mount(adb, dev, node):
     for l in adb.check_output(('shell','mount')).splitlines():
         f = l.split()
         if not l:
             pass
-        elif len(f)<4:
-            print( "WARNING: don't understand output from mount: %s" % (repr(l)), file=stderr )
         else:
-            mdev, mnode, mtype = (f[0], f[2], f[4]) if (f[1], f[3])==('on','type') else (f[0], f[1], f[2])
-            if mdev==dev or mnode==node:
-                return mtype
+            # Some systems have `mount` output lines that look like:
+            #    /dev/node on /filesystem/mountpoint type fstype (options)
+            # ... while others look like this:
+            #    /dev/node /filesystem/mountpoint fstype options
+            if len(f)<3:
+                print( "WARNING: don't understand output from mount: %s" % (repr(l)), file=stderr )
+            else:
+                mdev, mnode, mtype = (f[0], f[2], f[4]) if len(f)>=5 else f[:3]
+                if mdev==dev or mnode==node:
+                    return (mdev, mnode, mtype)
+    else:
+        return (None, None, None)
+
+def really_mount(adb, dev, node, mode='ro'):
+    for opts in (mode, 'remount,'+mode):
+        if adb.check_output(('shell','mount -o %s %s %s 2>/dev/null && echo ok' % (opts, dev, node))).strip():
+            break
+    mdev, mnode, mtype = find_mount(adb, dev, node)
+    return mtype
 
 def really_umount(adb, dev, node):
     for opts in ('','-f','-l','-r'):
         if adb.check_output(('shell','umount %s 2>/dev/null && echo ok' % dev)).strip():
             break
         if adb.check_output(('shell','umount %s 2>/dev/null && echo ok' % node)).strip():
             break
-    for l in adb.check_output(('shell','mount')).splitlines():
-        f = l.split()
-        if not l:
-            pass
-        elif len(f)<4:
-            print( "WARNING: don't understand output from mount: %s" % (repr(l)), file=stderr )
-        else:
-            mdev, mnode = (f[0], f[2]) if (f[1], f[3])==('on','type') else (f[0], f[1])
-            if mdev==dev or mnode==node:
-                return False
-    return True
+    mdev, mnode, mtype = find_mount(adb, dev, node)
+    return (mtype==None)
 
 def really_forward(adb, port1, port2):
     for port in range(port1, port2):

tetherback/adb_wrapper.py

@@ -1,14 +1,16 @@
+from sys import stderr
 import subprocess as sp
 import re
 
 class AdbWrapper(object):
-    def __init__(self, adbbin='adb', devsel=()):
+    def __init__(self, adbbin='adb', devsel=(), debug=None):
         self.adbbin = adbbin
         self.devsel = tuple(devsel)
+        self.debug = debug
 
     def get_version(self):
          try:
-             s, output = sp.getstatusoutput(self.adbcmd('version'))
+             s, output = sp.getstatusoutput(self.adbcmd(('version',)))
          except FileNotFoundError:
              raise
          except sp.CalledProcessError:
@@ -23,12 +25,18 @@ def get_version(self):
          return adbversions, adbversion
 
     def adbcmd(self, adbargs):
-        return (self.adbbin,) + self.devsel + tuple(adbargs)
+        args = (self.adbbin,) + self.devsel + tuple(adbargs)
+        if self.debug:
+            print("ADB: %s"%repr(args), file=stderr)
+        return args
 
     def check_output(self, adbargs, **kwargs):
         un = kwargs.pop('universal_newlines', True)
         return sp.check_output(self.adbcmd(adbargs), universal_newlines=un, **kwargs)
 
+    def pipe_in(self, adbargs, **kwargs):
+        return sp.Popen(self.adbcmd(adbargs), stdin=sp.PIPE, **kwargs)
+
     def pipe_out(self, adbargs, **kwargs):
         return sp.Popen(self.adbcmd(adbargs), stdout=sp.PIPE, **kwargs)
 

tetherback/tetherback.py

@@ -9,12 +9,13 @@
 import os, sys, datetime, socket, time, argparse, re
 from sys import stderr
 from base64 import standard_b64decode as b64dec
-from progressbar import ProgressBar, Percentage, ETA, FileTransferSpeed, DataSize
+from progressbar import ProgressBar, Percentage, FileTransferSpeed, DataSize
 from tabulate import tabulate
 from enum import Enum
 from hashlib import md5
 from collections import namedtuple, OrderedDict as odict
 
+from .version import __version__
 from .adb_wrapper import AdbWrapper
 from .adb_stuff import *
 
@@ -29,6 +30,8 @@
 
 def parse_args(args=None):
     p = argparse.ArgumentParser(description='''Tool to create TWRP and nandroid-style backups of an Android device running TWRP recovery, using adb-over-USB, without touching the device's internal storage or SD card.''')
+    p.version=__version__
+    p.add_argument('--version', action='version')
     p.add_argument('-s', dest='specific', metavar='DEVICE_ID', default=None, help="Specific device ID (shown by adb devices). Default is sole USB-connected device.")
     p.add_argument('-o', '--output-path', default=".", help="Set optional output path for backup files.")
     p.add_argument('-N', '--nandroid', action='store_true', help="Make nandroid backup; raw images rather than tarballs for /system and /data partitions (default is TWRP backup)")
@@ -55,7 +58,8 @@ def parse_args(args=None):
     g.add_argument('-U', '--no-userdata', dest='userdata', action='store_false', default=True, help="Omit /data partition from backup")
     g.add_argument('-S', '--no-system', dest='system', action='store_false', default=True, help="Omit /system partition from backup")
     g.add_argument('-B', '--no-boot', dest='boot', action='store_false', default=True, help="Omit boot partition from backup")
-    g.add_argument('-X', '--extra', action='append', dest='extra', metavar='NAME', default=[], help="Include extra partition as raw image")
+    g.add_argument('-X', '--extra', action='append', metavar='NAME', default=[], help="Include extra partition (as a tarball if this partition is mountable and TWRP backup type is chosen, otherwise as raw image)")
+    g.add_argument('--extra-raw', action='append', metavar='NAME', default=[], help="Include extra partition (always as raw image)")
     return p, p.parse_args(args)
 
 def check_adb_version(p, adb):
@@ -119,49 +123,59 @@ def sensible_transport(transport, adbversion):
             return adbxp.tcp
     return transport
 
-def build_partmap(adb, mmcblk='mmcblk0', fstab='/etc/fstab'):
+def build_partmap(adb, mmcblks=None, fstab='/etc/fstab'):
     # build partition map
     partmap = odict()
     fstab = fstab_dict(adb, fstab)
-    d = uevent_dict(adb, '/sys/block/%s/uevent' % mmcblk)
-    nparts = int(d['NPARTS'])
-    print("Reading partition map for %s (%d partitions)..." % (mmcblk, nparts), file=stderr)
-    pbar = ProgressBar(max_value=nparts, widgets=['  partition map: ', Percentage(), ' ', ETA()]).start()
-    for ii in range(1, nparts+1):
-        d = uevent_dict(adb, '/sys/block/%s/%sp%d/uevent'%(mmcblk, mmcblk, ii))
-        devname, partn = d['DEVNAME'], int(d['PARTN'])
-        size = int(adb.check_output(('shell','cat /sys/block/%s/%sp%d/size'%(mmcblk, mmcblk, ii))))
-        mountpoint, fstype = fstab.get('/dev/block/%s'%d['DEVNAME'], (None, None))
-
-        # some devices have uppercase names, see #14
-        partname = d['PARTNAME'].lower()
-
-        # some devices apparently use non-standard partition names, though standard mount points, see #18
-        if partname=='system' or mountpoint=='/system':
-            standard = 'system'
-        elif partname=='userdata' or mountpoint=='/data':
-            standard = 'userdata'
-        elif partname=='cache' or mountpoint=='/cache':
-            standard = 'cache'
+    if mmcblks is None:
+        mmcblks = adb.check_output(('shell','cd /sys/block; ls -1d mmcblk*')).splitlines()
+    for mmcblk in mmcblks:
+        d = uevent_dict(adb, '/sys/block/%s/uevent' % mmcblk)
+        nparts = int(d.get('NPARTS',0))
+        print("Reading partition map for %s (%d partitions)..." % (mmcblk, nparts), file=stderr)
+        pbar = ProgressBar(max_value=nparts, widgets=['  partition map: ', Percentage()]).start()
+        for ii in range(1, nparts+1):
+            d = uevent_dict(adb, '/sys/block/%s/%sp%d/uevent'%(mmcblk, mmcblk, ii))
+            devname, partn = d['DEVNAME'], int(d['PARTN'])
+            size = int(adb.check_output(('shell','cat /sys/block/%s/%sp%d/size'%(mmcblk, mmcblk, ii))))
+            mountpoint, fstype = fstab.get('/dev/block/%s'%d['DEVNAME'], (None, None))
+
+            # some devices have uppercase names, see #14
+            partname = d['PARTNAME'].lower()
+
+            # some devices apparently use non-standard partition names, though standard mount points, see #18
+            if partname=='system' or mountpoint=='/system':
+                standard = 'system'
+            elif partname=='userdata' or mountpoint=='/data':
+                standard = 'userdata'
+            elif partname=='cache' or mountpoint=='/cache':
+                standard = 'cache'
+            else:
+                standard = partname
+
+            partmap[standard] = PartInfo(partname, devname, partn, size, mountpoint, fstype)
+            pbar.update(ii)
         else:
-            standard = partname
-
-        partmap[standard] = PartInfo(partname, devname, partn, size, mountpoint, fstype)
-        pbar.update(ii)
+            pbar.finish()
     else:
-        pbar.finish()
         return partmap
 
-def plan_backup(args):
+def plan_backup(args, partmap):
     # Build table of partitions requested for backup
     if args.nandroid:
-        rp = args.extra + [x for x in ('boot','recovery','system','userdata','cache') if getattr(args, x)]
+        rp = args.extra + args.extra_raw + [x for x in ('boot','recovery','system','userdata','cache') if getattr(args, x)]
         plan = odict((p,BackupPlan('%s.emmc.gz'%p, None)) for p in rp)
     else:
-        rp = args.extra + [x for x in ('boot','recovery') if getattr(args, x)]
+        # Figure out which of the --extra partitions can't actually be mounted and exile them to --extra-raw
+        extra_raw = args.extra_raw
+        extra_mount = []
+        for p in args.extra:
+            (extra_mount if p in partmap and partmap[p].fstype else extra_raw).append(p)
+
+        rp = extra_raw + [x for x in ('boot','recovery') if getattr(args, x)]
         plan = odict((p,BackupPlan('%s.emmc.win'%p, None)) for p in rp)
-        mp = [x for x in ('cache','system') if getattr(args, x)]
-        plan.update((p,BackupPlan('%s.ext4.win'%p, '-p')) for p in mp)
+        mp = extra_mount + [x for x in ('cache','system') if getattr(args, x)]
+        plan.update((p,BackupPlan('%s.%s.win'%(p, partmap[p].fstype), '-p')) for p in mp)
 
         if args.userdata:
             data_omit = []
@@ -244,7 +258,7 @@ def backup_partition(adb, pi, bp, transport, verify=True):
         s.connect(('localhost', port))
         block_iter = iter(lambda: s.recv(65536), b'')
 
-    pbwidgets = ['  %s: ' % bp.fn, Percentage(), ' ', ETA(), ' ', FileTransferSpeed(), ' ', DataSize() ]
+    pbwidgets = ['  %s: ' % bp.fn, Percentage(), ' ', FileTransferSpeed(), ' ', DataSize() ]
     pbar = ProgressBar(max_value=pi.size*512, widgets=pbwidgets).start()
 
     with open(bp.fn, 'wb') as out:
@@ -275,7 +289,9 @@ def backup_partition(adb, pi, bp, transport, verify=True):
 
 def main(args=None):
     p, args = parse_args(args)
-    adb = AdbWrapper('adb', ('-s',args.specific) if args.specific else ('-d',))
+    adb = AdbWrapper('adb', ('-s',args.specific) if args.specific else ('-d',), debug=(args.verbose > 1))
+
+    print('%s v%s' % (p.prog, p.version), file=stderr)
 
     # check adb version, and TWRP recovery
     adbversion = check_adb_version(p, adb)
@@ -284,7 +300,7 @@ def main(args=None):
 
     # build partition map and backup plan
     partmap = build_partmap(adb)
-    plan = plan_backup(args)
+    plan = plan_backup(args, partmap)
     missing = set(plan) - set(partmap)
 
     # print partition map and backup explanation

version.py renamed to tetherback/version.py

@@ -1,2 +1,2 @@
 # Do not edit this file, tetherback versioning is governed by git tags
-__version__="0.6.3"
+__version__="0.7"