2to3: Apply unicode fixer.

The unicode fixer strips the u from u'hi' and converts the unicode type
to str. The first won't work for Python 2 and instead we replace the u
prefix with the sixu function borrowed from the six compatibility
package. That function calls the unicode constructor with the
'unicode_escape' encoder so that the many tests using escaped unicode
characters like u'\u0900' will be handled correctly. That makes the
sixu function a bit different from the asunicode function currently in
numpy.compat and also provides a target that can be converted back to
the u prefix when support for Python 3.2 is dropped. Python 3.3
reintroduced the u prefix for compatibility.

The unicode fixer also replaces 'unicode' with 'str' as 'unicode' is no
longer a builtin in Python 3. For code compatibility, 'unicode' is
defined either as 'str' or 'unicode' in numpy.compat so that checks like

    if isinstance(x, unicode):
        ...

will work properly for all python versions.

Closes #3089.

Author: charris

numpydoc/docscrape_sphinx.py

@@ -1,10 +1,16 @@
 from __future__ import division, absolute_import, print_function
 
-import re, inspect, textwrap, pydoc
+import re, inspect, textwrap, pydoc, aya
 import sphinx
 import collections
 from .docscrape import NumpyDocString, FunctionDoc, ClassDoc
 
+if sys.version_info[0] >= 3:
+    sixu = lambda s: s
+else:
+    sixu = lambda s: unicode(s, 'unicode_escape')
+
+
 class SphinxDocString(NumpyDocString):
     def __init__(self, docstring, config={}):
         self.use_plots = config.get('use_plots', False)
@@ -95,11 +101,11 @@ def _str_member_list(self, name):
 
             if others:
                 maxlen_0 = max(3, max([len(x[0]) for x in others]))
-                hdr = u"="*maxlen_0 + u"  " + u"="*10
-                fmt = u'%%%ds  %%s  ' % (maxlen_0,)
+                hdr = sixu("=")*maxlen_0 + sixu("  ") + sixu("=")*10
+                fmt = sixu('%%%ds  %%s  ') % (maxlen_0,)
                 out += ['', hdr]
                 for param, param_type, desc in others:
-                    desc = u" ".join(x.strip() for x in desc).strip()
+                    desc = sixu(" ").join(x.strip() for x in desc).strip()
                     if param_type:
                         desc = "(%s) %s" % (param_type, desc)
                     out += [fmt % (param.strip(), desc)]

numpydoc/numpydoc.py

@@ -17,16 +17,22 @@
 """
 from __future__ import division, absolute_import, print_function
 
+import os, sys, re, pydoc
 import sphinx
+import inspect
 import collections
 
 if sphinx.__version__ < '1.0.1':
     raise RuntimeError("Sphinx 1.0.1 or newer is required")
 
-import os, sys, re, pydoc
 from .docscrape_sphinx import get_doc_object, SphinxDocString
 from sphinx.util.compat import Directive
-import inspect
+
+if sys.version_info[0] >= 3:
+    sixu = lambda s: s
+else:
+    sixu = lambda s: unicode(s, 'unicode_escape')
+
 
 def mangle_docstrings(app, what, name, obj, options, lines,
                       reference_offset=[0]):
@@ -36,32 +42,32 @@ def mangle_docstrings(app, what, name, obj, options, lines,
 
     if what == 'module':
         # Strip top title
-        title_re = re.compile(u'^\\s*[#*=]{4,}\\n[a-z0-9 -]+\\n[#*=]{4,}\\s*',
+        title_re = re.compile(sixu('^\\s*[#*=]{4,}\\n[a-z0-9 -]+\\n[#*=]{4,}\\s*'),
                               re.I|re.S)
-        lines[:] = title_re.sub(u'', u"\n".join(lines)).split(u"\n")
+        lines[:] = title_re.sub(sixu(''), sixu("\n").join(lines)).split(sixu("\n"))
     else:
-        doc = get_doc_object(obj, what, u"\n".join(lines), config=cfg)
+        doc = get_doc_object(obj, what, sixu("\n").join(lines), config=cfg)
         if sys.version_info[0] >= 3:
             doc = str(doc)
         else:
             doc = str(doc).decode('utf-8')
-        lines[:] = doc.split(u"\n")
+        lines[:] = doc.split(sixu("\n"))
 
     if app.config.numpydoc_edit_link and hasattr(obj, '__name__') and \
            obj.__name__:
         if hasattr(obj, '__module__'):
-            v = dict(full_name=u"%s.%s" % (obj.__module__, obj.__name__))
+            v = dict(full_name=sixu("%s.%s") % (obj.__module__, obj.__name__))
         else:
             v = dict(full_name=obj.__name__)
-        lines += [u'', u'.. htmlonly::', u'']
-        lines += [u'    %s' % x for x in
+        lines += [sixu(''), sixu('.. htmlonly::'), sixu('')]
+        lines += [sixu('    %s') % x for x in
                   (app.config.numpydoc_edit_link % v).split("\n")]
 
     # replace reference numbers so that there are no duplicates
     references = []
     for line in lines:
         line = line.strip()
-        m = re.match(u'^.. \\[([a-z0-9_.-])\\]', line, re.I)
+        m = re.match(sixu('^.. \\[([a-z0-9_.-])\\]'), line, re.I)
         if m:
             references.append(m.group(1))
 
@@ -70,14 +76,14 @@ def mangle_docstrings(app, what, name, obj, options, lines,
     if references:
         for i, line in enumerate(lines):
             for r in references:
-                if re.match(u'^\\d+$', r):
-                    new_r = u"R%d" % (reference_offset[0] + int(r))
+                if re.match(sixu('^\\d+$'), r):
+                    new_r = sixu("R%d") % (reference_offset[0] + int(r))
                 else:
-                    new_r = u"%s%d" % (r, reference_offset[0])
-                lines[i] = lines[i].replace(u'[%s]_' % r,
-                                            u'[%s]_' % new_r)
-                lines[i] = lines[i].replace(u'.. [%s]' % r,
-                                            u'.. [%s]' % new_r)
+                    new_r = sixu("%s%d") % (r, reference_offset[0])
+                lines[i] = lines[i].replace(sixu('[%s]_') % r,
+                                            sixu('[%s]_') % new_r)
+                lines[i] = lines[i].replace(sixu('.. [%s]') % r,
+                                            sixu('.. [%s]') % new_r)
 
     reference_offset[0] += len(references)
 
@@ -93,8 +99,8 @@ def mangle_signature(app, what, name, obj, options, sig, retann):
 
     doc = SphinxDocString(pydoc.getdoc(obj))
     if doc['Signature']:
-        sig = re.sub(u"^[^(]*", u"", doc['Signature'])
-        return sig, u''
+        sig = re.sub(sixu("^[^(]*"), sixu(""), doc['Signature'])
+        return sig, sixu('')
 
 def setup(app, get_doc_object_=get_doc_object):
     if not hasattr(app, 'add_config_value'):

numpydoc/tests/test_docscrape.py

@@ -7,6 +7,12 @@
 from numpydoc.docscrape_sphinx import SphinxDocString, SphinxClassDoc
 from nose.tools import *
 
+if sys.version_info[0] >= 3:
+    sixu = lambda s: s
+else:
+    sixu = lambda s: unicode(s, 'unicode_escape')
+
+
 doc_txt = '''\
   numpy.multivariate_normal(mean, cov, shape=None, spam=None)
 
@@ -220,12 +226,12 @@ def test_str():
 
 Raises
 ------
-RuntimeError : 
+RuntimeError :
     Some error
 
 Warns
 -----
-RuntimeWarning : 
+RuntimeWarning :
     Some warning
 
 Warnings
@@ -324,7 +330,7 @@ def test_sphinx_str():
         The drawn samples, arranged according to `shape`.  If the
         shape given is (m,n,...), then the shape of `out` is is
         (m,n,...,N).
-        
+
         In other words, each entry ``out[i,j,...,:]`` is an N-dimensional
         value drawn from the distribution.
 
@@ -333,16 +339,16 @@ def test_sphinx_str():
     **spam** : parrot
 
         A parrot off its mortal coil.
- 
+
 :Raises:
 
-    **RuntimeError** : 
+    **RuntimeError** :
 
         Some error
 
 :Warns:
 
-    **RuntimeWarning** : 
+    **RuntimeWarning** :
 
         Some warning
 
@@ -351,12 +357,12 @@ def test_sphinx_str():
     Certain warnings apply.
 
 .. seealso::
-    
+
     :obj:`some`, :obj:`other`, :obj:`funcs`
-    
+
     :obj:`otherfunc`
         relationship
-    
+
 .. rubric:: Notes
 
 Instead of specifying the full covariance matrix, popular
@@ -403,7 +409,7 @@ def test_sphinx_str():
 [True, True]
 """)
 
-       
+
 doc2 = NumpyDocString("""
     Returns array of indices of the maximum values of along the given axis.
 
@@ -558,9 +564,9 @@ def test_unicode():
     """)
     assert isinstance(doc['Summary'][0], str)
     if sys.version_info[0] >= 3:
-        assert doc['Summary'][0] == u'öäöäöäöäöåååå'
+        assert doc['Summary'][0] == sixu('öäöäöäöäöåååå')
     else:
-        assert doc['Summary'][0] == u'öäöäöäöäöåååå'.encode('utf-8')
+        assert doc['Summary'][0] == sixu('öäöäöäöäöåååå').encode('utf-8')
 
 def test_plot_examples():
     cfg = dict(use_plots=True)
@@ -578,7 +584,7 @@ def test_plot_examples():
     Examples
     --------
     .. plot::
-    
+
        import matplotlib.pyplot as plt
        plt.plot([1,2,3],[4,5,6])
        plt.show()
@@ -695,13 +701,13 @@ def test_class_members_doc():
 
     Methods
     -------
-    a : 
+    a :
 
-    b : 
+    b :
 
-    c : 
+    c :
 
-    .. index:: 
+    .. index::
 
     """)
 
@@ -728,21 +734,20 @@ def test_class_members_doc_sphinx():
     .. rubric:: Attributes
 
     ===  ==========
-      t  (float) Current time.  
-      y  (ndarray) Current variable values.  
+      t  (float) Current time.
+      y  (ndarray) Current variable values.
     ===  ==========
 
     .. rubric:: Methods
 
     ===  ==========
-      a    
-      b    
-      c    
+      a
+      b
+      c
     ===  ==========
 
     """)
 
 if __name__ == "__main__":
     import nose
     nose.run()
-