Merge pull request #38 from huggingface/extension

Code moved in Cython, removed speakers handling, integration in spaCy pipeline

Author: thomwolf

.gitignore

@@ -1,22 +1,63 @@
-*.pyc
-*#*
-*.DS_STORE
-*.log
-*Data.fs*
-*flymake*
-dist/*
-*egg*
-urllist*
-build/
-__pycache__/
+# NeuralCoref
+/neuralcoref/train/runs/*
+/models/
+
+# Cython / C extensions
+cythonize.json
+neuralcoref/*.html
+*.cpp
+*.so
+
+# Vim / VSCode / editors
+*.swp
+*.sw*
+Profile.prof
+.vscode
+.sass-cache
+
+# Python
 /.Python
-/bin/
-/include/
-/lib/
-/pip-selfcheck.json
-/runs/*
-test_corefs.txt
-test_mentions.txt
-.cache
-/.vscode/*
-/.vscode
+.python-version
+__pycache__/
+*.py[cod]
+.env/
+.env*
+.~env/
+.venv
+venv/
+.dev
+.denv
+.pypyenv
+
+# Temporary files
+*.~*
+tmp/
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Distribution / packaging
+env/
+build/
+develop-eggs/
+dist/
+eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+.eggs
+MANIFEST
+
+# Windows
+*.bat
+Thumbs.db
+Desktop.ini
+
+# Mac OS X
+*.DS_Store

LICENCE.txt

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2016 Huggingface Inc.
+Copyright (c) 2018 Huggingface Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

MANIFEST

@@ -1,2 +1,3 @@
-include neuralcoref/*
-include neuralcoref/weights/*
+recursive-include include *.h
+include LICENSE
+include README.rst

MANIFEST.in

@@ -0,0 +1,157 @@
+#!/usr/bin/env python
+""" cythonize.py
+
+Cythonize pyx files into C++ files as needed.
+
+Usage: cythonize.py [root]
+
+Checks pyx files to see if they have been changed relative to their
+corresponding C++ files. If they have, then runs cython on these files to
+recreate the C++ files.
+
+Additionally, checks pxd files and setup.py if they have been changed. If
+they have, rebuilds everything.
+
+Change detection based on file hashes stored in JSON format.
+
+For now, this script should be run by developers when changing Cython files
+and the resulting C++ files checked in, so that end-users (and Python-only
+developers) do not get the Cython dependencies.
+
+Based upon:
+
+https://raw.github.com/dagss/private-scipy-refactor/cythonize/cythonize.py
+https://raw.githubusercontent.com/numpy/numpy/master/tools/cythonize.py
+
+Note: this script does not check any of the dependent C++ libraries.
+"""
+from __future__ import print_function
+
+import os
+import sys
+import json
+import hashlib
+import subprocess
+import argparse
+
+
+HASH_FILE = 'cythonize.json'
+
+
+def process_pyx(fromfile, tofile):
+    print('Processing %s' % fromfile)
+    try:
+        from Cython.Compiler.Version import version as cython_version
+        from distutils.version import LooseVersion
+        if LooseVersion(cython_version) < LooseVersion('0.19'):
+            raise Exception('Require Cython >= 0.19')
+
+    except ImportError:
+        pass
+
+    flags = ['--fast-fail']
+    if tofile.endswith('.cpp'):
+        flags += ['--cplus']
+
+    try:
+        try:
+            r = subprocess.call(['cython'] + flags + ['-o', tofile, fromfile],
+                                env=os.environ) # See Issue #791
+            if r != 0:
+                raise Exception('Cython failed')
+        except OSError:
+            # There are ways of installing Cython that don't result in a cython
+            # executable on the path, see gh-2397.
+            r = subprocess.call([sys.executable, '-c',
+                                'import sys; from Cython.Compiler.Main import '
+                                'setuptools_main as main; sys.exit(main())'] + flags +
+                                ['-o', tofile, fromfile])
+            if r != 0:
+                raise Exception('Cython failed')
+    except OSError:
+        raise OSError('Cython needs to be installed')
+
+
+def preserve_cwd(path, func, *args):
+    orig_cwd = os.getcwd()
+    try:
+        os.chdir(path)
+        func(*args)
+    finally:
+        os.chdir(orig_cwd)
+
+
+def load_hashes(filename):
+    try:
+        return json.load(open(filename))
+    except (ValueError, IOError):
+        return {}
+
+
+def save_hashes(hash_db, filename):
+    with open(filename, 'w') as f:
+        f.write(json.dumps(hash_db))
+
+
+def get_hash(path):
+    return hashlib.md5(open(path, 'rb').read()).hexdigest()
+
+
+def hash_changed(base, path, db):
+    full_path = os.path.normpath(os.path.join(base, path))
+    return not get_hash(full_path) == db.get(full_path)
+
+
+def hash_add(base, path, db):
+    full_path = os.path.normpath(os.path.join(base, path))
+    db[full_path] = get_hash(full_path)
+
+
+def process(base, filename, db):
+    root, ext = os.path.splitext(filename)
+    if ext in ['.pyx', '.cpp']:
+        if hash_changed(base, filename, db) or not os.path.isfile(os.path.join(base, root + '.cpp')):
+            preserve_cwd(base, process_pyx, root + '.pyx', root + '.cpp')
+            hash_add(base, root + '.cpp', db)
+            hash_add(base, root + '.pyx', db)
+
+
+def check_changes(root, db):
+    res = False
+    new_db = {}
+
+    setup_filename = 'setup.py'
+    hash_add('.', setup_filename, new_db)
+    if hash_changed('.', setup_filename, db):
+        res = True
+
+    for base, _, files in os.walk(root):
+        for filename in files:
+            if filename.endswith('.pxd'):
+                hash_add(base, filename, new_db)
+                if hash_changed(base, filename, db):
+                    res = True
+
+    if res:
+        db.clear()
+        db.update(new_db)
+    return res
+
+
+def run(root):
+    db = load_hashes(HASH_FILE)
+
+    try:
+        check_changes(root, db)
+        for base, _, files in os.walk(root):
+            for filename in files:
+                process(base, filename, db)
+    finally:
+        save_hashes(db, HASH_FILE)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Cythonize pyx files into C++ files as needed')
+    parser.add_argument('root', help='root directory')
+    args = parser.parse_args()
+    run(args.root)

bin/cythonize.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""Coreference resolution server example.
+A simple server serving the coreference system.
+"""
+from __future__ import unicode_literals
+
+import sys
+import json
+from wsgiref.simple_server import make_server
+import falcon
+import spacy
+
+IS_PYTHON2 = int(sys.version[0]) == 2
+unicode_ = unicode if IS_PYTHON2 else str
+
+class AllResource(object):
+    def __init__(self):
+        self.nlp = spacy.load('en_coref_sm')
+        print("Server loaded")
+        self.response = None
+
+    def on_get(self, req, resp):
+        self.response = {}
+
+        text_param = req.get_param("text")
+        if text_param is not None:
+            text = ",".join(text_param) if isinstance(text_param, list) else text_param
+            text = unicode_(text)
+            doc = self.nlp(text)
+            if doc._.has_coref:
+                mentions = [{'start':    span.start_char,
+                             'end':      span.end_char,
+                             'text':     span.text,
+                             'resolved': span._.coref_main_mention.text
+                            } for span in doc._.coref_mentions]
+                clusters = list(list(span.text for span in cluster)
+                                for cluster in doc._.coref_clusters)
+                resolved = doc._.coref_resolved
+                self.response['mentions'] = mentions
+                self.response['clusters'] = clusters
+                self.response['resolved'] = resolved
+
+        resp.body = json.dumps(self.response)
+        resp.content_type = 'application/json'
+        resp.append_header('Access-Control-Allow-Origin', "*")
+        resp.status = falcon.HTTP_200
+
+if __name__ == '__main__':
+    RESSOURCE = AllResource()
+    APP = falcon.API()
+    APP.add_route('/', RESSOURCE)
+    HTTPD = make_server('0.0.0.0', 8000, APP)
+    HTTPD.serve_forever()

examples/server.py

@@ -1 +1,5 @@
-from .algorithm import Coref
+# coding: utf8
+from __future__ import unicode_literals
+from .neuralcoref import NeuralCoref
+
+__all__ = ['NeuralCoref']