Improve copyright detection

* Handle better various parens, markup and quotes

Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: pombredanne

src/cluecode/copyrights.py

@@ -2389,7 +2389,7 @@ def build_detection_from_node(
     NAME-EMAIL: {<NAME> <EMAIL>}        #530
 
     # Project Mayo.
-    NAME-YEAR: {<YR-RANGE> <NAME-EMAIL|COMPANY>+ <NNP>?}        #535
+    NAME-YEAR: {<PARENS>? <YR-RANGE> <NAME-EMAIL|COMPANY>+ <NNP>? <PARENS>?}        #535
 
     NAME-YEAR: {<YR-RANGE> <NAME-EMAIL|COMPANY>+ <CC> <YR-RANGE>}        #540
 
@@ -2404,7 +2404,6 @@ def build_detection_from_node(
     # Copyright 2018, OpenCensus Authors
     COPYRIGHT: {<COPY>+ <YR-RANGE> <NNP> <AUTHS>}     #1579991
 
-
     NAME-YEAR: {<YR-RANGE> <NNP>+ <CAPS>?} #5612
 
     #Academy of Motion Picture Arts and Sciences
@@ -2418,6 +2417,8 @@ def build_detection_from_node(
 
     NAME-YEAR: {<YR-RANGE> <NAME>+ <CONTRIBUTORS>?}        #570
 
+    URL: {<PARENS> <URL> <PARENS>}        #5700
+
     #also accept trailing email and URLs
     NAME-YEAR: {<NAME-YEAR> <EMAIL>?<URL>?}        #5701
     NAME-YEAR: {<NAME-YEAR>+}        #5702
@@ -2470,7 +2471,6 @@ def build_detection_from_node(
     # this is catching a wide net by treating any bare URL as a company
     COMPANY: {<NNP>? <URL|URL2>}        #830
 
-
     COMPANY: {<COMPANY> <COMP|COMPANY>}        #840
 
     # the Software and Component Technologies group of Trimble Navigation, Ltd.
@@ -2649,6 +2649,9 @@ def build_detection_from_node(
     # Copyright (c) Ian F. Darwin 1986, 1987, 1989, 1990, 1991, 1992, 1994, 1995.
     COPYRIGHT: {<COPY>+ <NAME|NAME-EMAIL|NAME-YEAR>+ <YR-RANGE>*}        #157999
 
+    # portions copyright The Internet Society, Tom Tromey and Red Hat, Inc.
+    COPYRIGHT: {<PORTIONS>  <COPY>  <NN>  <NAME>}        #157998
+
     COPYRIGHT: {<COPY>+ <CAPS|NNP>+ <CC> <NN> <COPY> <YR-RANGE>?}        #1590
 
     # // (c) (C) → ©
@@ -2737,6 +2740,9 @@ def build_detection_from_node(
     # (c) Copyright 1985-1999 SOME TECHNOLOGY SYSTEMS
     COPYRIGHT2: {<COPY> <COPY> <YR-RANGE> <CAPS> <CAPS> <CAPS>? <CAPS>?} #2271
 
+    # Minpack Copyright Notice (1999) University of Chicago
+    COPYRIGHT: {<COPY>  <NOTICE>  <NAME-YEAR>}  #2273.1
+
     # NAME-COPY is a name with a trailing copyright
     # Daisy (c) 1998
     NAME-COPY: {<NNP> <COPY>} #2272
@@ -3081,6 +3087,12 @@ def build_detection_from_node(
     # Copyright (c) 2014 The Rust Project Developers
     COPYRIGHT: {<COPYRIGHT>  <MAINT> }       #83020
 
+    # copyright its authors
+    COPYRIGHT: {<COPY> <NN> <AUTHS>}       #83030
+
+    # Copyright: 2004-2007 by Internet Systems Consortium, Inc. ("ISC")
+    #            1995-2003 by Internet Software Consortium
+    COPYRIGHT: {<YR-RANGE>  <BY>  <COMPANY> } #1615
 
 #######################################
 # Copyright is held by ....

src/textcode/markup.py

@@ -108,7 +108,22 @@ def demarkup(location):
         yield demarkup_text(line)
 
 
-get_tags_and_entities = re.compile(r'(</?[^\s></]+(?:>|\s)?|&[^\s&]+;|href|[\'"]?\/\>)', re.IGNORECASE).split
+get_tags_and_entities = re.compile(
+    r'('
+    r'</?[^\s></]+(?:>'
+    r'|'
+    r'\s)?'
+    r'|'
+    r'&[^\s&]+;'
+    r'|'
+    r'href'
+    r'|'
+    '[\'"]?\/\>'
+    r'|'
+    r'/>'
+    r')',
+    re.IGNORECASE,
+).split
 
 
 def demarkup_text(text):
@@ -136,7 +151,7 @@ def demarkup_text(text):
     cleaned_append = cleaned.append
     for token in tags_and_ents:
         tlow = token.lower()
-        if tlow.startswith(('<', '/>', '&', 'href',)) and not any(k in tlow for k in kept_tags):
+        if tlow.startswith(('<', '/>', '"/>', "'/>", '&', 'href',)) and not any(k in tlow for k in kept_tags):
             cleaned_append(' ')
         else:
             cleaned_append(token)

tests/cluecode/data/copyright_fossology/testdata87_raw

@@ -6,7 +6,7 @@ It was downloaded from http://ftp.isc.org/isc/dhcp/
 Upstream <s>Author: Internet Systems Consortium (ISC) <dhcp-users@isc.org></s>
 
 <s>Copyright 2004-2007 by Internet Systems Consortium, Inc.</s> ("ISC")
-        1995-2003 by Internet Software Consortium
+        <s>1995-2003 by Internet Software Consortium</s>
 
 License:
 

tests/cluecode/data/copyright_fossology/testdata93_raw

@@ -33,10 +33,10 @@ Copyright:
   CTOCWidget.js:
                  <s>Copyright (c) 2003 The Netscape Corporation</s>.
   xbCollapsibleLists.js: 
-                 <s>Copyright (c) 1997 Michael Bostock</s> (Netscape Communications).
-                 <s>Copyright (c) 2001 Bob Clary</s> (Netscape Communications).
-                 <s>Copyright (c) 2001 Seth Dillingham</s> (Macrobyte Resources).
-                 <s>Copyright (c) 2002 Mark Filanowicz</s> (Amdahl IT Services).
+                 <s>Copyright (c) 1997 Michael Bostock (Netscape Communications)</s>.
+                 <s>Copyright (c) 2001 Bob Clary (Netscape Communications)</s>.
+                 <s>Copyright (c) 2001 Seth Dillingham (Macrobyte Resources)</s>.
+                 <s>Copyright (c) 2002 Mark Filanowicz (Amdahl IT Services)</s>.
 
 Upstream <s>Author:
 

tests/cluecode/data/copyrights/libcdio10-libcdio.label.yml

@@ -20,7 +20,7 @@ copyrights:
   - Copyright (c) 1985, 1987, 1988, 1989, 1990, 1991, 1992, 1993, 1994, 1996, 1997, 1998, 1999,
     2000 Free Software Foundation, Inc.
   - Copyright (c) 2003 Matthias Drochner
-  - Copyright (c) 1998-2001 VideoLAN ( Johan Bilien <jobi@via.ecp.fr> and Gildas Bazin <gbazin@netcourrier.com> )
+  - Copyright (c) 1998-2001 VideoLAN Johan Bilien <jobi@via.ecp.fr> and Gildas Bazin <gbazin@netcourrier.com>
   - Copyright (c) 1992, 1993 Eric Youngdale
   - Copyright (c) 2003, 2004, 2005, 2006, 2007, 2008 Rocky Bernstein and Herbert Valerio Riedel
 holders:

tests/cluecode/data/copyrights/misco2/distributed_3.txt

@@ -3,3 +3,11 @@ what:
   - holders
   - holders_summary
   - authors
+copyrights:
+  - copyright its authors
+holders:
+  - its authors
+holders_summary:
+  - value: its authors
+    count: 1
+

tests/cluecode/data/copyrights/misco2/distributed_3.txt.yml

@@ -0,0 +1,4 @@
+     Copyright © 1998-2009 Bill Spitzak
+     (spitzak@users.sourceforge.net ) and others, including:
+
+

tests/cluecode/data/copyrights/misco2/its-authors.txt.yml

@@ -0,0 +1,12 @@
+what:
+  - copyrights
+  - holders
+  - holders_summary
+  - authors
+copyrights:
+  - Copyright (c) 1998-2009 Bill Spitzak (spitzak@users.sourceforge.net ) and others
+holders:
+  - Bill Spitzak and others
+holders_summary:
+  - value: Bill Spitzak and others
+    count: 1