Improve copyright detection

Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: pombredanne

src/cluecode/copyrights.py

@@ -215,8 +215,16 @@ class CopyrightDetector(object):
     """
 
     def __init__(self):
-        self.lexer = lex.Lexer(patterns)
-        self.parser = parse.Parser(grammar, trace=TRACE_DEEP, validate=VALIDATE)
+        """
+        Initialize this detector with a lexer and a parser.
+        """
+        self.lexer = lex.Lexer(matchers=PATTERNS)
+        self.parser = parse.Parser(
+            grammar=GRAMMAR,
+            loop=1,
+            trace=TRACE_DEEP,
+            validate=VALIDATE,
+        )
 
     def detect(self,
         numbered_lines,
@@ -591,7 +599,7 @@ def build_detection_from_node(
 _YEAR_THEN_YEAR_SHORT = fr'({_YEAR_OR_YEAR_YEAR_WITH_PUNCT}({_YEAR_SHORT_PUNCT})*)'
 _YEAR_DASH_PRESENT = _YEAR + r'[\-~]? ?[Pp]resent\.?,?'
 
-patterns = [
+PATTERNS = [
     ############################################################################
     # COPYRIGHT
     ############################################################################
@@ -877,6 +885,7 @@ def build_detection_from_node(
     (r'^DISCLAIMS?$', 'JUNK'),
     (r'^SPECIFICALLY$', 'JUNK'),
 
+    (r'^identifying', 'JUNK'),
     (r'^IDENTIFICATION$', 'JUNK'),
     (r'^WARRANTIE?S?$', 'JUNK'),
     (r'^WARRANTS?$', 'JUNK'),
@@ -1267,6 +1276,7 @@ def build_detection_from_node(
     (r'^But$', 'NN'),
     (r'^Builders?\.?$', 'NN'),
     (r'^Cacute$', 'NN'),
+    (r'^CD$', 'JUNK'),
     (r'^Cell.$', 'NN'),
     (r'^Change\.?[lL]og$', 'NN'),
     (r'^CHANGElogger$', 'NN'),
@@ -1304,17 +1314,18 @@ def build_detection_from_node(
     (r'^Education$', 'NN'),
     (r'^Extended', 'NN'),
     (r'^Every$', 'NN'),
-    (r'^Exhibit$', 'NN'),
+    (r'^EXHIBIT$', 'JUNK'),
+    (r'^Exhibit$', 'JUNK'),
     (r'^Digitized', 'NN'),
     (r'^[Ds]istributed?.?$', 'NN'),
+    (r'^Distributions?', 'NN'),
     (r'^Multiply$', 'NN'),
     (r'^Convert$', 'NN'),
     (r'^Compute$', 'NN'),
     (r'^Case$', 'NN'),
     (r'^Hessian$', 'NN'),
     (r'^Include', 'NN'),
     (r'^Downstream', 'NN'),
-    (r'^Distributions?', 'NN'),
     (r'^Volumes?', 'NN'),
     (r'^Manuals?.?', 'NN'),
     (r'^Update.?', 'NN'),
@@ -1435,8 +1446,8 @@ def build_detection_from_node(
     (r'^Packaging$', 'NN'),
     (r'^Patent', 'NN'),
     (r'^Pentium$', 'NN'),
-    (r'^[Pp]ermission', 'NN'),
-    (r'^PERMISSIONS?', 'NN'),
+    (r'^[Pp]ermission', 'JUNK'),
+    (r'^PERMISSIONS?', 'JUNK'),
     (r'^PGP$', 'NN'),
     (r'^Phrase', 'NN'),
     (r'^Plugin', 'NN'),
@@ -1871,15 +1882,16 @@ def build_detection_from_node(
 
     # same for developed, etc...
     (r'^[Cc]oded$', 'AUTH2'),
-    (r'^[Rr]ecoded$', 'AUTH2'),
-    (r'^[Mm]odified$', 'AUTH2'),
-    (r'^[Cc]reated$', 'AUTH2'),
+    (r'^\(?[Rr]ecoded$', 'AUTH2'),
+    (r'^\(?[Mm]odified$', 'AUTH2'),
+    (r'^\(?[Cc]reated$', 'AUTH2'),
     # written is often mispelled
-    (r'^[Ww]ritt?e[dn]$', 'AUTH2'),
+    (r'^\(?[Ww]ritt?e[dn]$', 'AUTH2'),
     # rewritten is often mispelled
-    (r'^[Rr]ewritt?e[dn]$', 'AUTH2'),
-    (r'^[Mm]aintained$', 'AUTH2'),
-    (r'^[Dd]eveloped$', 'AUTH2'),
+    (r'^\(?[Rr]ewritt?e[dn]$', 'AUTH2'),
+    (r'^\(?[Mm]aintained$', 'AUTH2'),
+    (r'^\(?[Dd]eveloped$', 'AUTH2'),
+    (r'^\(?[Au]thored$', 'AUTH2'),
 
     # commiters is interesting, and so a tag of its own
     (r'[Cc]ommitters\.?,?', 'COMMIT'),
@@ -1888,7 +1900,6 @@ def build_detection_from_node(
     (r'^[Aa]dmins?$', 'MAINT'),
     (r'^[Dd]evelopers?\.?$', 'MAINT'),
     (r'^[Mm]aintainers?\.?$', 'MAINT'),
-    (r'^[Cc]o-maintainers?\.?$', 'MAINT'),
 
     ############################################################################
     # Conjunctions and related
@@ -1965,7 +1976,7 @@ def build_detection_from_node(
     ############################################################################
 
     # rare cases of trailing + signon years
-    (r'^20[0-1][0-9]\+$', 'YR-PLUS'),
+    (r'^20[0-2][0-9]\+$', 'YR-PLUS'),
 
     # year or year ranges
     # plain year with various leading and trailing punct
@@ -2003,6 +2014,9 @@ def build_detection_from_node(
     # slash dates as in 08/95
     (r'^(0?[1-9]|1[012])/[6-9][0-9][\.,]?$', 'YR'),
 
+    # ISO Dates: YYYY-MM-DD
+    (fr'^{_YEAR}-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])$', 'YR'),
+
     # weird year
     (r'today.year', 'YR'),
     (r'^\$?LastChangedDate\$?$', 'YR'),
@@ -2162,7 +2176,7 @@ def build_detection_from_node(
 # Comments in the Grammar are lines that start with #
 # End of line commenst are rules descriptions.
 # One rule per line.
-grammar = """
+GRAMMAR = """
 
 #######################################
 # YEARS
@@ -2336,7 +2350,7 @@ def build_detection_from_node(
     NAME: {<NNP|PN>+ <NNP>+}        #351
 
     # Distributed Management Task Force
-    # NAME: {<NN> <NNP>{3}} #881111
+    NAME: {<NN> <NNP>{3}} #881111
 
     # @author <a href="mailto:stephane@hillion.org">Stephane Hillion</a>
     NAME: { <NN>? <NN>? <EMAIL> <NAME> } #351.1
@@ -2407,10 +2421,10 @@ def build_detection_from_node(
     NAME-YEAR: {<YR-RANGE> <NNP>+ <CAPS>?} #5612
 
     #Academy of Motion Picture Arts and Sciences
-    NAME: {<NAME> <CC> <NNP>} # 561
+    NAME: {<NAME> <CC> <NNP>} #561
 
     # Adam Weinberger and the GNOME Foundation
-    NAME: {<CC> <NN> <COMPANY>} # 565
+    NAME: {<CC> <NN> <COMPANY>} #565
 
     # (c) 1991-1992, Thomas G. Lane , Part of the Independent JPEG Group's
     NAME: {<PORTIONS> <OF> <NN> <NAME>+} #566
@@ -2497,8 +2511,10 @@ def build_detection_from_node(
     # <s>Timothy Terriberry</s>, <s>CSIRO</s>, and other contributors
     ANDCO: {<CC> <CAPS|COMPANY|NAME|NAME-EMAIL|NAME-YEAR>+}          #960
 
+    COMPANY: {<COMPANY|NAME|NAME-EMAIL|NAME-YEAR> <ANDCO>+}     #970
+
     # Copyright © 1998-2009 Bill Spitzak (spitzak@users.sourceforge.net ) and others,
-    COMPANY: {<COMPANY|NAME|NAME-EMAIL|NAME-YEAR> <PARENS>? <ANDCO>+}     #970
+    COMPANY: {<COMPANY|NAME|NAME-EMAIL|NAME-YEAR> <PARENS>? <ANDCO>+}     #970.11
 
     # de Nemours and Company
     NAME: {<VAN>? <NNP> <ANDCO>+}                             #980
@@ -2596,7 +2612,7 @@ def build_detection_from_node(
     # Copyright 2015 The Error Prone Authors.
     NAME: {<NN> <NAME> <CONTRIBUTORS|AUTHS>}        #196023
 
-    # Copyright (C) <s>Suresh P <suresh@ippimail.com></s> 
+    # Copyright (C) <s>Suresh P <suresh@ippimail.com></s>
     NAME: {<NNP> <PN> <EMAIL>} #19601.1
 
     # Copyright or Copr. Mines Paristech, France - Mark NOBLE, Alexandrine GESRET
@@ -2611,9 +2627,14 @@ def build_detection_from_node(
     #   Copyright (C) 1998-2001 VideoLAN ( Johan Bilien <jobi@via.ecp.fr> and Gildas Bazin <gbazin@netcourrier.com> )
     NAME: {<PARENS> <NAME> <PARENS>} #19653
 
+    #  by the Initial Developer
+    INITIALDEV:  {<BY>?  <NN>  <NN>  <MAINT>} #19663
+
+    # UNIVERTSITY OF CHICAGO
+    NAME: {<UNI>  <OF>  <CAPS>} #19673
+
 
-#################################    #COPYRIGHT: {<COPY> <COPY> <MIT>}        #1802
-######
+#######################################
 # VARIOUS FORMS OF COPYRIGHT
 #######################################
 
@@ -2649,9 +2670,6 @@ def build_detection_from_node(
     # Copyright (c) Ian F. Darwin 1986, 1987, 1989, 1990, 1991, 1992, 1994, 1995.
     COPYRIGHT: {<COPY>+ <NAME|NAME-EMAIL|NAME-YEAR>+ <YR-RANGE>*}        #157999
 
-    # portions copyright The Internet Society, Tom Tromey and Red Hat, Inc.
-    COPYRIGHT: {<PORTIONS>  <COPY>  <NN>  <NAME>}        #157998
-
     COPYRIGHT: {<COPY>+ <CAPS|NNP>+ <CC> <NN> <COPY> <YR-RANGE>?}        #1590
 
     # // (c) (C) → ©
@@ -2740,8 +2758,8 @@ def build_detection_from_node(
     # (c) Copyright 1985-1999 SOME TECHNOLOGY SYSTEMS
     COPYRIGHT2: {<COPY> <COPY> <YR-RANGE> <CAPS> <CAPS> <CAPS>? <CAPS>?} #2271
 
-    # Minpack Copyright Notice (1999) University of Chicago
-    COPYRIGHT: {<COPY>  <NOTICE>  <NAME-YEAR>}  #2273.1
+    # Copyright Notice (1999) University of Chicago. All rights reserved
+    COPYRIGHT: {<COPY>  <NOTICE>  <NAME-YEAR>  <ALLRIGHTRESERVED>? } #2271.1
 
     # NAME-COPY is a name with a trailing copyright
     # Daisy (c) 1998
@@ -2789,7 +2807,6 @@ def build_detection_from_node(
     # Copyright (C) 2006 XStream committers.
     # Copyright (c) 2019-2021, Open source contributors.
     # Copyright 2007 ZXing authors
-    # Copyright (c) 2002 the Initial Developer
     # Copyright (c) 2024 bgme <i@bgme.me>.
     COPYRIGHT: {<COPY>+  <YR-RANGE>  <NN>+ <CONTRIBUTORS|COMMIT|AUTHS|MAINT>? <EMAIL>? <ALLRIGHTRESERVED>?} #22793.3
 
@@ -2881,9 +2898,6 @@ def build_detection_from_node(
     # maintainer Norbert Tretkowski <nobse@debian.org> 2005-04-16
     AUTHOR: {<BY|MAINT> <NAME-EMAIL> <YR-RANGE>?}  #26382
 
-    # Russ Dill <Russ.Dill@asu.edu> 2001-2003
-    COPYRIGHT: {<NAME-EMAIL> <YR-RANGE>}       #2638
-
     # (C) 2001-2009, <s>Takuo KITAME, Bart Martens, and  Canonical, LTD</s>
     COPYRIGHT: {<COPYRIGHT> <NNP> <COMPANY>}       #26381
 
@@ -3081,9 +3095,6 @@ def build_detection_from_node(
     # Copyright OProfile authors
     COPYRIGHT: {<COPY> <NN>?<NNP>+ <AUTHS>}         #83004
 
-    # (C) Distributed Management Task Force (Distributed is an NN)
-    # COPYRIGHT: {<COPY> <NN> <NAME>}         #83010
-
     # Copyright (c) 2014 The Rust Project Developers
     COPYRIGHT: {<COPYRIGHT>  <MAINT> }       #83020
 
@@ -3092,15 +3103,45 @@ def build_detection_from_node(
 
     # Copyright: 2004-2007 by Internet Systems Consortium, Inc. ("ISC")
     #            1995-2003 by Internet Software Consortium
-    COPYRIGHT: {<YR-RANGE>  <BY>  <COMPANY> } #1615
+    COPYRIGHT: {<COPYRIGHT> <NN> <YR-RANGE>  <BY>  <COMPANY> } #1615
+
+    # Russ Dill <Russ.Dill@asu.edu> 2001-2003
+    # Rewrited by Vladimir Oleynik <dzo@simtreas.ru> (C) 2003
+    COPYRIGHT: {<NAME-EMAIL>  <YR-RANGE>  <AUTH2>  <BY>  <NAME-EMAIL>  <COPY>  <YR-RANGE>} #22793.5
+
+    # portions copyright The Internet Society, Tom Tromey and Red Hat, Inc.
+    COPYRIGHT: {<PORTIONS>  <COPY>  <NN>  <NAME>}        #157998
+
+    # Minpack Copyright Notice (1999) University of Chicago
+    COPYRIGHT: {<COPY>  <NOTICE>  <NAME-YEAR>}  #2273.1
+
+    # Portions created by the Initial Developer are Copyright (C)
+    # the Initial Developer. All Rights Reserved.
+    COPYRIGHT: {<PORTIONS>  <AUTH2>  <INITIALDEV>  <IS>  <COPY|COPYRIGHT2>+  <YR-RANGE>? <INITIALDEV>} #2609.1
+
+    # Portions created by the Initial Developer are Copyright (C) 
+    # the Initial Developer. All Rights Reserved.
+    # and
+    # Portions created by the Initial Developer are Copyright (C) 2002
+    # the Initial Developer. All Rights Reserved.
+    COPYRIGHT: {<COPYRIGHT|COPYRIGHT2>  <INITIALDEV>  <ALLRIGHTRESERVED>?}  #2609.2
+
+    # Copyright (C) the Initial Developer.
+    COPYRIGHT: {<COPY>+  <INITIALDEV>} #35012
+
+    # (C) Distributed Management Task Force (Distributed is an NN)
+    # COPYRIGHT: {<COPY> <NN>? <NAME>}         #83010
+
+    # Gracenote, Inc., copyright © 2000-2008 Gracenote.
+    # Gracenote Software, copyright © 2000-2008 Gracenote.
+    COPYRIGHT: {<COMPANY> <COPY>{1,2} <NAME-YEAR>}        #157999.12
 
 #######################################
 # Copyright is held by ....
 #######################################
     # Copyright is held by ....
     COPYRIGHT: {<COPY> <IS> <HELD> <BY> <NNP|NAME|COMPANYNAME-EMAIL>+ }         #10989898
 
-
 #######################################
 # Authors
 #######################################
@@ -3194,7 +3235,8 @@ def build_detection_from_node(
     # Copyright (c) 2015 Jon Schlinkert, contributors.
     COPYRIGHT: { <COPYRIGHT>  <CONTRIBUTORS>} #420121
 
-
+    # J. Schoenwaelder, Copyright (c) 1999
+    # COPYRIGHT: {<NAME> <COPYRIGHT>} #22793.7
 
 #######################################
 # Last resort catch all ending with ALLRIGHTRESERVED
@@ -3246,6 +3288,19 @@ def refine_copyright(c):
     return c.strip()
 
 
+def remove_dupe_holder(h):
+    """
+    Remove duplicated holders
+    """
+    dupes_holders = {
+        "the Initial Developer the Initial Developer": "the Initial Developer",
+    }
+    for src, tgt in dupes_holders.items():
+        if src in h:
+            h = h.replace(src, tgt)
+    return h
+
+
 def refine_holder(h):
     """
     Refine a detected holder.
@@ -3278,6 +3333,7 @@ def refine_holder(h):
     h = h.strip('+- ')
     h = strip_trailing_period(h)
     h = h.strip('+- ')
+    h = remove_dupe_holder(h)
     h = ' '.join(h.split())
     if h and h.lower() not in HOLDERS_JUNK:
         return h
@@ -3694,7 +3750,9 @@ def remove_some_extra_words_and_punct(c):
     c = c.replace(".net'", ".net")
     c = c.replace("mailto:", "")
     c = c.replace("@see", "")
-    return c
+    if c.endswith('as represented by'):
+        c, _, _ = c.partition('as represented by')
+    return c.strip()
 
 
 def strip_prefixes(s, prefixes=()):

src/licensedcode/data/licenses/cups.LICENSE

@@ -16,7 +16,6 @@ ignorable_holders:
     - Jelmer Vernooij
 ignorable_authors:
     - Apple Inc.
-    - permission from Apple Inc.
 ignorable_urls:
     - http://www.cups.org/
 ---
@@ -74,4 +73,4 @@ Altered source versions must be plainly marked as such, and must not be misrepre
 This notice may not be removed or altered from any source distribution.
 Trademarks
 
-CUPS and the CUPS logo (the "CUPS Marks") are trademarks of Apple Inc. Apple grants you a non-exclusive and non-transferable right to use the CUPS Marks in any direct port or binary distribution incorporating CUPS software and in any promotional material therefor. You agree that your products will meet the highest levels of quality and integrity for similar goods, not be unlawful, and be developed, manufactured, and distributed in compliance with this license. You will not interfere with Apple's rights in the CUPS Marks, and all use of the CUPS Marks shall inure to the benefit of Apple. This license does not apply to use of the CUPS Marks in a derivative products, which requires prior written permission from Apple Inc.
+CUPS and the CUPS logo (the "CUPS Marks") are trademarks of Apple Inc. Apple grants you a non-exclusive and non-transferable right to use the CUPS Marks in any direct port or binary distribution incorporating CUPS software and in any promotional material therefor. You agree that your products will meet the highest levels of quality and integrity for similar goods, not be unlawful, and be developed, manufactured, and distributed in compliance with this license. You will not interfere with Apple's rights in the CUPS Marks, and all use of the CUPS Marks shall inure to the benefit of Apple. This license does not apply to use of the CUPS Marks in a derivative products, which requires prior written permission from Apple Inc.

src/licensedcode/data/licenses/ms-directx-sdk-eula.LICENSE

@@ -7,8 +7,6 @@ owner: Microsoft
 spdx_license_key: LicenseRef-scancode-ms-directx-sdk-eula
 text_urls:
     - ftp://ftp.physik.hu-berlin.de/pub/useful/dx7asdk/DXF/license/directx%20sdk%20eula.txt
-ignorable_authors:
-    - permission from Microsoft Corporation
 ---
 
 DirectX Software Development Kit END-USER LICENSE AGREEMENT FOR MICROSOFT SOFTWARE 
@@ -74,4 +72,4 @@ If you acquired this product in Canada, this Agreement is governed by the laws o
 
 If this product was acquired outside the United States, then local law may apply.
 
-Should you have any questions concerning this Agreement, or if you desire to contact Microsoft for any reason, please contact the Microsoft subsidiary serving your country, or write: Microsoft Customer Sales and Service/One Microsoft Way/Redmond, WA  98052-6399.
+Should you have any questions concerning this Agreement, or if you desire to contact Microsoft for any reason, please contact the Microsoft subsidiary serving your country, or write: Microsoft Customer Sales and Service/One Microsoft Way/Redmond, WA  98052-6399.

src/licensedcode/data/licenses/mtll.LICENSE

@@ -10,10 +10,6 @@ notes: |
     derived from the Apache Software License v1.1, but it is modified in
     several ways. It is Free, but GPL-incompatible.
 spdx_license_key: MTLL
-ignorable_authors:
-    - at the University of Notre Dame, the Pervasive Technology Labs at Indiana University,
-        and Dresden University of Technology
-    - permission, please contact Indiana University Advanced Research & Technology
 ---
 
 Software License for MTL
@@ -36,4 +32,4 @@ Alternatively, this acknowledgement may appear in the software itself, and where
 
 TUD and IU provide no reassurances that the source code provided does not infringe the patent or any other intellectual property rights of any other entity. TUD and IU disclaim any liability to any recipient for claims brought by any other entity based on infringement of intellectual property rights or otherwise.
 
-LICENSEE UNDERSTANDS THAT SOFTWARE IS PROVIDED "AS IS" FOR WHICH NO WARRANTIES AS TO CAPABILITIES OR ACCURACY ARE MADE. DRESDEN UNIVERSITY OF TECHNOLOGY AND INDIANA UNIVERSITY GIVE NO WARRANTIES AND MAKE NO REPRESENTATION THAT SOFTWARE IS FREE OF INFRINGEMENT OF THIRD PARTY PATENT, COPYRIGHT, OR OTHER PROPRIETARY RIGHTS. DRESDEN UNIVERSITY OF TECHNOLOGY AND INDIANA UNIVERSITY MAKE NO WARRANTIES THAT SOFTWARE IS FREE FROM "BUGS", "VIRUSES", "TROJAN HORSES", "TRAP DOORS", "WORMS", OR OTHER HARMFUL CODE. LICENSEE ASSUMES THE ENTIRE RISK AS TO THE PERFORMANCE OF SOFTWARE AND/OR ASSOCIATED MATERIALS, AND TO THE PERFORMANCE AND VALIDITY OF INFORMATION GENERATED USING SOFTWARE.
+LICENSEE UNDERSTANDS THAT SOFTWARE IS PROVIDED "AS IS" FOR WHICH NO WARRANTIES AS TO CAPABILITIES OR ACCURACY ARE MADE. DRESDEN UNIVERSITY OF TECHNOLOGY AND INDIANA UNIVERSITY GIVE NO WARRANTIES AND MAKE NO REPRESENTATION THAT SOFTWARE IS FREE OF INFRINGEMENT OF THIRD PARTY PATENT, COPYRIGHT, OR OTHER PROPRIETARY RIGHTS. DRESDEN UNIVERSITY OF TECHNOLOGY AND INDIANA UNIVERSITY MAKE NO WARRANTIES THAT SOFTWARE IS FREE FROM "BUGS", "VIRUSES", "TROJAN HORSES", "TRAP DOORS", "WORMS", OR OTHER HARMFUL CODE. LICENSEE ASSUMES THE ENTIRE RISK AS TO THE PERFORMANCE OF SOFTWARE AND/OR ASSOCIATED MATERIALS, AND TO THE PERFORMANCE AND VALIDITY OF INFORMATION GENERATED USING SOFTWARE.