Improve copyright detection more

pombredanne · pombredanne · commit b4309dde81f7 · 2024-09-11T18:56:57.000+02:00
Handle more edge cases with new preparation

Signed-off-by: Philippe Ombredanne &lt;pombredanne@nexb.com&gt;
diff --git a/src/cluecode/copyrights.py b/src/cluecode/copyrights.py
@@ -2177,6 +2177,10 @@ def build_detection_from_node(
     # exceptions to all CAPS words
     (r'^[A-Z]{3,4}[0-9]{4},?$', 'NN'),
 
+    # exceptions to CAPS used in obfuscated emails like in joe AT foo DOT com
+    (r'^AT$', 'AT'),
+    (r'^DOT$', 'DOT'),
+
     # all CAPS word, at least 1 char long such as MIT, including an optional trailing comma or dot
     (r'^[A-Z0-9]+,?$', 'CAPS'),
 
@@ -2245,6 +2249,12 @@ def build_detection_from_node(
     (r'Keio_University\)?,?$', 'NAME'),
     (r'__MyCompanyName__[\.,]?$', 'NAME'),
 
+    # email in brackets <brett_AT_jdom_DOT_org>
+    #(karl AT indy.rr.com)
+    #<fdlibm-comments AT sun.com>
+    (r'(?i:^[<\(][\w\.\-\+]+at[\w\.\-\+]+(dot)?[\w\.\-\+]+[/)>]$)', 'EMAIL'),
+    
+
     # Code variable names including snake case
     (r'^.*(_.*)+$', 'JUNK'),
 
@@ -2322,6 +2332,9 @@ def build_detection_from_node(
 
     EMAIL: {<EMAIL_START> <CC> <NN>* <EMAIL_END>} # composite_email
 
+    # created by Jason Hunter <jhunter AT jdom DOT org>
+    EMAIL: {<EMAIL_START>  <AT>  <NN|NNP>  <DOT>  <NN|NNP> } # email_start
+
     EMAIL: { <NN>  <CC>  <NN>  <DOT>  <NN> } # foo at bat dot com
 
     # foo@bar.com or baz@bar.com
@@ -2470,6 +2483,9 @@ def build_detection_from_node(
     # Copyright (C) 1995-06 ICP vortex, Achim Leubner
     COPYRIGHT: {<COPY>  <COPY>  <YR-RANGE>  <CAPS>  <NN>  <NNP> <NNP> } #350.2
 
+    # Jason Hunter <jhunter AT jdom DOT org>
+    EMAIL: {<NAME|NNP|NN>  <AT>  <NN|NNP>  <DOT>  <NN|NNP>} #350.3
+
     # Academy of Motion Picture Arts
     NAME: {<NNP|PN>+ <NNP>+}        #351
 
@@ -2563,7 +2579,8 @@ def build_detection_from_node(
     URL: {<PARENS> <URL> <PARENS>}        #5700
 
     #also accept trailing email and URLs
-    NAME-YEAR: {<NAME-YEAR> <EMAIL>?<URL>?}        #5701
+    # and "VAN" e.g. Du: Copyright (c) 2008 Alek Du <alek.du@intel.com>
+    NAME-YEAR: {<NAME-YEAR> <VAN>? <EMAIL>?<URL>?}        #5701
     NAME-YEAR: {<NAME-YEAR>+}        #5702
 
     NAME: {<NNP> <OF> <NNP>}        #580
@@ -4322,7 +4339,7 @@ def remove_code_comment_markers(s):
     """
     Return ``s`` removing code comments such as C and C++ style comment markers and assimilated
 
-    >>> remove_code_comment_markers("\\*#%; /\\/*a*/b/*c\\d#e%f \\*#%; /")
+    >>> remove_code_comment_markers(r"\\*#%; /\\/*a*/b/*c\\d#e%f \\*#%; /")
     'a b c\\d e f'
     """
     return (s
@@ -4445,6 +4462,12 @@ def prepare_text_line(line):
         .replace('§', " ")
         # keep http
         .replace('<http', " http")
+        # placeholders
+        .replace('<insert ', " ")
+        .replace('year>', " ")
+        .replace('<year>', " ")
+        .replace('<name>', " ")
+        
     )
 
     if TRACE_TOK:
diff --git a/src/licensedcode/data/licenses/jdom.LICENSE b/src/licensedcode/data/licenses/jdom.LICENSE
@@ -9,11 +9,11 @@ notes: this based on an Apache 1.1 license
 spdx_license_key: LicenseRef-scancode-jdom
 faq_url: http://www.jdom.org/docs/faq.html#a0030
 ignorable_authors:
-    - Jason Hunter and Brett McLaughlin
-    - the JDOM Project (http://www.jdom.org/)
+  - Jason Hunter <jhunter_AT_jdom_DOT_org> and Brett McLaughlin <brett_AT_jdom_DOT_org>
+  - the JDOM Project (http://www.jdom.org/)
 ignorable_urls:
-    - http://www.jdom.org/
-    - http://www.jdom.org/images/logos
+  - http://www.jdom.org/
+  - http://www.jdom.org/images/logos
 ---
 
 Redistribution and use in source and binary forms, with or without
@@ -61,4 +61,4 @@ This software consists of voluntary contributions made by many
 individuals on behalf of the JDOM Project and was originally
 created by Jason Hunter <jhunter_AT_jdom_DOT_org> and
 Brett McLaughlin <brett_AT_jdom_DOT_org>.  For more information
-on the JDOM Project, please see <http://www.jdom.org/>.
+on the JDOM Project, please see <http://www.jdom.org/>.
diff --git a/src/licensedcode/data/licenses/lucent-pl-1.0.LICENSE b/src/licensedcode/data/licenses/lucent-pl-1.0.LICENSE
@@ -13,6 +13,10 @@ osi_url: http://www.opensource.org/licenses/plan9.php
 other_urls:
     - http://opensource.org/licenses/LPL-1.0
     - https://opensource.org/licenses/LPL-1.0
+ignorable_copyrights:
+  - Copyright (c), <ORGANIZATION> and others
+ignorable_holders:
+  - others
 ---
 
 Lucent Public License Version 1.0
@@ -231,4 +235,4 @@ This Agreement is governed by the laws of the State of <STATE> and the
 intellectual property laws of the United States of America. No party to this
 Agreement will bring a legal action under this Agreement more than one year
 after the cause of action arose. Each party waives its rights to a jury trial in
-any resulting litigation.
+any resulting litigation.
diff --git a/src/licensedcode/data/rules/apache-2.0_1297.RULE b/src/licensedcode/data/rules/apache-2.0_1297.RULE
@@ -3,6 +3,10 @@ license_expression: apache-2.0
 is_license_notice: yes
 ignorable_urls:
     - http://www.apache.org/licenses/LICENSE-2.0
+ignorable_copyrights:
+  - Copyright (c), Oracle and/or its affiliates
+ignorable_holders:
+  - Oracle and/or its affiliates
 ---
 
 The Apache Software License, Version 2.0
@@ -16,4 +20,4 @@ on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
 express or implied.  See the License for the specific language governing
 permissions and  limitations under the License.
 
-Apache License Version 2.0, January 2004
+Apache License Version 2.0, January 2004
diff --git a/src/licensedcode/data/rules/gpl-1.0-plus_118.RULE b/src/licensedcode/data/rules/gpl-1.0-plus_118.RULE
@@ -2,12 +2,8 @@
 license_expression: gpl-1.0-plus
 is_license_tag: yes
 relevance: 100
-ignorable_copyrights:
-    - copyright GNU GENERAL PUBLIC LICENSE a href http://www.gnu.org/copyleft/gpl
-ignorable_holders:
-    - GNU GENERAL PUBLIC LICENSE
 ignorable_urls:
     - http://www.gnu.org/copyleft/gpl
 ---
 
-copyright: GNU GENERAL PUBLIC LICENSE | | <a href="http://www.gnu.org/copyleft/gpl.
+copyright: GNU GENERAL PUBLIC LICENSE | | <a href="http://www.gnu.org/copyleft/gpl.
diff --git a/src/licensedcode/data/rules/gpl-1.0-plus_119.RULE b/src/licensedcode/data/rules/gpl-1.0-plus_119.RULE
@@ -2,12 +2,8 @@
 license_expression: gpl-1.0-plus
 is_license_tag: yes
 relevance: 100
-ignorable_copyrights:
-    - copyright GNU GENERAL PUBLIC LICENCE a href http://www.gnu.org/copyleft/gpl
-ignorable_holders:
-    - GNU GENERAL PUBLIC LICENCE
 ignorable_urls:
     - http://www.gnu.org/copyleft/gpl
 ---
 
-copyright: GNU GENERAL PUBLIC LICENCE | | <a href="http://www.gnu.org/copyleft/gpl.
+copyright: GNU GENERAL PUBLIC LICENCE | | <a href="http://www.gnu.org/copyleft/gpl.
diff --git a/src/licensedcode/data/rules/gpl-1.0-plus_487.RULE b/src/licensedcode/data/rules/gpl-1.0-plus_487.RULE
@@ -2,12 +2,8 @@
 license_expression: gpl-1.0-plus
 is_license_tag: yes
 relevance: 100
-ignorable_copyrights:
-    - copyright GNU GENERAL PUBLIC LICENSE a href https://www.gnu.org/copyleft/gpl
-ignorable_holders:
-    - GNU GENERAL PUBLIC LICENSE
 ignorable_urls:
     - https://www.gnu.org/copyleft/gpl
 ---
 
-copyright: GNU GENERAL PUBLIC LICENSE | | <a href="https://www.gnu.org/copyleft/gpl.
+copyright: GNU GENERAL PUBLIC LICENSE | | <a href="https://www.gnu.org/copyleft/gpl.
diff --git a/src/licensedcode/data/rules/gpl-1.0-plus_524.RULE b/src/licensedcode/data/rules/gpl-1.0-plus_524.RULE
@@ -2,12 +2,8 @@
 license_expression: gpl-1.0-plus
 is_license_tag: yes
 relevance: 100
-ignorable_copyrights:
-    - copyright GNU GENERAL PUBLIC LICENCE a href https://www.gnu.org/copyleft/gpl
-ignorable_holders:
-    - GNU GENERAL PUBLIC LICENCE
 ignorable_urls:
     - https://www.gnu.org/copyleft/gpl
 ---
 
-copyright: GNU GENERAL PUBLIC LICENCE | | <a href="https://www.gnu.org/copyleft/gpl.
+copyright: GNU GENERAL PUBLIC LICENCE | | <a href="https://www.gnu.org/copyleft/gpl.
diff --git a/src/licensedcode/data/rules/jdom_3.RULE b/src/licensedcode/data/rules/jdom_3.RULE
@@ -7,7 +7,7 @@ ignorable_copyrights:
 ignorable_holders:
     - Jason Hunter & Brett McLaughlin
 ignorable_authors:
-    - Jason Hunter and Brett McLaughlin
+    - Jason Hunter jhunter AT jdom DOT org and Brett McLaughlin brett AT jdom DOT org
     - the JDOM Project (http://www.jdom.org/)
 ignorable_urls:
     - http://www.jdom.org/
@@ -64,4 +64,4 @@ JdomLicense 1.1.txt
  individuals on behalf of the JDOM Project and was originally 
  created by Jason Hunter <jhunter AT jdom DOT org> and
  Brett McLaughlin <brett AT jdom DOT org>.  For more information on
- the JDOM Project, please see <http://www.jdom.org/>.
+ the JDOM Project, please see <http://www.jdom.org/>.
diff --git a/src/licensedcode/data/rules/mpeg-iso_2.RULE b/src/licensedcode/data/rules/mpeg-iso_2.RULE
@@ -5,6 +5,10 @@ relevance: 95
 minimum_coverage: 95
 notes: Seen in  https://sourceforge.net/p/opencore-amr/code/ci/master/tree/opencore/NOTICE The
     site for this code at https://portal.3gpp.org/desktopmodules/Specifications/SpecificationDetails.aspx?specificationId=1421
+ignorable_copyrights:
+  - (c), 3GPP Organizational Partners
+ignorable_holders:
+  - 3GPP Organizational Partners
 ignorable_urls:
     - http://www.3gpp.org/
 ---
@@ -17,4 +21,4 @@ Portions of this file are derived from the following 3GPP standard:
  
 (C) , 3GPP Organizational Partners (ARIB, ATIS, CCSA, ETSI, TTA, TTC)
 Permission to distribute, modify and use this file under the standard license
-terms listed above has been obtained from the copyright holder.
+terms listed above has been obtained from the copyright holder.
diff --git a/src/licensedcode/data/rules/proprietary-license_526.RULE b/src/licensedcode/data/rules/proprietary-license_526.RULE
@@ -4,7 +4,7 @@ is_license_text: yes
 relevance: 100
 minimum_coverage: 80
 ignorable_copyrights:
-    - Copyright (c) , Inc.
+    - Copyright (c), Inc.
 ignorable_holders:
     - Inc.
 ---
@@ -167,4 +167,4 @@ jurisdiction of the English courts.
 
     Upon termination of this Agreement, the licenses granted hereunder will
 terminate and Sections 5, 6, 7, 8, 9, 10 and 11 shall survive any termination
-of this Agreement.
+of this Agreement.
diff --git a/src/licensedcode/data/rules/w3c-software-19980720_21.RULE b/src/licensedcode/data/rules/w3c-software-19980720_21.RULE
@@ -2,8 +2,8 @@
 license_expression: w3c-software-19980720
 is_license_text: yes
 ignorable_copyrights:
-    - Copyright (c) 1994-2002 http://www.w3.org/> World Wide Web Consortium
-    - Copyright (c) 2002 http://www.w3.org/> World Wide Web Consortium
+    - Copyright (c) 1994-2002 World Wide Web Consortium
+    - Copyright (c) 2002 World Wide Web Consortium
 ignorable_holders:
     - World Wide Web Consortium
 ignorable_urls:
@@ -69,4 +69,4 @@ or publicity pertaining to the software without specific, written prior
 permission. Title to copyright in this software and any associated
 documentation will at all times remain with copyright holders.</p>
 </body>
-</html>
+</html>
diff --git a/src/licensedcode/data/rules/w3c-software-19980720_6.RULE b/src/licensedcode/data/rules/w3c-software-19980720_6.RULE
@@ -2,7 +2,7 @@
 license_expression: w3c-software-19980720
 is_license_text: yes
 ignorable_copyrights:
-    - Copyright (c) http://www.w3.org/> World Wide Web Consortium
+    - Copyright (c) World Wide Web Consortium
 ignorable_holders:
     - World Wide Web Consortium
 ignorable_urls:
@@ -58,4 +58,4 @@ This W3C work (including software, documents, or other related items) is
       advertising or publicity pertaining to the software without specific,
       written prior permission. Title to copyright in this software and any
       associated documentation will at all times remain with copyright
-      holders.
+      holders.
diff --git a/src/textcode/markup.py b/src/textcode/markup.py
@@ -147,7 +147,7 @@ def starts_or_ends_with_digit(s):
     True
     """
     # first and last character (works even if the string is empty)
-    return s[:1].isdigit() or s[-1:].isdigit()
+    return s and (s[:1].isdigit() or s[-1:].isdigit())
 
 
 def get_demarkuped_text(text, splitter, keeper):
@@ -1914,6 +1914,11 @@ def strip_known_markup_from_text(text):
         # not XML/HTML
         "<year>",
         "<name>",
+        "</name>",
+        "<url>",
+        "</url>",
+        "<date-of-document>",
+
         # common XML namespaces
         "http://www.w3.org/1998/math/mathml",
         "http://www.w3.org/1999/xhtml",
@@ -1932,6 +1937,7 @@ def strip_known_markup_from_text(text):
     "@end",
     "@group",
     "mailto:",
+    "xmlns",
 )
 
 
diff --git a/tests/cluecode/data/copyrights/misco4/linux8/comp.txt b/tests/cluecode/data/copyrights/misco4/linux8/comp.txt
@@ -0,0 +1,4 @@
+ * Copyright (C) 2011 - 2012 Michal Simek <monstr@monstr.eu>
+ * Copyright (c) 2012 Wind River Systems, Inc.
+ * Copyright (C) 2013 Pengutronix e.K.
+ * Copyright (C) 2013 Xilinx Inc.
diff --git a/tests/cluecode/data/copyrights/misco4/linux8/comp.txt.yml b/tests/cluecode/data/copyrights/misco4/linux8/comp.txt.yml
@@ -0,0 +1,14 @@
+what:
+  - copyrights
+  - holders
+  - authors
+copyrights:
+  - Copyright (c) 2011 - 2012 Michal Simek <monstr@monstr.eu>
+  - Copyright (c) 2012 Wind River Systems, Inc.
+  - Copyright (c) 2013 Pengutronix e.K.
+  - Copyright (c) 2013 Xilinx Inc.
+holders:
+  - Michal Simek
+  - Wind River Systems, Inc.
+  - Pengutronix e.K.
+  - Xilinx Inc.
diff --git a/tests/cluecode/data/copyrights/misco4/linux8/du.txt b/tests/cluecode/data/copyrights/misco4/linux8/du.txt
@@ -0,0 +1,2 @@
+
+ Copyright (c) 2008 Alek Du <alek.du@intel.com>
diff --git a/tests/cluecode/data/copyrights/misco4/linux8/du.txt.yml b/tests/cluecode/data/copyrights/misco4/linux8/du.txt.yml
@@ -0,0 +1,8 @@
+what:
+  - copyrights
+  - holders
+  - authors
+copyrights:
+  - Copyright (c) 2008 Alek Du <alek.du@intel.com>
+holders:
+  - Alek Du
diff --git a/tests/cluecode/data/copyrights/misco4/linux8/insert-year.txt b/tests/cluecode/data/copyrights/misco4/linux8/insert-year.txt
@@ -0,0 +1 @@
+Copyright (c) <insert year> IETF Trust
diff --git a/tests/cluecode/data/copyrights/misco4/linux8/insert-year.txt.yml b/tests/cluecode/data/copyrights/misco4/linux8/insert-year.txt.yml
@@ -0,0 +1,8 @@
+what:
+  - copyrights
+  - holders
+  - authors
+copyrights:
+  - Copyright (c) IETF Trust
+holders:
+  - IETF Trust
diff --git a/tests/cluecode/data/copyrights/misco4/linux8/jdom.txt b/tests/cluecode/data/copyrights/misco4/linux8/jdom.txt
@@ -0,0 +1,2 @@
+created by Jason Hunter <jhunter_AT_jdom_DOT_org> and
+Brett McLaughlin <brett_AT_jdom_DOT_org>. 
diff --git a/tests/cluecode/data/copyrights/misco4/linux8/jdom.txt.yml b/tests/cluecode/data/copyrights/misco4/linux8/jdom.txt.yml
@@ -0,0 +1,6 @@
+what:
+  - copyrights
+  - holders
+  - authors
+authors:
+  - Jason Hunter <jhunter_AT_jdom_DOT_org> and Brett McLaughlin <brett_AT_jdom_DOT_org>
diff --git a/tests/cluecode/data/copyrights/misco4/linux8/jdom2.txt b/tests/cluecode/data/copyrights/misco4/linux8/jdom2.txt
@@ -0,0 +1 @@
+created by Jason Hunter <jhunter AT jdom DOT org> 
diff --git a/tests/cluecode/data/copyrights/misco4/linux8/jdom2.txt.yml b/tests/cluecode/data/copyrights/misco4/linux8/jdom2.txt.yml
@@ -0,0 +1,6 @@
+what:
+  - copyrights
+  - holders
+  - authors
+authors:
+  - Jason Hunter jhunter AT jdom DOT org
diff --git a/tests/cluecode/data/copyrights/misco4/linux8/jdom3.txt b/tests/cluecode/data/copyrights/misco4/linux8/jdom3.txt
@@ -0,0 +1 @@
+created by Jason Hunter jhunter AT jdom DOT org 
diff --git a/tests/cluecode/data/copyrights/misco4/linux8/jdom3.txt.yml b/tests/cluecode/data/copyrights/misco4/linux8/jdom3.txt.yml
@@ -0,0 +1,6 @@
+what:
+  - copyrights
+  - holders
+  - authors
+authors:
+  - Jason Hunter jhunter AT jdom DOT org
diff --git a/tests/cluecode/test_copyrights_basic.py b/tests/cluecode/test_copyrights_basic.py
diff --git a/tests/textcode/data/markup_expected/double_pygment_in_html.html.stripmarkup.json b/tests/textcode/data/markup_expected/double_pygment_in_html.html.stripmarkup.json
diff --git a/tests/textcode/data/markup_expected/java.jsp.stripmarkup.json b/tests/textcode/data/markup_expected/java.jsp.stripmarkup.json
diff --git a/tests/textcode/test_markup.py b/tests/textcode/test_markup.py
