Improve copyright detection more

Empty lines can stop a notice continuity.

Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: pombredanne

src/cluecode/copyrights.py

@@ -381,19 +381,30 @@ def get_tokens(numbered_lines, splitter=re.compile(r'[\t =;]+').split):
 
     We perform a simple tokenization on spaces, tabs and some punctuation: =;
     """
+    last_line = ""
     for start_line, line in numbered_lines:
         pos = 0
 
         if TRACE_TOK:
             logger_debug('  get_tokens: bare line: ' + repr(line))
 
-        # if not line.strip():
-        #     yield Token(value="\n", label="EMPTY_LINE", start_line=start_line, pos=pos)
-        #     pos += 1
-        #     continue
+        if not line.strip():
+            stripped = last_line.lower().strip(string.punctuation)
+            if (
+                stripped.startswith("copyright")
+                or stripped.endswith(("by", "copyright","0", "1", "2", "3", "4", "5", "6", "7", "8", "9"))
+            ):
+                continue
+            else:
+                yield Token(value="\n", label="EMPTY_LINE", start_line=start_line, pos=pos)
+                pos += 1
+                last_line = ""
+                continue
 
         line = prepare_text_line(line)
 
+        last_line = line
+
         if TRACE_TOK:
             logger_debug('  get_tokens: preped line: ' + repr(line))
 
@@ -922,6 +933,7 @@ def build_detection_from_node(
     (r'DeclareUnicodeCharacter$', 'JUNK'),
     (r'^Language-Team$', 'JUNK'),
     (r'^Last-Translator$', 'JUNK'),
+    (r'^Translated$', 'JUNK'),
     (r'^OMAP730$', 'JUNK'),
     (r'^Law\.$', 'JUNK'),
     (r'^dylid$', 'JUNK'),
@@ -1601,7 +1613,7 @@ def build_detection_from_node(
     (r'^Branched$', 'NN'),
 
     (r'^Improved$', 'NN'),
-    (r'^Designe[dr]$', 'NN'),
+    (r'^Designed$', 'NN'),
     (r'^Organised$', 'NN'),
     (r'^Re-organised$', 'NN'),
     (r'^Swap$', 'NN'),
@@ -1900,6 +1912,8 @@ def build_detection_from_node(
 
     # Various rare company names/suffix
     (r'^FedICT$', 'COMPANY'),
+    (r'^10gen$', 'COMPANY'),
+
 
     # Division, District
     (r'^(District|Division)\)?[,\.]?$', 'COMP'),

src/licensedcode/data/licenses/ko-man-page.LICENSE

@@ -6,10 +6,6 @@ category: Permissive
 owner: Korean Manpage Project
 homepage_url: https://git.centos.org/rpms/man-pages-ko/blob/c7/f/SOURCES/Man_Page_Copyright
 spdx_license_key: LicenseRef-scancode-ko-man-page
-ignorable_copyrights:
-    - Copyright Man Page
-ignorable_holders:
-    - Man Page
 ---
 
 Translated Copyright Man Page 
@@ -26,4 +22,4 @@ The copyrights of all translated manpages in the Korean Manpage Project are inhe
 
 Exception 
 
-It is possible the documents on this site may contain false information due to  a technical error or mistranslation. However, Korean Manpage Project does not guarantee anything even in this case. If there is false information, please let administrator know or report the error to the appropriate place on the homepage. The documents of this site are subject to change, delete, or move without notice due to error correction of the documents.
+It is possible the documents on this site may contain false information due to  a technical error or mistranslation. However, Korean Manpage Project does not guarantee anything even in this case. If there is false information, please let administrator know or report the error to the appropriate place on the homepage. The documents of this site are subject to change, delete, or move without notice due to error correction of the documents.

src/licensedcode/data/licenses/mediatek-proprietary-2008.LICENSE

@@ -7,9 +7,9 @@ owner: MediaTek
 homepage_url: https://github.com/LIFECorp/mediatek/blob/master/external/mp3dec/mp3dec_exp.h
 spdx_license_key: LicenseRef-scancode-mediatek-proprietary-2008
 ignorable_copyrights:
-    - MediaTek Inc. (c) 2008 BY OPENING
+    - MediaTek Inc. (c) 2008
 ignorable_holders:
-    - MediaTek Inc. BY OPENING
+    - MediaTek Inc.
 ---
 
 This software is protected by Copyright and the information contained
@@ -40,4 +40,4 @@ THE TRANSACTION CONTEMPLATED HEREUNDER SHALL BE CONSTRUED IN ACCORDANCE
 WITH THE LAWS OF THE STATE OF CALIFORNIA, USA, EXCLUDING ITS CONFLICT OF
 LAWS PRINCIPLES.  ANY DISPUTES, CONTROVERSIES OR CLAIMS ARISING THEREOF AND
 RELATED THERETO SHALL BE SETTLED BY ARBITRATION IN SAN FRANCISCO, CA, UNDER
-THE RULES OF THE INTERNATIONAL CHAMBER OF COMMERCE (ICC).
+THE RULES OF THE INTERNATIONAL CHAMBER OF COMMERCE (ICC).

src/licensedcode/data/licenses/qhull.LICENSE

@@ -38,4 +38,4 @@ This software includes Qhull from The Geometry Center. Qhull is copyrighted as n
 
 4. When distributing modified versions of Qhull, or other software products that include Qhull, you must provide notice that the original source code may be obtained as noted above.
 
-5. There is no warranty or other guarantee of fitness for Qhull, it is provided solely "as is". Bug reports or fixes may be sent to qhull_bug@qhull.org; the authors may or may not act on them as they desire.
+5. There is no warranty or other guarantee of fitness for Qhull, it is provided solely "as is". Bug reports or fixes may be sent to qhull_bug@qhull.org; the authors may or may not act on them as they desire.

src/licensedcode/data/licenses/vhfpl-1.1.LICENSE

@@ -10,10 +10,10 @@ notes: The original name of Cenon was vhf interservice GmbH at http://www.vhf-in
 spdx_license_key: LicenseRef-scancode-vhfpl-1.1
 ignorable_copyrights:
     - Copyright (c) 2003 vhf interservice GmbH service@vhf.de
-    - Copyright (c) 2003/2004 vhf interservice GmbH, Im Marxle 3
+    - Copyright (c) 2003/2004 vhf interservice GmbH, Im Marxle
 ignorable_holders:
     - vhf interservice GmbH
-    - vhf interservice GmbH, Im Marxle 3
+    - vhf interservice GmbH, Im Marxle
 ignorable_emails:
     - service@vhf.de
 ---
@@ -120,4 +120,4 @@ INCLUDING THE WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A
 PARTICULAR PURPOSE.
 
 ----------------------------------------------------------------------------
-Copyright (C) 2003 vhf interservice GmbH                      service@vhf.de
+Copyright (C) 2003 vhf interservice GmbH                      service@vhf.de

src/licensedcode/data/rules/sgi-freeb-2.0_3.RULE

@@ -4,12 +4,12 @@ is_license_reference: yes
 ignorable_copyrights:
     - Copyright (c) September 18, 2008 Silicon Graphics, Inc.
 ignorable_holders:
-    - September 18, Silicon Graphics, Inc.
+    - September Silicon Graphics, Inc.
 ignorable_urls:
     - http://oss.sgi.com/projects/FreeB/
 ---
 
 SGI FREE SOFTWARE LICENSE B (Version 2.0, Sept. 18, 2008)
 * Copyright (C) September 18, 2008 Silicon Graphics, Inc. All Rights Reserved.
 * This document is licensed under the SGI Free Software B License Version
-* 2.0. For details, see http://oss.sgi.com/projects/FreeB/ .
+* 2.0. For details, see http://oss.sgi.com/projects/FreeB/ .

src/licensedcode/data/rules/uoi-ncsa_and_other.RULE

@@ -2,10 +2,6 @@
 license_expression: uoi-ncsa AND other-permissive AND other-copyleft
 is_license_notice: yes
 minimum_coverage: 95
-ignorable_copyrights:
-    - Copyright ----- The LLVM project
-ignorable_holders:
-    - The LLVM project
 ignorable_urls:
     - http://www.opensource.org/licenses/UoI-NCSA.php
     - http://www.opensource.org/licenses/mit-license.php
@@ -121,4 +117,4 @@ you or your employer own the rights to a patent and would like to contribute
 code to LLVM that relies on it, we require that the copyright owner sign an
 agreement that allows any other user of LLVM to freely use your patent.  Please
 contact the `oversight group <mailto:llvm-oversight@cs.uiuc.edu>`_ for more
-details.
+details.

tests/packagedcode/data/debian/copyright/debian-slim-2021-04-07/usr/share/doc/libcrypt1/copyright-detailed.expected.yml

@@ -498,10 +498,10 @@ copyright: |
   Copyright Maarten Bosmans FSF
   Copyright Guido U. Draheim, Maarten Bosmans FSF
   Copyright Mike Frysinger FSF
-  Copyright Scott James Remnant, Dan Nicholson GPL
+  Copyright Scott James Remnant, Dan Nicholson
   Copyright Tim Toolan FSF
   Copyright Philip Withnall FSF
-  Copyright Steven G. Johnson, Daniel Richard G. GPL
+  Copyright Steven G. Johnson, Daniel Richard G.
   Copyright Francesco Salvestrini FSF
   Copyright Andrew Collier FSF
   Copyright 2002, 2003, 2004 SuSE Linux AG, Germany

tests/summarycode/data/tallies/end-2-end/bug-1141.expected.json

@@ -68,11 +68,11 @@
         "count": 2
       },
       {
-        "value": "Copyright (c) - Members of the Gmerlin project",
+        "value": "Copyright (c) Members of the Gmerlin project",
         "count": 1
       },
       {
-        "value": "Copyright (c) - Members of the Gmerlin project gmerlin-general@lists.sourceforge.net http://gmerlin.sourceforge.net",
+        "value": "Copyright (c) Members of the Gmerlin project gmerlin-general@lists.sourceforge.net http://gmerlin.sourceforge.net",
         "count": 1
       }
     ],