Merge branch 'develop' into add-license-detection

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

Author: AyanSinhaMahapatra

.gitignore

@@ -95,3 +95,20 @@ tcl
 /00-*.txt
 /z-todo-licenses-*
 
+# Extra ignores from licensedb
+*.pyc
+*.db
+.installed.cfg
+parts
+develop-eggs
+eggs
+downloads
+.settings
+TAGS
+Procfile
+local.cfg
+geckodriver.log
+var
+.metaflow
+selenium
+/dist/

CHANGELOG.rst

@@ -46,14 +46,49 @@ License detection:
   matches in a larger license detecion. This remove a larger number of false
   positive or ambiguous license detections.
 
-
 - The data structure of the JSON output has changed for licenses. We now
   return match details once for each matched license expression rather than
   once for each license in a matched expression. There is a new top-level
   "license_references" attribute that contains the data details for each
   detected license only once. This data can contain the reference license text
   as an option.
 
+- There is a new "scancode-reindex-licenses" command that replace the
+  "scancode --reindex-licenses" command line option which has been
+  removed. This new command supports simpler reindexing using custom
+  license texts and license rules contributed by plugins or stored in an
+  additional directory. The "--reindex-licenses-for-all-languages" CLI option
+  is also moved to the "scancode-reindex-licenses" command as an option
+  "--all-languages".
+
+- We can now detect licenses using custom license texts and license rules.
+  These can be provided as a one off in a directory or packaged as a plugin
+  for consistent reuse and deployment. There is an option "--additional-directory"
+  with the "scancode-reindex-licenses" command and also a new "--only-builtin"
+  option to only use the builtin licenses to build the cache.
+
+- Scancode LICENSE and RULE files now also contain their data as YAML frontmatter,
+  which previously used to be in their respective YAML files. This reduces number of
+  files in those directories, 'rules' and 'licenses' to half. Git line history is
+  preserved for the files.
+
+- A new command line option "--get-license-data" is added to dump license data in
+  JSON, YAML and HTML formats, and also generates a local index and a static website
+  to view the data. This will essentially be an API/way to get scancode license data
+  as opposed to just reading the files. 
+
+
+Package detection:
+~~~~~~~~~~~~~~~~~~~~~
+
+- Code for parsing a Maven POM, npm package.json, freebsd manifest and haxelib
+  JSON have been separated into two functions: one that creates a PackageData
+  object from the parsed Resource, and another that calls the previous function
+  and yields the PackageData. This was done such that we can use the package
+  manifest data parsing code outside of the scancode-toolkit context in other
+  libraries.
+
+
 v31.2.1 - 2022-10-05
 ----------------------------------
 

azure-pipelines.yml

@@ -33,6 +33,7 @@ jobs:
                   --ignore=tests/licensedcode/test_detection_datadriven2.py \
                   --ignore=tests/licensedcode/test_detection_datadriven3.py \
                   --ignore=tests/licensedcode/test_detection_datadriven4.py \
+                  --ignore=tests/licensedcode/test_additional_license.py \
                   tests/licensedcode
 
               license_datadriven1_2: |
@@ -78,6 +79,18 @@ jobs:
                 venv/bin/pytest -n 3 -vvs --test-suite=all \
                   tests/licensedcode/test_zzzz_cache.py
 
+              # this test runs in isolation because it modifies the actual
+              # license index with additional licenses provided by a plugin
+              # and we use the special --test-suite=plugins marker for these
+              # tests
+              additional_license_combined: |
+                venv/bin/pip install tests/licensedcode/data/additional_licenses/additional_plugin_1/
+                venv/bin/pip install tests/licensedcode/data/additional_licenses/additional_plugin_2/
+                venv/bin/scancode-reindex-licenses \
+                  --additional-directory tests/licensedcode/data/additional_licenses/additional_dir/
+                venv/bin/pytest -vvs --test-suite=plugins \
+                  tests/licensedcode/test_additional_license.py
+
     - template: etc/ci/azure-posix.yml
       parameters:
           job_name: ubuntu18_cpython

conftest.py

@@ -38,6 +38,7 @@
 ################################################################################
 SLOW_TEST = 'scanslow'
 VALIDATION_TEST = 'scanvalidate'
+PLUGINS_TEST = 'scanplugins'
 
 
 def pytest_configure(config):
@@ -53,8 +54,14 @@ def pytest_configure(config):
         ': Mark a ScanCode test as a validation test, super slow, long running test.',
     )
 
+    config.addinivalue_line(
+        'markers',
+        PLUGINS_TEST +
+        ': Mark a ScanCode test as a special CI test to tests installing additional plugins.',
+    )
+
 
-TEST_SUITES = 'standard', 'all', 'validate'
+TEST_SUITES = ('standard', 'all', 'validate', 'plugins',)
 
 
 def pytest_addoption(parser):
@@ -72,9 +79,11 @@ def pytest_addoption(parser):
         help='Select which test suite to run: '
              '"standard" runs the standard test suite designed to run reasonably fast. '
              '"all" runs "standard" and "slow" (long running) tests. '
-             '"validate" runs all the tests. '
+             '"validate" runs all the tests, except the "plugins" tests. '
+             '"plugins" runs special plugins tests. Needs extra setup, and is used only in the CI. '
              'Use the @pytest.mark.scanslow marker to mark a test as "slow" test. '
              'Use the @pytest.mark.scanvalidate marker to mark a test as a "validate" test.'
+             'Use the @pytest.mark.scanplugins marker to mark a test as a "plugins" test.'
         )
 
 ################################################################################
@@ -87,13 +96,19 @@ def pytest_collection_modifyitems(config, items):
     test_suite = config.getvalue('test_suite')
     run_everything = test_suite == 'validate'
     run_slow_test = test_suite in ('all', 'validate')
+    run_only_plugins = test_suite == 'plugins'
 
     tests_to_run = []
     tests_to_skip = []
 
     for item in items:
         is_validate = bool(item.get_closest_marker(VALIDATION_TEST))
         is_slow = bool(item.get_closest_marker(SLOW_TEST))
+        is_plugins = bool(item.get_closest_marker(PLUGINS_TEST))
+
+        if is_plugins and not run_only_plugins:
+            tests_to_skip.append(item)
+            continue
 
         if is_validate and not run_everything:
             tests_to_skip.append(item)

docs/source/cli-reference/core-options.rst

@@ -69,22 +69,6 @@ Comparing Progress Message Options
 
 ----
 
-``--reindex-licenses`` Option
------------------------------
-
-    ScanCode maintains a license index to search for and detect licenses. When Scancode is
-    configured for the first time, a license index is built and used in every scan thereafter.
-
-    This ``--reindex-licenses`` option rebuilds the license index. Running a scan with this option
-    displays the following message to the terminal in addition to what it normally shows::
-
-        Checking and rebuilding the license index...
-
-    ..
-        [ToDo] Research and Write Better
-
-----
-
 ``--from-json`` Option
 ----------------------