Add an entry point for executing pipeline as a single command (#1256)

Signed-off-by: tdruez <tdruez@nexb.com>

Author: tdruez

.gitignore

@@ -3,6 +3,7 @@ __pycache__/
 *.py[cod]
 
 *.db
+*.sqlite3
 .installed.cfg
 parts
 develop-eggs

CHANGELOG.rst

@@ -25,6 +25,9 @@ v34.6.0 (unreleased)
   Add full dependency tree in the CycloneDX output.
   https://github.com/nexB/scancode.io/issues/1066
 
+- Add a new ``run`` entry point for executing pipeline as a single command.
+  https://github.com/nexB/scancode.io/pull/1256
+
 v34.5.0 (2024-05-22)
 --------------------
 

docs/command-line-interface.rst

@@ -238,6 +238,7 @@ Displays status information about the ``PROJECT`` project.
     The full logs of each pipeline execution are displayed by default.
     This can be disabled providing the ``--verbosity 0`` option.
 
+.. _cli_output:
 
 `$ scanpipe output --project PROJECT --format {json,csv,xlsx,spdx,cyclonedx,attribution}`
 -----------------------------------------------------------------------------------------
@@ -324,3 +325,52 @@ API key.
 Optional arguments:
 
 - ``--no-input`` Does not prompt the user for input of any kind.
+
+.. _cli_run:
+
+`$ run PIPELINE_NAME INPUT_LOCATION`
+------------------------------------
+
+A ``run`` command is available for executing pipelines and printing the results
+without providing any configuration. This can be useful for running a pipeline to get
+the results without the need to persist the data in the database or access the UI to
+review the results.
+
+.. tip:: You can run multiple pipelines by providing their names, comma-separated,
+  such as `pipeline1,pipeline2`.
+
+Optional arguments:
+
+- ``--project PROJECT_NAME``: Provide a project name; otherwise, a random value is
+  generated.
+- ``--format {json,spdx,cyclonedx,attribution}``: Specify the output format.
+  **The default format is JSON**.
+
+For example, running the ``inspect_packages`` pipeline on a manifest file:
+
+.. code-block:: bash
+
+    $ run inspect_packages path/to/package.json > results.json
+
+In the following example, running the ``scan_codebase`` followed by the
+``find_vulnerabilities`` pipelines on a codebase directory:
+
+.. code-block:: bash
+
+    $ run scan_codebase,find_vulnerabilities path/to/codebase/ > results.json
+
+Using a URL as input is also supported:
+
+.. code-block:: bash
+
+    $ run scan_single_package https://url.com/package.zip > results.json
+    $ run analyze_docker_image docker://postgres:16 > results.json
+
+In the last example, the ``--format`` option is used to generate a CycloneDX SBOM
+instead of the default JSON output.
+
+.. code-block:: bash
+
+    $ run scan_codebase codebase/ --format cyclonedx > bom.json
+
+See the :ref:`cli_output` for more information about supported output formats.

docs/faq.rst

@@ -9,10 +9,25 @@ our frequently asked questions.
 How can I run a scan?
 ---------------------
 
-You simply start by creating a :ref:`new project <user_interface_create_new_project>`
+Once you've completed the ScanCode.io app installation,
+you simply start by creating a :ref:`new project <user_interface_create_new_project>`
 and run the appropriate pipeline.
 
-ScanCode.io offers several :ref:`built_in_pipelines` depending on your input, see above.
+ScanCode.io offers several :ref:`built_in_pipelines` depending on your input, see
+the :ref:`faq_which_pipeline` bellow.
+
+As an alternative, I you simply which to run a pipeline without installing ScanCode.io
+you may use the Docker image to run pipelines as a single command:
+
+.. code-block:: bash
+
+  docker run --rm \
+    -v "$(pwd)":/codedrop \
+    ghcr.io/nexb/scancode.io:latest \
+    sh -c "run scan_codebase /codedrop" \
+    > results.json
+
+Refer to the :ref:`cli_run` section for more about this approach.
 
 .. _faq_which_pipeline:
 

scancodeio/__init__.py

@@ -95,3 +95,27 @@ def command_line():
 
     os.environ.setdefault("DJANGO_SETTINGS_MODULE", "scancodeio.settings")
     execute_from_command_line(sys.argv)
+
+
+def combined_run():
+    """
+    Command line entry point for executing pipeline as a single command.
+
+    This function sets up a pre-configured settings context, requiring no additional
+    configuration.
+    It combines the creation, execution, and result retrieval of the project into a
+    single process.
+    """
+    from django.core.checks.security.base import SECRET_KEY_INSECURE_PREFIX
+    from django.core.management import execute_from_command_line
+    from django.core.management.utils import get_random_secret_key
+
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "scancodeio.settings")
+    secret_key = SECRET_KEY_INSECURE_PREFIX + get_random_secret_key()
+    os.environ.setdefault("SECRET_KEY", secret_key)
+    os.environ.setdefault("SCANCODEIO_DB_ENGINE", "django.db.backends.sqlite3")
+    os.environ.setdefault("SCANCODEIO_DB_NAME", "scancodeio.sqlite3")
+    os.environ.setdefault("SCANCODEIO_PROCESSES", "0")  # Disable multiprocessing
+
+    sys.argv.insert(1, "run")
+    execute_from_command_line(sys.argv)

scanpipe/management/commands/__init__.py

@@ -21,6 +21,7 @@
 # Visit https://github.com/nexB/scancode.io for support and download.
 
 import shutil
+import sys
 from pathlib import Path
 
 from django.apps import apps
@@ -45,16 +46,18 @@
 class ProjectCommand(BaseCommand):
     """
     Base class for management commands that take a mandatory --project argument.
-    The project is retrieved from the database and stored on the intance as
+    The project is retrieved from the database and stored on the instance as
     `self.project`.
     """
 
     project = None
+    verbosity = 1
 
     def add_arguments(self, parser):
         parser.add_argument("--project", required=True, help="Project name.")
 
     def handle(self, *args, **options):
+        self.verbosity = options["verbosity"]
         project_name = options["project"]
         try:
             self.project = Project.objects.get(name=project_name)
@@ -237,6 +240,15 @@ def extract_group_from_pipelines(pipelines):
     return pipelines_data
 
 
+def validate_pipeline(pipeline_name):
+    """Raise an error if the ``pipeline_name`` is not available."""
+    if pipeline_name not in scanpipe_app.pipelines:
+        raise CommandError(
+            f"{pipeline_name} is not a valid pipeline. \n"
+            f"Available: {', '.join(scanpipe_app.pipelines.keys())}"
+        )
+
+
 def validate_pipelines(pipelines_data):
     """Raise an error if one of the `pipeline_names` is not available."""
     # Backward compatibility with old pipeline names.
@@ -246,11 +258,7 @@ def validate_pipelines(pipelines_data):
     }
 
     for pipeline_name in pipelines_data.keys():
-        if pipeline_name not in scanpipe_app.pipelines:
-            raise CommandError(
-                f"{pipeline_name} is not a valid pipeline. \n"
-                f"Available: {', '.join(scanpipe_app.pipelines.keys())}"
-            )
+        validate_pipeline(pipeline_name)
 
     return pipelines_data
 
@@ -316,7 +324,7 @@ def handle_input_files(project, input_files_data, command=None):
             tag=tag,
         )
 
-    if command:
+    if command and command.verbosity > 0:
         msg = f"File{pluralize(copied)} copied to the project inputs directory:"
         command.stdout.write(msg, command.style.SUCCESS)
         msg = "\n".join(["- " + filename for filename in copied])
@@ -333,7 +341,7 @@ def handle_input_urls(project, input_urls, command=None):
     if downloads:
         project.add_downloads(downloads)
         msg = "File(s) downloaded to the project inputs directory:"
-        if command:
+        if command and command.verbosity > 0:
             command.stdout.write(msg, command.style.SUCCESS)
             msg = "\n".join(["- " + downloaded.filename for downloaded in downloads])
             command.stdout.write(msg)
@@ -347,7 +355,7 @@ def handle_input_urls(project, input_urls, command=None):
 def handle_copy_codebase(project, copy_from, command=None):
     """Copy `codebase_path` tree to the project's `codebase` directory."""
     project_codebase = project.codebase_path
-    if command:
+    if command and command.verbosity > 0:
         msg = f"{copy_from} content copied in {project_codebase}"
         command.stdout.write(msg, command.style.SUCCESS)
     shutil.copytree(src=copy_from, dst=project_codebase, dirs_exist_ok=True)
@@ -371,7 +379,8 @@ def add_project_inputs(
         handle_copy_codebase(project=project, copy_from=copy_from, command=command)
 
 
-def execute_project(project, run_async=False, command=None):
+def execute_project(project, run_async=False, command=None):  # noqa: C901
+    verbosity = getattr(command, "verbosity", 1) if command else 0
     run = project.get_next_run()
 
     if not run:
@@ -383,29 +392,31 @@ def execute_project(project, run_async=False, command=None):
             raise CommandError(msg)
 
         run.start()
-        if command:
+        if verbosity > 0:
             msg = f"{run.pipeline_name} added to the tasks queue for execution."
             command.stdout.write(msg, command.style.SUCCESS)
-    else:
+        sys.exit(0)
+
+    if verbosity > 0:
         command.stdout.write(f"Start the {run.pipeline_name} pipeline execution...")
 
-        try:
-            tasks.execute_pipeline_task(run.pk)
-        except KeyboardInterrupt:
-            run.set_task_stopped()
-            raise CommandError("Pipeline execution stopped.")
-        except Exception as e:
-            run.set_task_ended(exitcode=1, output=str(e))
-            raise CommandError(e)
-
-        run.refresh_from_db()
-
-        if run.task_succeeded and command:
-            msg = f"{run.pipeline_name} successfully executed on " f"project {project}"
-            command.stdout.write(msg, command.style.SUCCESS)
-        else:
-            msg = f"Error during {run.pipeline_name} execution:\n{run.task_output}"
-            raise CommandError(msg)
+    try:
+        tasks.execute_pipeline_task(run.pk)
+    except KeyboardInterrupt:
+        run.set_task_stopped()
+        raise CommandError("Pipeline execution stopped.")
+    except Exception as e:
+        run.set_task_ended(exitcode=1, output=str(e))
+        raise CommandError(e)
+
+    run.refresh_from_db()
+
+    if not run.task_succeeded:
+        msg = f"Error during {run.pipeline_name} execution:\n{run.task_output}"
+        raise CommandError(msg)
+    elif verbosity > 0:
+        msg = f"{run.pipeline_name} successfully executed on " f"project {project}"
+        command.stdout.write(msg, command.style.SUCCESS)
 
 
 def create_project(
@@ -419,6 +430,8 @@ def create_project(
     run_async=False,
     command=None,
 ):
+    verbosity = getattr(command, "verbosity", 1)
+
     if execute and not pipelines:
         raise CommandError("The execute argument requires one or more pipelines.")
 
@@ -440,7 +453,7 @@ def create_project(
     if command:
         command.project = project
 
-    if command:
+    if command and verbosity > 0:
         msg = f"Project {name} created with work directory {project.work_directory}"
         command.stdout.write(msg, command.style.SUCCESS)
 

scanpipe/management/commands/add-pipeline.py

@@ -49,8 +49,10 @@ def handle(self, *pipelines, **options):
             self.project.add_pipeline(pipeline_name, selected_groups=selected_groups)
 
         pipeline_names = pipelines_data.keys()
-        msg = (
-            f"Pipeline{pluralize(pipeline_names)} {', '.join(pipeline_names)} "
-            f"added to the project"
-        )
-        self.stdout.write(msg, self.style.SUCCESS)
+
+        if self.verbosity > 0:
+            msg = (
+                f"Pipeline{pluralize(pipeline_names)} {', '.join(pipeline_names)} "
+                f"added to the project"
+            )
+            self.stdout.write(msg, self.style.SUCCESS)

scanpipe/management/commands/archive-project.py

@@ -68,7 +68,8 @@ def handle(self, *inputs, **options):
                 f"Type 'yes' to continue, or 'no' to cancel: "
             )
             if confirm != "yes":
-                self.stdout.write("Archive cancelled.")
+                if self.verbosity > 0:
+                    self.stdout.write("Archive cancelled.")
                 sys.exit(0)
 
         try:
@@ -80,5 +81,6 @@ def handle(self, *inputs, **options):
         except RunInProgressError as error:
             raise CommandError(error)
 
-        msg = f"The {self.project} project has been archived."
-        self.stdout.write(msg, self.style.SUCCESS)
+        if self.verbosity > 0:
+            msg = f"The {self.project} project has been archived."
+            self.stdout.write(msg, self.style.SUCCESS)

scanpipe/management/commands/create-project.py

@@ -29,6 +29,7 @@
 
 class Command(CreateProjectCommandMixin, AddInputCommandMixin, BaseCommand):
     help = "Create a ScanPipe project."
+    verbosity = 1
 
     def add_arguments(self, parser):
         super().add_arguments(parser)
@@ -55,6 +56,7 @@ def add_arguments(self, parser):
         )
 
     def handle(self, *args, **options):
+        self.verbosity = options["verbosity"]
         name = options["name"]
         pipelines = options["pipelines"]
         input_files = options["input_files"]

scanpipe/management/commands/create-user.py

@@ -65,7 +65,7 @@ def handle(self, *args, **options):
         user = self.UserModel._default_manager.create_user(username, password=password)
         token, _ = Token._default_manager.get_or_create(user=user)
 
-        if options["verbosity"] >= 1:
+        if options["verbosity"] > 0:
             msg = f"User {username} created with API key: {token.key}"
             self.stdout.write(msg, self.style.SUCCESS)