[No Ticket] Tee output (#1546)

* --tee-output for fossa analyze

* tee-output for fossa container analyze.

* fmt-ci

* Update changelog

Author: csasarak

Changelog.md

@@ -1,5 +1,9 @@
 # FOSSA CLI Changelog
 
+## 3.10.9
+
+- CLI Args: Add a `--tee-output` argument to allow uploading results and also printing them to stdout.([#1546](https://github.com/fossas/fossa-cli/pull/1546))
+
 ## 3.10.8
 
 - Custom license scans: Apply `licenseScanPathFilters` to custom license scans ([#1535](https://github.com/fossas/fossa-cli/pull/1535)).

docs/references/subcommands/analyze.md

@@ -42,14 +42,20 @@ The paths and targets filtering options allow you to specify the exact targets w
 | `--without-default-filters`      | Ignore default path filters. See [default path filters](./analyze.md#what-are-the-default-path-filters)                  |
 
 
-### Printing results without uploading to FOSSA
+### Printing FOSSA results
 
 The `--output` flag can be used to print projects and dependency graph information to stdout, rather than uploading to FOSSA
 
 ```sh
 fossa analyze --output
 ```
 
+To print projects and dependency graph information to stdout *in addition* to uploading to FOSSA as normal, use the `--tee-output` flag.
+
+```sh
+fossa analyze --tee-output
+```
+
 ### Printing project metadata
 
 The `--json` flag can be used to print project metadata after running `fossa analyze` successfully. This metadata can be used to reference your project when integrating with the FOSSA API.

src/App/Fossa/Analyze.hs

@@ -49,6 +49,7 @@ import App.Fossa.Config.Analyze (
   WithoutDefaultFilters (..),
  )
 import App.Fossa.Config.Analyze qualified as Config
+import App.Fossa.Config.Common (DestinationMeta (..), destinationApiOpts, destinationMetadata)
 import App.Fossa.FirstPartyScan (runFirstPartyScan)
 import App.Fossa.Lernie.Analyze (analyzeWithLernie)
 import App.Fossa.Lernie.Types (LernieResults (..))
@@ -105,6 +106,7 @@ import Data.List.NonEmpty qualified as NE
 import Data.Maybe (fromMaybe, mapMaybe)
 import Data.String.Conversion (decodeUtf8, toText)
 import Data.Text.Extra (showT)
+import Data.Traversable (for)
 import Diag.Diagnostic as DI
 import Diag.Result (resultToMaybe)
 import Discovery.Archive qualified as Archive
@@ -278,9 +280,8 @@ analyze ::
 analyze cfg = Diag.context "fossa-analyze" $ do
   capabilities <- sendIO getNumCapabilities
 
-  let maybeApiOpts = case destination of
-        OutputStdout -> Nothing
-        UploadScan opts _ -> Just opts
+  let maybeDestMeta = destinationMetadata destination
+      maybeApiOpts = destinationApiOpts <$> maybeDestMeta
       BaseDir basedir = Config.baseDir cfg
       destination = Config.scanDestination cfg
       filters = Config.filterSet cfg
@@ -305,10 +306,12 @@ analyze cfg = Diag.context "fossa-analyze" $ do
           pure Nothing
         else Diag.context "fossa-deps" . runStickyLogger SevInfo $ analyzeFossaDepsFile basedir customFossaDepsFile maybeApiOpts vendoredDepsOptions
 
-  orgInfo <- case destination of
-    OutputStdout -> pure Nothing
-    UploadScan apiOpts metadata ->
-      fmap Just . runFossaApiClient apiOpts . preflightChecks $ AnalyzeChecks revision metadata
+  orgInfo <-
+    for
+      maybeDestMeta
+      ( \(DestinationMeta (apiOpts, metadata)) ->
+          runFossaApiClient apiOpts . preflightChecks $ AnalyzeChecks revision metadata
+      )
 
   -- additional source units are built outside the standard strategy flow, because they either
   -- require additional information (eg API credentials), or they return additional information (eg user deps).
@@ -387,21 +390,20 @@ analyze cfg = Diag.context "fossa-analyze" $ do
   let filteredProjects = mapMaybe toProjectResult projectScans
   logDebug $ "Filtered project scans: " <> pretty (show filteredProjects)
 
-  maybeEndpointAppVersion <- case destination of
-    UploadScan apiOpts _ -> runFossaApiClient apiOpts $ do
+  maybeEndpointAppVersion <- fmap join . for maybeApiOpts $
+    \apiOpts -> runFossaApiClient apiOpts $ do
       -- Using 'recovery' as API corresponding to 'getEndpointVersion',
       -- seems to be not stable and we sometimes see TimeoutError in telemetry
       version <- recover getEndpointVersion
       debugMetadata "FossaEndpointCoreVersion" version
       pure version
-    _ -> pure Nothing
 
   -- In our graph, we may have unresolved path dependencies
   -- If we are in output mode, do nothing. If we are in upload mode
   -- license scan all path dependencies, and upload findings to Endpoint,
   -- and queue a build for all path+ dependencies
-  filteredProjects' <- case (shouldAnalyzePathDependencies, destination) of
-    (True, UploadScan apiOpts _) ->
+  filteredProjects' <- case (shouldAnalyzePathDependencies, destinationMetadata destination) of
+    (True, Just (DestinationMeta (apiOpts, _))) ->
       Diag.context "path-dependencies"
         . runFossaApiClient apiOpts
         $ runStickyLogger SevInfo
@@ -440,19 +442,23 @@ analyze cfg = Diag.context "fossa-analyze" $ do
       (False, FilteredAll) -> Diag.warn ErrFilteredAllProjects $> emptyScanUnits
       (True, FilteredAll) -> Diag.warn ErrOnlyKeywordSearchResultsFound $> emptyScanUnits
       (_, CountedScanUnits scanUnits) -> pure scanUnits
-  doUpload outputResult iatAssertion destination basedir jsonOutput revision scanUnits reachabilityUnits
+  sendToDestination outputResult iatAssertion destination basedir jsonOutput revision scanUnits reachabilityUnits
 
   pure outputResult
   where
-    doUpload result iatAssertion destination basedir jsonOutput revision scanUnits reachabilityUnits =
-      case destination of
-        OutputStdout -> logStdout . decodeUtf8 $ Aeson.encode result
-        UploadScan apiOpts metadata ->
-          Diag.context "upload-results"
-            . runFossaApiClient apiOpts
-            $ do
-              locator <- uploadSuccessfulAnalysis (BaseDir basedir) metadata jsonOutput revision scanUnits reachabilityUnits
-              doAssertRevisionBinaries iatAssertion locator
+    sendToDestination result iatAssertion destination basedir jsonOutput revision scanUnits reachabilityUnits =
+      let doUpload (DestinationMeta (apiOpts, metadata)) =
+            Diag.context "upload-results"
+              . runFossaApiClient apiOpts
+              $ do
+                locator <- uploadSuccessfulAnalysis (BaseDir basedir) metadata jsonOutput revision scanUnits reachabilityUnits
+                doAssertRevisionBinaries iatAssertion locator
+       in case destination of
+            OutputStdout -> logStdout . decodeUtf8 $ Aeson.encode result
+            UploadScan meta -> doUpload meta
+            OutputAndUpload meta -> do
+              logStdout . decodeUtf8 $ Aeson.encode result
+              doUpload meta
 
     emptyScanUnits :: ScanUnits
     emptyScanUnits = SourceUnitOnly []

src/App/Fossa/Config/Analyze.hs

@@ -36,6 +36,8 @@ import App.Docs (fossaAnalyzeDefaultFilterDocUrl)
 import App.Fossa.Config.Common (
   CacheAction (WriteOnly),
   CommonOpts (..),
+  DestinationMeta (..),
+  OutputStyle (..),
   ScanDestination (..),
   applyReleaseGroupDeprecationWarning,
   baseDirArg,
@@ -46,6 +48,7 @@ import App.Fossa.Config.Common (
   collectRevisionData',
   commonOpts,
   metadataOpts,
+  outputStyleArgs,
   pathOpt,
   targetOpt,
   validateDir,
@@ -201,7 +204,7 @@ instance ToJSON VendoredDependencyOptions where
 
 data AnalyzeCliOpts = AnalyzeCliOpts
   { commons :: CommonOpts
-  , analyzeOutput :: Bool
+  , analyzeOutput :: OutputStyle
   , analyzeUnpackArchives :: Flag UnpackArchives
   , analyzeJsonOutput :: Flag JsonOutput
   , analyzeIncludeAllDeps :: Flag IncludeAll
@@ -235,9 +238,9 @@ data AnalyzeCliOpts = AnalyzeCliOpts
 
 instance GetCommonOpts AnalyzeCliOpts where
   getCommonOpts AnalyzeCliOpts{analyzeOutput, commons} =
-    if analyzeOutput
-      then Just commons{optTelemetry = Just NoTelemetry} -- When `--output` is used don't emit no telemetry.
-      else Just commons
+    case analyzeOutput of
+      Output -> Just commons{optTelemetry = Just NoTelemetry} -- When `--output` is used don't emit no telemetry.
+      _ -> Just commons
 
 instance GetSeverity AnalyzeCliOpts where
   getSeverity AnalyzeCliOpts{commons = CommonOpts{optDebug}} = if optDebug then SevDebug else SevInfo
@@ -309,7 +312,7 @@ cliParser :: Parser AnalyzeCliOpts
 cliParser =
   AnalyzeCliOpts
     <$> commonOpts
-    <*> switch (applyFossaStyle <> long "output" <> short 'o' <> stringToHelpDoc "Output results to stdout instead of uploading to FOSSA")
+    <*> outputStyleArgs
     <*> flagOpt UnpackArchives (applyFossaStyle <> long "unpack-archives" <> stringToHelpDoc "Recursively unpack and analyze discovered archives")
     <*> flagOpt JsonOutput (applyFossaStyle <> long "json" <> stringToHelpDoc "Output project metadata as JSON to the console. This is useful for communicating with the FOSSA API.")
     <*> flagOpt IncludeAll (applyFossaStyle <> long "include-unused-deps" <> stringToHelpDoc "Include all deps found, instead of filtering non-production deps. Ignored by VSI.")
@@ -643,14 +646,17 @@ collectScanDestination ::
   AnalyzeCliOpts ->
   m ScanDestination
 collectScanDestination maybeCfgFile envvars AnalyzeCliOpts{..} =
-  if analyzeOutput
-    then pure OutputStdout
-    else do
+  case analyzeOutput of
+    Output -> pure OutputStdout
+    TeeOutput -> getScanUploadDest OutputAndUpload
+    Default -> getScanUploadDest UploadScan
+  where
+    getScanUploadDest constructor = do
       apiOpts <- collectApiOpts maybeCfgFile envvars commons
       metaMerged <- maybe (pure analyzeMetadata) (mergeFileCmdMetadata analyzeMetadata) (maybeCfgFile)
       void $ applyReleaseGroupDeprecationWarning metaMerged
       when (length (projectLabel metaMerged) > 5) $ fatalText "Projects are only allowed to have 5 associated project labels"
-      pure $ UploadScan apiOpts metaMerged
+      pure $ constructor (DestinationMeta (apiOpts, metaMerged))
 
 collectVsiModeOptions ::
   ( Has Diagnostics sig m

src/App/Fossa/Config/Common.hs

@@ -35,6 +35,11 @@ module App.Fossa.Config.Common (
 
   -- * Configuration Types
   ScanDestination (..),
+  OutputStyle (..),
+  DestinationMeta (..),
+  destinationMetadata,
+  destinationApiOpts,
+  outputStyleArgs,
 
   -- * Global Defaults
   defaultTimeoutDuration,
@@ -127,6 +132,8 @@ import Options.Applicative (
   argument,
   auto,
   eitherReader,
+  flag,
+  flag',
   long,
   metavar,
   option,
@@ -151,12 +158,42 @@ import Text.Megaparsec (errorBundlePretty, runParser)
 import Text.URI (URI, mkURI)
 import Types (TargetFilter)
 
+newtype DestinationMeta = DestinationMeta (ApiOpts, ProjectMetadata)
+  deriving (Eq, Ord, Show, Generic)
+
+destinationApiOpts :: DestinationMeta -> ApiOpts
+destinationApiOpts (DestinationMeta m) = fst m
+
+instance ToJSON DestinationMeta where
+  toEncoding = genericToEncoding defaultOptions
+
+-- | CLI options describing what to do with analysis results.
+data OutputStyle
+  = -- | Upload results
+    Default
+  | -- | Output results to stdout, but do not upload
+    Output
+  | -- | Upload results as with `Upload`, but also output them to stdout as with `Output`
+    TeeOutput
+  deriving (Ord, Eq, Show)
+
+outputStyleArgs :: Parser OutputStyle
+outputStyleArgs =
+  flag' Output (applyFossaStyle <> long "output" <> short 'o' <> stringToHelpDoc "Output results to stdout instead of uploading to FOSSA")
+    <|> flag Default TeeOutput (applyFossaStyle <> long "tee-output" <> stringToHelpDoc "Like --output, but upload in addition to outputting to stdout.")
+
 data ScanDestination
   = -- | upload to fossa with provided api key and base url
-    UploadScan ApiOpts ProjectMetadata
+    UploadScan DestinationMeta
+  | OutputAndUpload DestinationMeta
   | OutputStdout
   deriving (Eq, Ord, Show, Generic)
 
+destinationMetadata :: ScanDestination -> Maybe DestinationMeta
+destinationMetadata (UploadScan meta) = Just meta
+destinationMetadata (OutputAndUpload meta) = Just meta
+destinationMetadata _ = Nothing
+
 instance ToJSON ScanDestination where
   toEncoding = genericToEncoding defaultOptions
 

src/App/Fossa/Config/Container/Analyze.hs

@@ -14,13 +14,16 @@ import App.Docs (fossaContainerAnalyzeDefaultFilterDocUrl)
 import App.Fossa.Config.Analyze (WithoutDefaultFilters, branchHelp, withoutDefaultFilterParser)
 import App.Fossa.Config.Common (
   CommonOpts (CommonOpts, optDebug, optProjectName, optProjectRevision),
+  DestinationMeta (..),
+  OutputStyle (..),
   ScanDestination (..),
   collectAPIMetadata,
   collectApiOpts,
   collectConfigFileFilters,
   collectRevisionOverride,
   commonOpts,
   metadataOpts,
+  outputStyleArgs,
  )
 import App.Fossa.Config.ConfigFile
 import App.Fossa.Config.Container.Common (
@@ -38,7 +41,7 @@ import App.Types (
 import Control.Effect.Diagnostics (Diagnostics, Has)
 import Data.Aeson (ToJSON, defaultOptions, genericToEncoding)
 import Data.Aeson.Types (ToJSON (toEncoding))
-import Data.Flag (Flag, flagOpt, fromFlag)
+import Data.Flag (Flag, flagOpt)
 import Data.Monoid.Extra (isMempty)
 import Data.Text (Text)
 import Discovery.Filters
@@ -84,7 +87,7 @@ instance ToJSON ContainerAnalyzeConfig where
 
 data ContainerAnalyzeOptions = ContainerAnalyzeOptions
   { analyzeCommons :: CommonOpts
-  , containerNoUpload :: Flag NoUpload
+  , containerOutputStyle :: OutputStyle
   , containerJsonOutput :: Flag JsonOutput
   , containerBranch :: Maybe Text
   , containerMetadata :: ProjectMetadata
@@ -109,13 +112,7 @@ cliParser :: Parser ContainerAnalyzeOptions
 cliParser =
   ContainerAnalyzeOptions
     <$> commonOpts
-    <*> flagOpt
-      NoUpload
-      ( applyFossaStyle
-          <> long "output"
-          <> short 'o'
-          <> stringToHelpDoc "Output results to stdout instead of uploading to FOSSA"
-      )
+    <*> outputStyleArgs
     <*> flagOpt JsonOutput (applyFossaStyle <> long "json" <> stringToHelpDoc "Output project metadata as JSON to the console. This is useful for communicating with the FOSSA API.")
     <*> optional
       ( strOption
@@ -173,12 +170,15 @@ collectScanDestination ::
   ContainerAnalyzeOptions ->
   m ScanDestination
 collectScanDestination maybeCfgFile envvars ContainerAnalyzeOptions{..} =
-  if fromFlag NoUpload containerNoUpload
-    then pure OutputStdout
-    else do
+  case containerOutputStyle of
+    Output -> pure OutputStdout
+    TeeOutput -> getScanUploadDest OutputAndUpload
+    Default -> getScanUploadDest UploadScan
+  where
+    getScanUploadDest constructor = do
       apiOpts <- collectApiOpts maybeCfgFile envvars analyzeCommons
       metaMerged <- collectAPIMetadata maybeCfgFile containerMetadata
-      pure $ UploadScan apiOpts metaMerged
+      pure $ constructor (DestinationMeta (apiOpts, metaMerged))
 
 collectFilters :: Maybe ConfigFile -> AllFilters
 collectFilters maybeConfig = do

src/App/Fossa/Container/AnalyzeNative.hs

@@ -10,7 +10,8 @@ import App.Fossa.API.BuildLink (getFossaBuildUrl)
 import App.Fossa.Analyze.Debug (collectDebugBundle)
 import App.Fossa.Analyze.Upload (emitBuildWarnings)
 import App.Fossa.Config.Common (
-  ScanDestination (OutputStdout, UploadScan),
+  DestinationMeta (..),
+  ScanDestination (..),
   applyReleaseGroupDeprecationWarning,
  )
 import App.Fossa.Config.Container.Analyze (
@@ -114,14 +115,19 @@ analyze cfg = do
         let branchText = fromMaybe "No branch (detached HEAD)" $ projectBranch revision
         logInfo ("Using branch: `" <> pretty branchText <> "`")
 
+  let doUpload apiOpts projectMetadata = runFossaApiClient apiOpts $ do
+        orgInfo <- preflightChecks $ AnalyzeChecks revision projectMetadata
+        logProjectInfo
+        void $ uploadScan orgInfo revision projectMetadata (jsonOutput cfg) scannedImage
+
+  let scannedImageToStdout = do
+        logProjectInfo
+        logStdout . decodeUtf8 $ Aeson.encode scannedImage
+
   case scanDestination cfg of
-    OutputStdout -> do
-      logProjectInfo
-      logStdout . decodeUtf8 $ Aeson.encode scannedImage
-    UploadScan apiOpts projectMetadata -> runFossaApiClient apiOpts $ do
-      orgInfo <- preflightChecks $ AnalyzeChecks revision projectMetadata
-      logProjectInfo
-      void $ uploadScan orgInfo revision projectMetadata (jsonOutput cfg) scannedImage
+    OutputStdout -> scannedImageToStdout
+    UploadScan (DestinationMeta (apiOpts, projectMetadata)) -> doUpload apiOpts projectMetadata
+    OutputAndUpload (DestinationMeta (apiOpts, projectMetadata)) -> scannedImageToStdout >> doUpload apiOpts projectMetadata
 
   pure scannedImage