[ane-2575] scan all layers for os info (#1566)

* scan all layers for os info

* add test

* lint

* accidently on purposed

* whitespace

* typo

* update changelog

* no need to log

* prepare for release

Author: jagonalez

Changelog.md

@@ -1,8 +1,9 @@
 # FOSSA CLI Changelog
 
-## Unreleased
+## 3.10.14
 
 - gradle: Do not report version constraints, version contraints are contained within an`DependencyResult`, filter out any constraints by checking [`isConstraint()`](https://docs.gradle.org/current/javadoc/org/gradle/api/artifacts/result/DependencyResult.html#isConstraint()). ([#1563](https://github.com/fossas/fossa-cli/pull/1563))
+- container scanning: search all layers for os information. ([#1566](https://github.com/fossas/fossa-cli/pull/1566))
 
 ## 3.10.13
 

src/App/Fossa/Container/Sources/DockerArchive.hs

@@ -120,13 +120,14 @@ analyzeFromDockerArchive systemDepsOnly filters withoutDefaultFilters tarball =
 
   -- Analyze Base Layer
   logInfo "Analyzing Base Layer"
-  baseFs <- context "Building base layer FS" $ mkFsFromChangeset $ baseLayer image
   let imageBaseLayer = baseLayer image
       baseDigest = layerDigest imageBaseLayer
-  osInfo <-
-    context "Retrieving OS Information"
-      . warnThenRecover @Text "Could not retrieve OS info"
-      $ runTarballReadFSIO baseFs tarball getOsInfo
+  baseFs <- context "Building Base Layer FS" $ mkFsFromChangeset $ baseLayer image
+  layersFs <-
+    if hasOtherLayers image
+      then pure <$> context "Building Other Layers FS" (mkFsFromChangeset (otherLayersSquashed image))
+      else pure Nothing
+  osInfo <- getOsInfoFromLayers layersFs baseFs tarball
 
   when (isNothing osInfo) $
     logInfo "No image system information detected. System dependencies will not be included with this scan."
@@ -163,11 +164,12 @@ analyzeFromDockerArchive systemDepsOnly filters withoutDefaultFilters tarball =
 
   let baseScanImageLayer = ContainerScanImageLayer baseDigest baseUnits baseObservations
 
-  if hasOtherLayers image
-    then do
+  case layersFs of
+    Nothing -> do
+      pure $ mkScan [baseScanImageLayer]
+    Just fs -> do
       logInfo "Analyzing Other Layers"
       let squashedDigest = layerDigest . otherLayersSquashed $ image
-      fs <- context "Building squashed FS from other layers" . mkFsFromChangeset $ otherLayersSquashed image
       otherUnits <-
         context "Analyzing from Other Layers" $
           analyzeLayer systemDepsOnly filters withoutDefaultFilters capabilities osInfo fs tarball
@@ -178,7 +180,6 @@ analyzeFromDockerArchive systemDepsOnly filters withoutDefaultFilters tarball =
               , ContainerScanImageLayer squashedDigest otherUnits otherObservations
               ]
       pure scan
-    else pure $ mkScan [baseScanImageLayer]
 
 analyzeLayer ::
   ( Has Diagnostics sig m
@@ -338,17 +339,20 @@ listTargetsFromDockerArchive tarball = do
   logWarn "fossa container list-targets only lists targets for experimental-scanner (when analyzed with --experimental-scanner flag)."
 
   logInfo "Analyzing Base Layer"
-  baseFs <- context "Building Base Layer FS" $ mkFsFromChangeset $ baseLayer image
-  osInfo <-
-    context "Retrieving OS Information"
-      . warnThenRecover @Text "Could not retrieve OS info"
-      $ runTarballReadFSIO baseFs tarball getOsInfo
+
+  baseFs <- mkFsFromChangeset $ baseLayer image
+  layersFs <-
+    if hasOtherLayers image
+      then pure <$> context "Building squashed FS from other layers" (mkFsFromChangeset $ otherLayersSquashed image)
+      else pure Nothing
+  osInfo <- getOsInfoFromLayers layersFs baseFs tarball
+
   context "Analyzing From Base Layer" $ listTargetLayer capabilities osInfo baseFs tarball "Base Layer"
 
-  when (hasOtherLayers image) $ do
-    logInfo "Analyzing Other Layers"
-    fs <- context "Building squashed FS from other layers" $ mkFsFromChangeset $ otherLayersSquashed image
-    void . context "Analyzing from Other Layers" $ listTargetLayer capabilities osInfo fs tarball "Other Layers"
+  case layersFs of
+    Nothing -> pure ()
+    Just fs -> do
+      void . context "Analyzing from Other Layers" $ listTargetLayer capabilities osInfo fs tarball "Other Layers"
 
 listTargetLayer ::
   ( Has Diagnostics sig m
@@ -384,3 +388,25 @@ listTargetLayer capabilities osInfo layerFs tarball layerType = do
     run = traverse_ findTargets $ layerAnalyzers osInfo False
     findTargets (DiscoverFunc f) = withDiscoveredProjects f basedir (renderLayerTarget basedir layerType)
     basedir = Path $ toString fixedVfsRoot
+
+-- | Retrieves OS information from the layers of a Docker image.
+-- It first checks the squashed layers, then the base layer.
+-- This is because layers are squashed in the order they are applied,
+-- so the last layer is the one that should have the most accurate OS info.
+getOsInfoFromLayers ::
+  ( Has Diagnostics sig m
+  , Has (Lift IO) sig m
+  ) =>
+  Maybe (SomeFileTree TarEntryOffset) ->
+  SomeFileTree TarEntryOffset ->
+  Path Abs File ->
+  m (Maybe OsInfo)
+getOsInfoFromLayers layersFs baseFs tarball =
+  context "Retrieving OS Information" $ do
+    layersFSOsInfo <- case layersFs of
+      Just fs -> warnThenRecover @Text "Could not retrieve OS info from other layers, falling back to base layer." $ runTarballReadFSIO fs tarball getOsInfo
+      Nothing -> pure Nothing
+    if isNothing layersFSOsInfo
+      then do
+        warnThenRecover @Text "Could not retrieve OS info from base layer" $ runTarballReadFSIO baseFs tarball getOsInfo
+      else pure layersFSOsInfo

test/App/Fossa/Container/AnalyzeNativeSpec.hs

@@ -7,7 +7,7 @@ import App.Fossa.Container.Scan (
   parseDockerEngineSource,
  )
 import App.Fossa.Container.Sources.DockerArchive (analyzeFromDockerArchive)
-import Container.Types (ContainerScan (ContainerScan, imageData), ContainerScanImage (ContainerScanImage, imageLayers), layerId, observations, srcUnits)
+import Container.Types (ContainerScan (ContainerScan, imageData), ContainerScanImage (ContainerScanImage, imageLayers, imageOs), layerId, observations, srcUnits)
 import Control.Carrier.Debug (IgnoreDebugC, ignoreDebug)
 import Control.Carrier.Diagnostics (DiagnosticsC, Has)
 import Control.Carrier.Stack (StackC, runStack)
@@ -81,6 +81,7 @@ analyzeSpec :: Spec
 analyzeSpec = describe "analyze" $ do
   currDir <- runIO getCurrentDir
   let imageArchive = currDir </> appDepsImage
+  let osInfoArchive = currDir </> osInfoImage
 
   it' "should not analyze application dependencies when only-system-dependencies are requested" $ do
     containerScan <- analyzeFromDockerArchive True mempty (toFlag' False) imageArchive
@@ -111,6 +112,11 @@ analyzeSpec = describe "analyze" $ do
     buildImportsOf containerScan `shouldNotContain'` [black]
     buildImportsOf containerScan `shouldBeSupersetOf'` [numpy, scipy]
 
+  it' "should get os-info from any layer" $ do
+    containerScan <- analyzeFromDockerArchive False mempty (toFlag' False) osInfoArchive
+    let osInfo = imageData containerScan
+    (imageOs osInfo) `shouldBe'` Just "fakeos"
+
 buildImportsOf :: ContainerScan -> [Locator]
 buildImportsOf scan =
   concatMap buildImports $
@@ -156,6 +162,9 @@ runWithMockDockerEngineEff =
 appDepsImage :: Path Rel File
 appDepsImage = $(mkRelFile "test/Container/testdata/app_deps_example.tar")
 
+osInfoImage :: Path Rel File
+osInfoImage = $(mkRelFile "test/Container/testdata/osinfo_example.tar")
+
 -- | From "app/services/a/" project.
 numpy :: Locator
 numpy = Locator "pip" "numpy" Nothing

test/Container/testdata/Dockerfile.osinfo.sample

@@ -0,0 +1,18 @@
+# Testcase: osinfo_example.tar
+#
+# To Build:
+#   docker build -f Dockerfile.osinfo.sample . -t osinfo_example:latest
+#
+# To Export:
+#   docker save osinfo_example:latest > osinfo_example.tar
+#
+# To Run:
+#   docker run -it osinfo_example:latest /bin/sh
+FROM busybox
+
+# Generate fake os-release file
+RUN echo "NAME=\"Fake OS\"" > /etc/os-release && \
+    echo "VERSION=\"1.0\"" >> /etc/os-release && \
+    echo "ID=fakeos" >> /etc/os-release && \
+    echo "VERSION_ID=\"1.0\"" >> /etc/os-release && \
+    echo "PRETTY_NAME=\"Fake OS 1.0\"" >> /etc/os-release