From a9e5c5e2b0513d85c16982bfe50296f7235938df Mon Sep 17 00:00:00 2001 From: Jeffrey Huynh Date: Sun, 18 Aug 2024 11:18:35 -0700 Subject: [PATCH 1/8] Draft implementation of strict mode --- src/App/Fossa/Analyze.hs | 6 +- src/App/Fossa/Analyze/Discover.hs | 1 + src/App/Fossa/Analyze/Types.hs | 5 +- src/App/Fossa/Config/Analyze.hs | 6 ++ .../Fossa/Container/Sources/DockerArchive.hs | 9 +- src/App/Fossa/ListTargets.hs | 8 +- src/Strategy/Bundler.hs | 63 +++++++++++--- src/Strategy/Cocoapods.hs | 82 ++++++++++++------- src/Strategy/Conda.hs | 12 ++- src/Strategy/Godep.hs | 18 ++-- src/Strategy/Gomodules.hs | 13 ++- src/Strategy/Maven.hs | 65 +++++++++++++-- src/Strategy/Pub.hs | 70 +++++++++++----- src/Strategy/Scala.hs | 16 ++-- 14 files changed, 283 insertions(+), 91 deletions(-) diff --git a/src/App/Fossa/Analyze.hs b/src/App/Fossa/Analyze.hs index 6079feaf04..2c8be42d8f 100644 --- a/src/App/Fossa/Analyze.hs +++ b/src/App/Fossa/Analyze.hs @@ -45,6 +45,7 @@ import App.Fossa.Config.Analyze ( IncludeAll (IncludeAll), NoDiscoveryExclusion (NoDiscoveryExclusion), ScanDestination (..), + StrictMode (..), UnpackArchives (UnpackArchives), WithoutDefaultFilters (..), ) @@ -189,6 +190,7 @@ runDependencyAnalysis :: , Has Stack sig m , Has (Reader ExperimentalAnalyzeConfig) sig m , Has (Reader MavenScopeFilters) sig m + , Has (Reader (Flag StrictMode)) sig m , Has (Reader AllFilters) sig m , Has (Reader OverrideDynamicAnalysisBinary) sig m , Has Telemetry sig m @@ -294,6 +296,7 @@ analyze cfg = Diag.context "fossa-analyze" $ do shouldAnalyzePathDependencies = resolvePathDependencies $ Config.experimental cfg allowedTactics = Config.allowedTacticTypes cfg withoutDefaultFilters = Config.withoutDefaultFilters cfg + -- strictMode = Config.strictMode cfg manualSrcUnits <- Diag.errorBoundaryIO . diagToDebug $ @@ -361,7 +364,7 @@ analyze cfg = Diag.context "fossa-analyze" $ do pure Nothing else Diag.context "first-party-scans" . runStickyLogger SevInfo $ runFirstPartyScan basedir maybeApiOpts cfg let firstPartyScanResults = join . resultToMaybe $ maybeFirstPartyScanResults - + -- logDebug $ "Is in strict mode ------------- " <> pretty (show strictMode) let discoveryFilters = if fromFlag NoDiscoveryExclusion noDiscoveryExclusion then mempty else filters (projectScans, ()) <- Diag.context "discovery/analysis tasks" @@ -372,6 +375,7 @@ analyze cfg = Diag.context "fossa-analyze" $ do . runAtomicCounter . runReader (Config.experimental cfg) . runReader (Config.mavenScopeFilterSet cfg) + . runReader (Config.strictMode cfg) . runReader discoveryFilters . runReader (Config.overrideDynamicAnalysis cfg) $ do diff --git a/src/App/Fossa/Analyze/Discover.hs b/src/App/Fossa/Analyze/Discover.hs index 2def7d9daa..697dd6baaa 100644 --- a/src/App/Fossa/Analyze/Discover.hs +++ b/src/App/Fossa/Analyze/Discover.hs @@ -4,6 +4,7 @@ module App.Fossa.Analyze.Discover ( ) where import App.Fossa.Analyze.Types (AnalyzeProject, DiscoverTaskEffs) +import App.Fossa.Config.Analyze (StrictMode) import Control.Effect.Reader (Has, Reader) import Data.Aeson qualified as Aeson import Discovery.Filters (AllFilters) diff --git a/src/App/Fossa/Analyze/Types.hs b/src/App/Fossa/Analyze/Types.hs index 54af27d13b..d21e56a579 100644 --- a/src/App/Fossa/Analyze/Types.hs +++ b/src/App/Fossa/Analyze/Types.hs @@ -11,7 +11,7 @@ module App.Fossa.Analyze.Types ( ) where import App.Fossa.Analyze.Project (ProjectResult) -import App.Fossa.Config.Analyze (ExperimentalAnalyzeConfig) +import App.Fossa.Config.Analyze (ExperimentalAnalyzeConfig, StrictMode (..)) import App.Fossa.Lernie.Types (LernieResults) import App.Fossa.Reachability.Types (SourceUnitReachability (..)) import Control.Effect.Debug (Debug) @@ -19,6 +19,7 @@ import Control.Effect.Diagnostics (Diagnostics, Has) import Control.Effect.Lift (Lift) import Control.Effect.Reader (Reader) import Control.Effect.Telemetry (Telemetry) +import Data.Flag (Flag) import Data.Set (Set) import Data.Text (Text) import Diag.Result (Result (Failure, Success)) @@ -44,6 +45,7 @@ type DiscoverTaskEffs sig m = , Has Debug sig m , Has (Reader ExperimentalAnalyzeConfig) sig m , Has (Reader MavenScopeFilters) sig m + , Has (Reader (Flag StrictMode)) sig m , Has (Reader AllFilters) sig m , Has Telemetry sig m ) @@ -69,6 +71,7 @@ type AnalyzeStaticTaskEffs sig m = , Has Debug sig m , Has (Reader ExperimentalAnalyzeConfig) sig m , Has (Reader MavenScopeFilters) sig m + , Has (Reader (Flag StrictMode)) sig m , Has (Reader AllFilters) sig m , Has Telemetry sig m ) diff --git a/src/App/Fossa/Config/Analyze.hs b/src/App/Fossa/Config/Analyze.hs index 9a7584715c..6c3e4251da 100644 --- a/src/App/Fossa/Config/Analyze.hs +++ b/src/App/Fossa/Config/Analyze.hs @@ -23,6 +23,7 @@ module App.Fossa.Config.Analyze ( GoDynamicTactic (..), StaticOnlyTactics (..), WithoutDefaultFilters (..), + StrictMode (..), mkSubCommand, loadConfig, cliParser, @@ -135,6 +136,7 @@ data ForceFirstPartyScans = ForceFirstPartyScans deriving (Generic) data ForceNoFirstPartyScans = ForceNoFirstPartyScans deriving (Generic) data IgnoreOrgWideCustomLicenseScanConfigs = IgnoreOrgWideCustomLicenseScanConfigs deriving (Generic) data StaticOnlyTactics = StaticOnlyTactics deriving (Generic) +data StrictMode = StrictMode deriving (Generic, Show) data BinaryDiscovery = BinaryDiscovery deriving (Generic) data IncludeAll = IncludeAll deriving (Generic) @@ -226,6 +228,7 @@ data AnalyzeCliOpts = AnalyzeCliOpts , analyzeCustomFossaDepsFile :: Maybe FilePath , analyzeStaticOnlyTactics :: Flag StaticOnlyTactics , analyzeWithoutDefaultFilters :: Flag WithoutDefaultFilters + , analyzeStrictMode :: Flag StrictMode } deriving (Eq, Ord, Show) @@ -264,6 +267,7 @@ data AnalyzeConfig = AnalyzeConfig , allowedTacticTypes :: AnalysisTacticTypes , reachabilityConfig :: ReachabilityConfig , withoutDefaultFilters :: Flag WithoutDefaultFilters + , strictMode :: Flag StrictMode } deriving (Eq, Ord, Show, Generic) @@ -333,6 +337,7 @@ cliParser = <*> optional (strOption (applyFossaStyle <> long "fossa-deps-file" <> helpDoc fossaDepsFileHelp <> metavar "FILEPATH")) <*> flagOpt StaticOnlyTactics (applyFossaStyle <> long "static-only-analysis" <> stringToHelpDoc "Only analyze the project using static strategies.") <*> withoutDefaultFilterParser fossaAnalyzeDefaultFilterDocUrl + <*> flagOpt StrictMode (applyFossaStyle <> long "strict" <> stringToHelpDoc "Strict mode") where fossaDepsFileHelp :: Maybe (Doc AnsiStyle) fossaDepsFileHelp = @@ -541,6 +546,7 @@ mergeStandardOpts maybeConfig envvars cliOpts@AnalyzeCliOpts{..} = do <*> pure allowedTacticType <*> resolveReachabilityOptions reachabilityConfig <*> pure analyzeWithoutDefaultFilters + <*> pure analyzeStrictMode collectMavenScopeFilters :: ( Has Diagnostics sig m diff --git a/src/App/Fossa/Container/Sources/DockerArchive.hs b/src/App/Fossa/Container/Sources/DockerArchive.hs index 1de4ab80bb..ab43fdbcde 100644 --- a/src/App/Fossa/Container/Sources/DockerArchive.hs +++ b/src/App/Fossa/Container/Sources/DockerArchive.hs @@ -16,7 +16,7 @@ import App.Fossa.Analyze.Types ( DiscoveredProjectIdentifier (..), DiscoveredProjectScan (..), ) -import App.Fossa.Config.Analyze (ExperimentalAnalyzeConfig (ExperimentalAnalyzeConfig), GoDynamicTactic (GoModulesBasedTactic), WithoutDefaultFilters (..)) +import App.Fossa.Config.Analyze (ExperimentalAnalyzeConfig (ExperimentalAnalyzeConfig), GoDynamicTactic (GoModulesBasedTactic), StrictMode (..), WithoutDefaultFilters (..)) import App.Fossa.Container.Sources.Discovery (layerAnalyzers, renderLayerTarget) import App.Fossa.Container.Sources.JarAnalysis (analyzeContainerJars) import Codec.Archive.Tar.Index (TarEntryOffset) @@ -59,7 +59,7 @@ import Control.Monad (join, void, when) import Data.Bifunctor (bimap) import Data.ByteString.Lazy qualified as BS import Data.FileTree.IndexFileTree (SomeFileTree, fixedVfsRoot) -import Data.Flag (Flag, fromFlag) +import Data.Flag (Flag, fromFlag, toFlag) import Data.Foldable (traverse_) import Data.Map qualified as Map import Data.Maybe (isNothing, listToMaybe, mapMaybe) @@ -202,6 +202,7 @@ analyzeLayer systemDepsOnly filters withoutDefaultFilters capabilities osInfo la runTarballReadFSIO layerFs tarball . runReader noExperimental . runReader noMavenScopeFilters + . runReader testFlag . Diag.context "discovery/analysis tasks" . runOutput @DiscoveredProjectScan . runStickyLogger SevInfo @@ -213,6 +214,8 @@ analyzeLayer systemDepsOnly filters withoutDefaultFilters capabilities osInfo la pure projectResults ) where + testFlag :: Flag StrictMode + testFlag = toFlag StrictMode False noMavenScopeFilters :: MavenScopeFilters noMavenScopeFilters = MavenScopeIncludeFilters mempty noExperimental :: ExperimentalAnalyzeConfig @@ -256,6 +259,7 @@ runDependencyAnalysis :: , Has (Output DiscoveredProjectScan) sig m , Has (Reader ExperimentalAnalyzeConfig) sig m , Has (Reader MavenScopeFilters) sig m + , Has (Reader (Flag StrictMode)) sig m , Has (Reader AllFilters) sig m , Has Stack sig m , Has Telemetry sig m @@ -373,6 +377,7 @@ listTargetLayer capabilities osInfo layerFs tarball layerType = do False -- Targets are not impacted by path dependencies. ) . runReader (MavenScopeIncludeFilters mempty) + . runReader (toFlag StrictMode False) . runReader (mempty :: AllFilters) $ run where diff --git a/src/App/Fossa/ListTargets.hs b/src/App/Fossa/ListTargets.hs index 23c6fcf0c4..35ba7cdfe3 100644 --- a/src/App/Fossa/ListTargets.hs +++ b/src/App/Fossa/ListTargets.hs @@ -6,7 +6,7 @@ module App.Fossa.ListTargets ( ) where import App.Fossa.Analyze.Discover (DiscoverFunc (DiscoverFunc), discoverFuncs) -import App.Fossa.Config.Analyze (ExperimentalAnalyzeConfig) +import App.Fossa.Config.Analyze (ExperimentalAnalyzeConfig, StrictMode (..)) import App.Fossa.Config.ListTargets ( ListTargetOutputFormat (..), ListTargetsCliOpts, @@ -36,6 +36,7 @@ import Control.Effect.Stack (Stack) import Control.Effect.Telemetry (Telemetry) import Data.Aeson (ToJSON, encode, object, (.=)) import Data.Aeson.Extra (encodeJSONToText) +import Data.Flag (Flag (..), toFlag) import Data.Foldable (for_, traverse_) import Data.Set.NonEmpty (toSet) import Data.String.Conversion (decodeUtf8, toText) @@ -83,10 +84,14 @@ listTargetsMain ListTargetsConfig{..} = do . runReader experimental -- `fossa list-targets` does not support maven scope filters. . runReader (MavenScopeIncludeFilters mempty) + . runReader (testFlag) -- The current version of `fossa list-targets` does not support filters. -- TODO: support both discovery and post-discovery filtering. . runReader (mempty :: AllFilters) $ runAll listTargetOutputFormat (unBaseDir baseDir) + where + testFlag :: Flag StrictMode + testFlag = toFlag StrictMode False runAll :: ( Has ReadFS sig m @@ -99,6 +104,7 @@ runAll :: , Has Stack sig m , Has (Reader ExperimentalAnalyzeConfig) sig m , Has (Reader MavenScopeFilters) sig m + , Has (Reader (Flag StrictMode)) sig m , Has (Reader AllFilters) sig m , Has Telemetry sig m ) => diff --git a/src/Strategy/Bundler.hs b/src/Strategy/Bundler.hs index af7279b858..a02af21f82 100644 --- a/src/Strategy/Bundler.hs +++ b/src/Strategy/Bundler.hs @@ -11,6 +11,7 @@ import App.Fossa.Analyze.LicenseAnalyze ( LicenseAnalyzeProject (licenseAnalyzeProject), ) import App.Fossa.Analyze.Types (AnalyzeProject (analyzeProjectStaticOnly), analyzeProject) +import App.Fossa.Config.Analyze (StrictMode (..)) import Control.Effect.Diagnostics ( Diagnostics, context, @@ -21,8 +22,9 @@ import Control.Effect.Diagnostics ( (<||>), ) import Control.Effect.Diagnostics qualified as Diag -import Control.Effect.Reader (Reader) +import Control.Effect.Reader (Reader, ask) import Data.Aeson (ToJSON) +import Data.Flag (Flag, fromFlag) import Data.Glob as Glob (toGlob, ()) import Data.Text (isSuffixOf) import Diag.Common (AllDirectDeps (AllDirectDeps), MissingEdges (MissingEdges)) @@ -35,6 +37,7 @@ import Discovery.Walk ( walkWithFilters', ) import Effect.Exec (Exec, Has) +import Effect.Logger (Logger, logDebug) import Effect.ReadFS (ReadFS, readContentsParser) import GHC.Generics (Generic) import Path (Abs, Dir, File, Path, toFilePath) @@ -119,8 +122,16 @@ mkProject project = , projectData = project } -getDeps :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m) => BundlerProject -> m DependencyResults -getDeps project = analyzeGemfileLock project <||> context "Bundler" (analyzeBundleShow project) +getDeps :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m, Has (Reader (Flag StrictMode)) sig m, Has Logger sig m) => BundlerProject -> m DependencyResults +getDeps project = do + strictMode <- ask @((Flag StrictMode)) + if fromFlag StrictMode strictMode + then do + logDebug "Ruby strict mode" + analyzeGemfileLock project + else do + logDebug "Ruby NOT IN STRICT MODE ===========" + analyzeGemfileLock project <||> context "Bundler" (analyzeBundleShow project) analyzeBundleShow :: (Has Exec sig m, Has Diagnostics sig m) => BundlerProject -> m DependencyResults analyzeBundleShow project = do @@ -132,15 +143,11 @@ analyzeBundleShow project = do , dependencyManifestFiles = maybe [bundlerGemfile project] pure (bundlerGemfileLock project) } -analyzeGemfileLock :: (Has ReadFS sig m, Has Diagnostics sig m) => BundlerProject -> m DependencyResults -analyzeGemfileLock project = - warnOnErr AllDirectDeps - . warnOnErr MissingEdges - . errCtx (BundlerMissingLockFileCtx $ bundlerGemfile project) - . errHelp BundlerMissingLockFileHelp - . errDoc bundlerLockFileRationaleUrl - . errDoc rubyFossaDocUrl - $ do +analyzeGemfileLock :: (Has ReadFS sig m, Has Diagnostics sig m, Has (Reader (Flag StrictMode)) sig m) => BundlerProject -> m DependencyResults +analyzeGemfileLock project = do + strictMode <- ask @((Flag StrictMode)) + if fromFlag StrictMode strictMode + then do lockFile <- context "Retrieve Gemfile.lock" (Diag.fromMaybeText "No Gemfile.lock present in the project" (bundlerGemfileLock project)) graph <- context "Gemfile.lock analysis" . GemfileLock.analyze' $ lockFile pure $ @@ -149,3 +156,35 @@ analyzeGemfileLock project = , dependencyGraphBreadth = Complete , dependencyManifestFiles = [lockFile] } + else do + warnOnErr AllDirectDeps + . warnOnErr MissingEdges + . errCtx (BundlerMissingLockFileCtx $ bundlerGemfile project) + . errHelp BundlerMissingLockFileHelp + . errDoc bundlerLockFileRationaleUrl + . errDoc rubyFossaDocUrl + $ do + lockFile <- context "Retrieve Gemfile.lock" (Diag.fromMaybeText "No Gemfile.lock present in the project" (bundlerGemfileLock project)) + graph <- context "Gemfile.lock analysis" . GemfileLock.analyze' $ lockFile + pure $ + DependencyResults + { dependencyGraph = graph + , dependencyGraphBreadth = Complete + , dependencyManifestFiles = [lockFile] + } + +-- warnOnErr AllDirectDeps +-- . warnOnErr MissingEdges +-- . errCtx (BundlerMissingLockFileCtx $ bundlerGemfile project) +-- . errHelp BundlerMissingLockFileHelp +-- . errDoc bundlerLockFileRationaleUrl +-- . errDoc rubyFossaDocUrl +-- $ do +-- lockFile <- context "Retrieve Gemfile.lock" (Diag.fromMaybeText "No Gemfile.lock present in the project" (bundlerGemfileLock project)) +-- graph <- context "Gemfile.lock analysis" . GemfileLock.analyze' $ lockFile +-- pure $ +-- DependencyResults +-- { dependencyGraph = graph +-- , dependencyGraphBreadth = Complete +-- , dependencyManifestFiles = [lockFile] +-- } diff --git a/src/Strategy/Cocoapods.hs b/src/Strategy/Cocoapods.hs index 58df4ef8d9..85efc6117d 100644 --- a/src/Strategy/Cocoapods.hs +++ b/src/Strategy/Cocoapods.hs @@ -8,12 +8,14 @@ module Strategy.Cocoapods ( import App.Fossa.Analyze.LicenseAnalyze (LicenseAnalyzeProject, licenseAnalyzeProject) import App.Fossa.Analyze.Types (AnalyzeProject (analyzeProjectStaticOnly), analyzeProject) +import App.Fossa.Config.Analyze (StrictMode (..)) import Control.Applicative ((<|>)) import Control.Carrier.Diagnostics (errHelp) import Control.Effect.Diagnostics (Diagnostics, context, errCtx, errDoc, warnOnErr, (<||>)) import Control.Effect.Diagnostics qualified as Diag -import Control.Effect.Reader (Reader) +import Control.Effect.Reader (Reader, ask) import Data.Aeson (ToJSON) +import Data.Flag (Flag, fromFlag) import Data.Glob qualified as Glob import Data.List (find) import Data.List.Extra (singleton) @@ -110,33 +112,57 @@ mkProject project = , projectData = project } -getDeps :: (Has ReadFS sig m, Has Diagnostics sig m, Has Exec sig m, Has Logger sig m) => CocoapodsProject -> m DependencyResults -getDeps project = - context "Cocoapods" $ - context - "Podfile.lock analysis" - ( warnOnErr MissingEdges - . warnOnErr MissingDeepDeps - . errCtx MissingPodLockFileCtx - . errHelp MissingPodLockFileHelp - . errDoc refPodDocUrl - $ (analyzePodfileLock project) - ) - <||> context "Podfile analysis" (analyzePodfile project) - -getDeps' :: (Has ReadFS sig m, Has Diagnostics sig m) => CocoapodsProject -> m DependencyResults -getDeps' project = - context "Cocoapods" $ - context - "Podfile.lock analysis" - ( warnOnErr MissingEdges - . warnOnErr MissingDeepDeps - . errCtx MissingPodLockFileCtx - . errHelp MissingPodLockFileHelp - . errDoc refPodDocUrl - $ (analyzePodfileLockStatically project) - ) - <||> context "Podfile analysis" (analyzePodfile project) +getDeps :: (Has ReadFS sig m, Has Diagnostics sig m, Has Exec sig m, Has Logger sig m, Has (Reader (Flag StrictMode)) sig m) => CocoapodsProject -> m DependencyResults +getDeps project = do + strictMode <- ask @((Flag StrictMode)) + if fromFlag StrictMode strictMode + then + context "Cocoapods" $ + context + "Podfile.lock analysis" + ( errCtx MissingPodLockFileCtx + . errHelp MissingPodLockFileHelp + . errDoc refPodDocUrl + $ (analyzePodfileLock project) + ) + else + context "Cocoapods" $ + context + "Podfile.lock analysis" + ( warnOnErr MissingEdges + . warnOnErr MissingDeepDeps + . errCtx MissingPodLockFileCtx + . errHelp MissingPodLockFileHelp + . errDoc refPodDocUrl + $ (analyzePodfileLock project) + ) + <||> context "Podfile analysis" (analyzePodfile project) + +getDeps' :: (Has ReadFS sig m, Has Diagnostics sig m, Has (Reader (Flag StrictMode)) sig m) => CocoapodsProject -> m DependencyResults +getDeps' project = do + strictMode <- ask @((Flag StrictMode)) + if fromFlag StrictMode strictMode + then + context "Cocoapods" $ + context + "Strict mode Podfile.lock analysis" + ( errCtx MissingPodLockFileCtx + . errHelp MissingPodLockFileHelp + . errDoc refPodDocUrl + $ (analyzePodfileLockStatically project) + ) + else + context "Cocoapods" $ + context + "Podfile.lock analysis" + ( warnOnErr MissingEdges + . warnOnErr MissingDeepDeps + . errCtx MissingPodLockFileCtx + . errHelp MissingPodLockFileHelp + . errDoc refPodDocUrl + $ (analyzePodfileLockStatically project) + ) + <||> context "Podfile analysis" (analyzePodfile project) analyzePodfile :: (Has ReadFS sig m, Has Diagnostics sig m) => CocoapodsProject -> m DependencyResults analyzePodfile project = do diff --git a/src/Strategy/Conda.hs b/src/Strategy/Conda.hs index 7496deecd3..581564a1b1 100644 --- a/src/Strategy/Conda.hs +++ b/src/Strategy/Conda.hs @@ -5,14 +5,16 @@ module Strategy.Conda ( ) where import App.Fossa.Analyze.Types (AnalyzeProject (analyzeProjectStaticOnly), analyzeProject) +import App.Fossa.Config.Analyze (StrictMode (..)) import Control.Effect.Diagnostics ( Diagnostics, Has, fatalText, (<||>), ) -import Control.Effect.Reader (Reader) +import Control.Effect.Reader (Reader, ask) import Data.Aeson (ToJSON) +import Data.Flag (Flag, fromFlag) import Discovery.Filters (AllFilters) import Discovery.Simple (simpleDiscover) import Discovery.Walk ( @@ -73,8 +75,12 @@ mkProject project = -- There might be a dep with a version spec in an environment.yml file: i.e. conda+foo$1.2.*, and perhaps -- the same dep resolved to a known version in the users virtual environment: i.e. conda+'conda-forge':foo$1.2.4 (we get that from conda env create). -- If we combined the results then we would include both of those deps in the result, which is not correct behavior. -getDeps :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m) => CondaProject -> m DependencyResults -getDeps project = analyzeCondaEnvCreate project <||> analyzeEnvironmentYml project +getDeps :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m, Has (Reader (Flag StrictMode)) sig m) => CondaProject -> m DependencyResults +getDeps project = do + strictMode <- ask @((Flag StrictMode)) + if fromFlag StrictMode strictMode + then analyzeCondaEnvCreate project + else analyzeCondaEnvCreate project <||> analyzeEnvironmentYml project analyzeCondaEnvCreate :: (Has Exec sig m, Has Diagnostics sig m) => CondaProject -> m DependencyResults analyzeCondaEnvCreate CondaProject{..} = do diff --git a/src/Strategy/Godep.hs b/src/Strategy/Godep.hs index 9613d3c52e..eb76119186 100644 --- a/src/Strategy/Godep.hs +++ b/src/Strategy/Godep.hs @@ -3,11 +3,13 @@ module Strategy.Godep ( ) where import App.Fossa.Analyze.Types (AnalyzeProject (analyzeProjectStaticOnly), analyzeProject) +import App.Fossa.Config.Analyze (StrictMode (..)) import Control.Applicative ((<|>)) import Control.Effect.Diagnostics (Diagnostics, context, fatalText, (<||>)) import Control.Effect.Diagnostics qualified as Diag -import Control.Effect.Reader (Reader) +import Control.Effect.Reader (Reader, ask) import Data.Aeson (ToJSON) +import Data.Flag (Flag, fromFlag) import Discovery.Filters (AllFilters) import Discovery.Simple (simpleDiscover) import Discovery.Walk ( @@ -69,11 +71,15 @@ mkProject project = , projectData = project } -getDeps :: (Has ReadFS sig m, Has Exec sig m, Has Diagnostics sig m) => GodepProject -> m DependencyResults -getDeps project = - context "Godep" $ - context "Gopkg.lock analysis" (analyzeGopkgLock project) - <||> context "Gopkg.toml analysis" (analyzeGopkgToml project) +getDeps :: (Has ReadFS sig m, Has Exec sig m, Has Diagnostics sig m, Has (Reader (Flag StrictMode)) sig m) => GodepProject -> m DependencyResults +getDeps project = do + strictMode <- ask @((Flag StrictMode)) + if fromFlag StrictMode strictMode + then context "Strict mode Godep" $ context "Gopkg.lock analysis" (analyzeGopkgLock project) + else + context "Godep" $ + context "Gopkg.lock analysis" (analyzeGopkgLock project) + <||> context "Gopkg.toml analysis" (analyzeGopkgToml project) analyzeGopkgLock :: (Has ReadFS sig m, Has Exec sig m, Has Diagnostics sig m) => GodepProject -> m DependencyResults analyzeGopkgLock project = do diff --git a/src/Strategy/Gomodules.hs b/src/Strategy/Gomodules.hs index c8653dae4b..7b14e87edb 100644 --- a/src/Strategy/Gomodules.hs +++ b/src/Strategy/Gomodules.hs @@ -7,12 +7,13 @@ module Strategy.Gomodules ( ) where import App.Fossa.Analyze.Types (AnalyzeProject (analyzeProjectStaticOnly), analyzeProject) -import App.Fossa.Config.Analyze (ExperimentalAnalyzeConfig (useV3GoResolver), GoDynamicTactic (..)) +import App.Fossa.Config.Analyze (ExperimentalAnalyzeConfig (useV3GoResolver), GoDynamicTactic (..), StrictMode (..)) import Control.Carrier.Diagnostics (warn) import Control.Effect.Diagnostics (Diagnostics, context, fatalText, recover, (<||>)) -import Control.Effect.Reader (Reader, asks) +import Control.Effect.Reader (Reader, ask, asks) import Control.Monad (when) import Data.Aeson (ToJSON) +import Data.Flag (Flag, fromFlag) import Data.Text (Text) import Discovery.Filters (AllFilters) import Discovery.Simple (simpleDiscover) @@ -67,9 +68,13 @@ mkProject project = , projectData = project } -getDeps :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m) => GomodulesProject -> GoDynamicTactic -> m DependencyResults +getDeps :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m, Has (Reader (Flag StrictMode)) sig m) => GomodulesProject -> GoDynamicTactic -> m DependencyResults getDeps project goDynamicTactic = do - (graph, graphBreadth) <- context "Gomodules" $ dynamicAnalysis <||> staticAnalysis + strictMode <- ask @((Flag StrictMode)) + (graph, graphBreadth) <- + if fromFlag StrictMode strictMode + then context "Strict mode Gomodules" dynamicAnalysis + else context "Gomodules" $ dynamicAnalysis <||> staticAnalysis stdlib <- recover . context "Collect go standard library information" . listGoStdlibPackages $ gomodulesDir project pure $ DependencyResults diff --git a/src/Strategy/Maven.hs b/src/Strategy/Maven.hs index 5953ad9d21..87585cc6e1 100644 --- a/src/Strategy/Maven.hs +++ b/src/Strategy/Maven.hs @@ -7,11 +7,14 @@ module Strategy.Maven ( import App.Fossa.Analyze.LicenseAnalyze (LicenseAnalyzeProject, licenseAnalyzeProject) import App.Fossa.Analyze.Types (AnalyzeProject (analyzeProjectStaticOnly), analyzeProject) +import App.Fossa.Config.Analyze (StrictMode (..)) import Control.Algebra (Has) -import Control.Effect.Diagnostics (Diagnostics, context, warnOnErr, (<||>)) +import Control.Carrier.Diagnostics qualified as Diag +import Control.Effect.Diagnostics (Diag (..), Diagnostics, context, warnOnErr, (<||>)) import Control.Effect.Lift (Lift) import Control.Effect.Reader (Reader, ask) import Data.Aeson (ToJSON) +import Data.Flag (Flag, fromFlag) import Data.Set (Set) import Data.Set qualified as Set import Data.Set.NonEmpty (nonEmpty, toSet) @@ -21,6 +24,7 @@ import Diag.Common (MissingDeepDeps (MissingDeepDeps), MissingEdges (MissingEdge import Discovery.Filters (AllFilters, MavenScopeFilters, mavenScopeFilterSet) import Discovery.Simple (simpleDiscover) import Effect.Exec (CandidateCommandEffs) +import Effect.Logger (Logger, logDebug, pretty) import Effect.ReadFS (ReadFS) import GHC.Generics (Generic) import Graphing (Graphing, gmap, shrinkRoots) @@ -31,6 +35,7 @@ import Strategy.Maven.PluginStrategy qualified as Plugin import Strategy.Maven.Pom qualified as Pom import Strategy.Maven.Pom.Closure (MavenProjectClosure (..)) import Strategy.Maven.Pom.Closure qualified as PomClosure +import Text.Pretty.Simple (pShow) import Types (BuildTarget (..), DependencyResults (..), DiscoveredProject (..), DiscoveredProjectType (MavenProjectType), FoundTargets (..), GraphBreadth (..)) discover :: @@ -75,13 +80,22 @@ getDeps :: , Has ReadFS sig m , CandidateCommandEffs sig m , Has (Reader MavenScopeFilters) sig m + , Has (Reader (Flag StrictMode)) sig m + , Has Logger sig m ) => FoundTargets -> MavenProject -> m DependencyResults getDeps foundTargets (MavenProject closure) = do let submoduleTargets = submoduleTargetSet foundTargets - (graph, graphBreadth) <- context "Maven" $ getDepsDynamicAnalysis submoduleTargets closure <||> getStaticAnalysis submoduleTargets closure + strictMode <- ask @((Flag StrictMode)) + (graph, graphBreadth) <- + if fromFlag StrictMode strictMode + then do + logDebug "In switch case for strict mode (getDeps) &&&&&&&&&" + context "Maven" $ getDepsDynamicAnalysis submoduleTargets closure + else context "Maven" $ getDepsDynamicAnalysis submoduleTargets closure <||> getStaticAnalysis submoduleTargets closure + -- (graph, graphBreadth) <- context "Maven" $ getDepsDynamicAnalysis submoduleTargets closure <||> getStaticAnalysis submoduleTargets closure pure $ DependencyResults { dependencyGraph = graph @@ -93,6 +107,8 @@ getDeps' :: ( Has (Lift IO) sig m , Has Diagnostics sig m , Has (Reader MavenScopeFilters) sig m + , Has (Reader (Flag StrictMode)) sig m + , Has Logger sig m ) => FoundTargets -> MavenProject -> @@ -113,17 +129,34 @@ getDepsDynamicAnalysis :: , Has ReadFS sig m , CandidateCommandEffs sig m , Has (Reader MavenScopeFilters) sig m + , Has (Reader (Flag StrictMode)) sig m + , Has Logger sig m ) => Set Text -> MavenProjectClosure -> m (Graphing Dependency, GraphBreadth) getDepsDynamicAnalysis submoduleTargets closure = do let allSubmodules = PomClosure.closureSubmodules closure + strictMode <- ask @((Flag StrictMode)) + logDebug $ "Is in strict mode for maven strategy ------------- " <> pretty (pShow strictMode) (graph, graphBreadth) <- - context "Dynamic Analysis" - $ warnOnErr MissingEdges - . warnOnErr MissingDeepDeps - $ (getDepsPlugin closure <||> getDepsTreeCmd closure <||> getDepsPluginLegacy closure) + if fromFlag StrictMode strictMode + then do + logDebug "In switch case for strict mode ===========" + context "Dynamic Analysis" $ getDepsPlugin closure <||> getDepsTreeCmd closure <||> getDepsPluginLegacy closure + else do + logDebug "NOT IN STRICT MODE ===========" + context "Dynamic Analysis" + $ warnOnErr MissingEdges + . warnOnErr MissingDeepDeps + $ (getDepsPlugin closure <||> getDepsTreeCmd closure <||> getDepsPluginLegacy closure) + + logDebug $ "Is in strict mode for maven strategy ------------- " <> pretty (pShow strictMode) + -- (graph, graphBreadth) <- + -- context "Dynamic Analysis" + -- $ warnOnErr MissingEdges + -- . warnOnErr MissingDeepDeps + -- $ (getDepsPlugin closure <||> getDepsTreeCmd closure <||> getDepsPluginLegacy closure) filteredGraph <- applyMavenFilters submoduleTargets allSubmodules graph pure (withoutProjectAsDep filteredGraph, graphBreadth) where @@ -137,28 +170,42 @@ getDepsPlugin :: ( CandidateCommandEffs sig m , Has (Lift IO) sig m , Has ReadFS sig m + , Has Diagnostics sig m ) => MavenProjectClosure -> m (Graphing MavenDependency, GraphBreadth) -getDepsPlugin closure = context "Plugin analysis" (Plugin.analyze' . parent $ PomClosure.closurePath closure) +getDepsPlugin closure = do + Diag.fatal MissingDeepDeps + +-- context "Plugin analysis" (Plugin.analyze' . parent $ PomClosure.closurePath closure) getDepsPluginLegacy :: ( CandidateCommandEffs sig m , Has (Lift IO) sig m , Has ReadFS sig m + , Has Diagnostics sig m + , Has Logger sig m ) => MavenProjectClosure -> m (Graphing MavenDependency, GraphBreadth) -getDepsPluginLegacy closure = context "Legacy Plugin analysis" (Plugin.analyzeLegacy' . parent $ PomClosure.closurePath closure) +getDepsPluginLegacy closure = do + logDebug $ "In plugin command ++++++++ " + -- Diag.fatal MissingDeepDeps + context "Legacy Plugin analysis" (Plugin.analyzeLegacy' . parent $ PomClosure.closurePath closure) getDepsTreeCmd :: ( Has (Lift IO) sig m , Has ReadFS sig m , CandidateCommandEffs sig m + , Has Diagnostics sig m + , Has Logger sig m ) => MavenProjectClosure -> m (Graphing MavenDependency, GraphBreadth) -getDepsTreeCmd closure = +getDepsTreeCmd closure = do + logDebug $ "In deps tree comand ++++++++ " + Diag.fatal MissingDeepDeps + context "Dynamic analysis" $ DepTreeCmd.analyze . parent $ PomClosure.closurePath closure diff --git a/src/Strategy/Pub.hs b/src/Strategy/Pub.hs index 55c66db080..2d9b991319 100644 --- a/src/Strategy/Pub.hs +++ b/src/Strategy/Pub.hs @@ -2,11 +2,13 @@ module Strategy.Pub (discover) where import App.Fossa.Analyze.Types (AnalyzeProject (analyzeProjectStaticOnly), analyzeProject) +import App.Fossa.Config.Analyze (StrictMode (..), strictMode) import Control.Carrier.Diagnostics (errDoc, errHelp) import Control.Effect.Diagnostics (Diagnostics, errCtx, fatalText, recover, warnOnErr, (<||>)) -import Control.Effect.Reader (Reader) +import Control.Effect.Reader (Reader, ask) import Control.Monad (void) import Data.Aeson (ToJSON) +import Data.Flag (Flag, fromFlag) import Diag.Common ( MissingDeepDeps (MissingDeepDeps), MissingEdges (MissingEdges), @@ -15,7 +17,7 @@ import Discovery.Filters (AllFilters) import Discovery.Simple (simpleDiscover) import Discovery.Walk (WalkStep (WalkContinue), findFileNamed, walkWithFilters') import Effect.Exec (Exec, Has) -import Effect.Logger (Logger) +import Effect.Logger (Logger, logDebug) import Effect.ReadFS (ReadFS) import GHC.Generics (Generic) import Path (Abs, Dir, File, Path) @@ -64,18 +66,39 @@ mkProject project = , projectData = project } -getDeps :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m, Has Logger sig m) => PubProject -> m DependencyResults +getDeps :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m, Has Logger sig m, Has (Reader (Flag StrictMode)) sig m) => PubProject -> m DependencyResults getDeps project = do + strictMode <- ask @((Flag StrictMode)) (graph, graphBreadth) <- case pubLock project of - Just lockFile -> analyzeDepsCmd lockFile (pubSpecDir project) <||> analyzePubLockFile lockFile + Just lockFile -> do + if fromFlag StrictMode strictMode + then do + logDebug "Pub Analysis in strict mode" + analyzeDepsCmd lockFile (pubSpecDir project) + else do + logDebug "Pub Analysis NOT in strict mode" + analyzeDepsCmd lockFile (pubSpecDir project) <||> analyzePubLockFile lockFile Nothing -> do - void . recover - $ warnOnErr MissingDeepDeps - . warnOnErr MissingEdges - $ errCtx PubspecLimitationCtx - $ errHelp PubspecLimitationHelp - $ errDoc refPubDocUrl (fatalText "Missing pubspec.lock file") - analyzePubSpecFile $ pubSpec project + if fromFlag StrictMode strictMode + then do + logDebug "Pub Analysis (NOTHING) in strict mode" + analyzePubSpecFile $ pubSpec project + else do + logDebug "Pub Analysis (NOTHING) NOT in strict mode" + void . recover + $ warnOnErr MissingDeepDeps + . warnOnErr MissingEdges + $ errCtx PubspecLimitationCtx + $ errHelp PubspecLimitationHelp + $ errDoc refPubDocUrl (fatalText "Missing pubspec.lock file") + analyzePubSpecFile $ pubSpec project + -- void . recover + -- $ warnOnErr MissingDeepDeps + -- . warnOnErr MissingEdges + -- $ errCtx PubspecLimitationCtx + -- $ errHelp PubspecLimitationHelp + -- $ errDoc refPubDocUrl (fatalText "Missing pubspec.lock file") + -- analyzePubSpecFile $ pubSpec project pure $ DependencyResults { dependencyGraph = graph @@ -83,18 +106,27 @@ getDeps project = do , dependencyManifestFiles = [pubSpec project] } -getDeps' :: (Has ReadFS sig m, Has Diagnostics sig m, Has Logger sig m) => PubProject -> m DependencyResults +getDeps' :: (Has ReadFS sig m, Has Diagnostics sig m, Has Logger sig m, Has (Reader (Flag StrictMode)) sig m) => PubProject -> m DependencyResults getDeps' project = do + strictMode <- ask @((Flag StrictMode)) (graph, graphBreadth) <- case pubLock project of Just lockFile -> analyzePubLockFile lockFile Nothing -> do - void . recover - $ warnOnErr MissingDeepDeps - . warnOnErr MissingEdges - $ errCtx PubspecLimitationCtx - $ errHelp PubspecLimitationHelp - $ errDoc refPubDocUrl (fatalText "Missing pubspec.lock file") - analyzePubSpecFile $ pubSpec project + if fromFlag StrictMode strictMode + then do + void $ + errCtx PubspecLimitationCtx $ + errHelp PubspecLimitationHelp $ + errDoc refPubDocUrl (fatalText "Missing pubspec.lock file") + analyzePubSpecFile $ pubSpec project + else do + void . recover + $ warnOnErr MissingDeepDeps + . warnOnErr MissingEdges + $ errCtx PubspecLimitationCtx + $ errHelp PubspecLimitationHelp + $ errDoc refPubDocUrl (fatalText "Missing pubspec.lock file") + analyzePubSpecFile $ pubSpec project pure $ DependencyResults { dependencyGraph = graph diff --git a/src/Strategy/Scala.hs b/src/Strategy/Scala.hs index 0c332476e3..5ffb816924 100644 --- a/src/Strategy/Scala.hs +++ b/src/Strategy/Scala.hs @@ -13,12 +13,14 @@ module Strategy.Scala ( ) where import App.Fossa.Analyze.Types (AnalyzeProject (analyzeProjectStaticOnly), analyzeProject) +import App.Fossa.Config.Analyze (StrictMode (..)) import Control.Carrier.Diagnostics (errDoc) import Control.Effect.Diagnostics (Diagnostics, errCtx, errHelp, fatalText, fromMaybeText, recover, warnOnErr, (<||>)) -import Control.Effect.Reader (Reader) +import Control.Effect.Reader (Reader, ask) import Control.Effect.Stack (context) import Data.Aeson (KeyValue ((.=)), ToJSON (toJSON), object) import Data.ByteString.Lazy (ByteString) +import Data.Flag (Flag, fromFlag) import Data.Maybe (fromMaybe, listToMaybe, mapMaybe) import Data.String.Conversion (ConvertUtf8 (decodeUtf8), toString, toText) import Data.Text (Text) @@ -127,10 +129,14 @@ mkProject (ScalaProject sbtBuildDir sbtTreeJson closure) = , projectData = ScalaProject sbtBuildDir sbtTreeJson closure } -getDeps :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m, Has Logger sig m) => ScalaProject -> m DependencyResults -getDeps project = - warnOnErr MissingDeepDeps (analyzeWithDepTreeJson project <||> analyzeWithSbtDepTree project) - <||> analyzeWithPoms project +getDeps :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m, Has Logger sig m, Has (Reader (Flag StrictMode)) sig m) => ScalaProject -> m DependencyResults +getDeps project = do + strictMode <- ask @((Flag StrictMode)) + if fromFlag StrictMode strictMode + then analyzeWithDepTreeJson project + else + warnOnErr MissingDeepDeps (analyzeWithDepTreeJson project <||> analyzeWithSbtDepTree project) + <||> analyzeWithPoms project pathToText :: Path ar fd -> Text pathToText = toText . toFilePath From 7ff15c021a66e7957a082db4afe33df56e2bd4ea Mon Sep 17 00:00:00 2001 From: Jeffrey Huynh Date: Fri, 30 Aug 2024 12:31:57 -0700 Subject: [PATCH 2/8] Implementation changes. Fix tests --- src/App/Fossa/Analyze.hs | 10 ++- src/App/Fossa/Analyze/Discover.hs | 1 - src/App/Fossa/Analyze/Types.hs | 8 +- src/App/Fossa/Config/Analyze.hs | 18 +++-- .../Fossa/Container/Sources/DockerArchive.hs | 13 ++- src/App/Fossa/ListTargets.hs | 12 +-- src/App/Types.hs | 9 +++ src/App/Util.hs | 14 +++- src/Strategy/Bundler.hs | 62 ++++---------- src/Strategy/Cocoapods.hs | 81 +++++++------------ src/Strategy/Conda.hs | 12 ++- src/Strategy/Godep.hs | 17 ++-- src/Strategy/Gomodules.hs | 14 ++-- src/Strategy/Maven.hs | 74 +++++------------ src/Strategy/Pub.hs | 72 +++++------------ src/Strategy/Scala.hs | 13 ++- test/App/Fossa/AnalyzeSpec.hs | 4 +- test/Test/Fixtures.hs | 3 +- 18 files changed, 168 insertions(+), 269 deletions(-) diff --git a/src/App/Fossa/Analyze.hs b/src/App/Fossa/Analyze.hs index 2c8be42d8f..aeebc76397 100644 --- a/src/App/Fossa/Analyze.hs +++ b/src/App/Fossa/Analyze.hs @@ -45,7 +45,6 @@ import App.Fossa.Config.Analyze ( IncludeAll (IncludeAll), NoDiscoveryExclusion (NoDiscoveryExclusion), ScanDestination (..), - StrictMode (..), UnpackArchives (UnpackArchives), WithoutDefaultFilters (..), ) @@ -65,6 +64,7 @@ import App.Fossa.VSIDeps (analyzeVSIDeps) import App.Types ( BaseDir (..), FirstPartyScansFlag (..), + Mode (..), OverrideDynamicAnalysisBinary, ProjectRevision (..), ) @@ -190,7 +190,7 @@ runDependencyAnalysis :: , Has Stack sig m , Has (Reader ExperimentalAnalyzeConfig) sig m , Has (Reader MavenScopeFilters) sig m - , Has (Reader (Flag StrictMode)) sig m + , Has (Reader Mode) sig m , Has (Reader AllFilters) sig m , Has (Reader OverrideDynamicAnalysisBinary) sig m , Has Telemetry sig m @@ -375,9 +375,9 @@ analyze cfg = Diag.context "fossa-analyze" $ do . runAtomicCounter . runReader (Config.experimental cfg) . runReader (Config.mavenScopeFilterSet cfg) - . runReader (Config.strictMode cfg) . runReader discoveryFilters . runReader (Config.overrideDynamicAnalysis cfg) + . runReader (Config.mode cfg) $ do runAnalyzers allowedTactics filters withoutDefaultFilters basedir Nothing when (fromFlag UnpackArchives $ Config.unpackArchives cfg) $ @@ -599,3 +599,7 @@ updateProgress Progress{..} = <> " Completed" <> " ]" ) + +-- analyzeStrictModeGuard :: Has Diagnostics sig m => Mode -> m a -> m a +-- analyzeStrictModeGuard Strict _ = fatal "Strict mode enabled, skipping other strategies" +-- analyzeStrictModeGuard NonStrict action = action diff --git a/src/App/Fossa/Analyze/Discover.hs b/src/App/Fossa/Analyze/Discover.hs index 697dd6baaa..2def7d9daa 100644 --- a/src/App/Fossa/Analyze/Discover.hs +++ b/src/App/Fossa/Analyze/Discover.hs @@ -4,7 +4,6 @@ module App.Fossa.Analyze.Discover ( ) where import App.Fossa.Analyze.Types (AnalyzeProject, DiscoverTaskEffs) -import App.Fossa.Config.Analyze (StrictMode) import Control.Effect.Reader (Has, Reader) import Data.Aeson qualified as Aeson import Discovery.Filters (AllFilters) diff --git a/src/App/Fossa/Analyze/Types.hs b/src/App/Fossa/Analyze/Types.hs index d21e56a579..5ab47478c0 100644 --- a/src/App/Fossa/Analyze/Types.hs +++ b/src/App/Fossa/Analyze/Types.hs @@ -11,15 +11,15 @@ module App.Fossa.Analyze.Types ( ) where import App.Fossa.Analyze.Project (ProjectResult) -import App.Fossa.Config.Analyze (ExperimentalAnalyzeConfig, StrictMode (..)) +import App.Fossa.Config.Analyze (ExperimentalAnalyzeConfig) import App.Fossa.Lernie.Types (LernieResults) import App.Fossa.Reachability.Types (SourceUnitReachability (..)) +import App.Types (Mode) import Control.Effect.Debug (Debug) import Control.Effect.Diagnostics (Diagnostics, Has) import Control.Effect.Lift (Lift) import Control.Effect.Reader (Reader) import Control.Effect.Telemetry (Telemetry) -import Data.Flag (Flag) import Data.Set (Set) import Data.Text (Text) import Diag.Result (Result (Failure, Success)) @@ -45,7 +45,7 @@ type DiscoverTaskEffs sig m = , Has Debug sig m , Has (Reader ExperimentalAnalyzeConfig) sig m , Has (Reader MavenScopeFilters) sig m - , Has (Reader (Flag StrictMode)) sig m + , Has (Reader Mode) sig m , Has (Reader AllFilters) sig m , Has Telemetry sig m ) @@ -71,7 +71,7 @@ type AnalyzeStaticTaskEffs sig m = , Has Debug sig m , Has (Reader ExperimentalAnalyzeConfig) sig m , Has (Reader MavenScopeFilters) sig m - , Has (Reader (Flag StrictMode)) sig m + , Has (Reader Mode) sig m , Has (Reader AllFilters) sig m , Has Telemetry sig m ) diff --git a/src/App/Fossa/Config/Analyze.hs b/src/App/Fossa/Config/Analyze.hs index 6c3e4251da..3ab926a32a 100644 --- a/src/App/Fossa/Config/Analyze.hs +++ b/src/App/Fossa/Config/Analyze.hs @@ -24,6 +24,7 @@ module App.Fossa.Config.Analyze ( StaticOnlyTactics (..), WithoutDefaultFilters (..), StrictMode (..), + Mode (..), mkSubCommand, loadConfig, cliParser, @@ -72,6 +73,7 @@ import App.Fossa.VSI.Types qualified as VSI import App.Types ( BaseDir, FirstPartyScansFlag (..), + Mode (..), OverrideDynamicAnalysisBinary (..), OverrideProject (OverrideProject), ProjectMetadata (projectLabel), @@ -267,7 +269,7 @@ data AnalyzeConfig = AnalyzeConfig , allowedTacticTypes :: AnalysisTacticTypes , reachabilityConfig :: ReachabilityConfig , withoutDefaultFilters :: Flag WithoutDefaultFilters - , strictMode :: Flag StrictMode + , mode :: Mode } deriving (Eq, Ord, Show, Generic) @@ -504,7 +506,7 @@ mergeStandardOpts maybeConfig envvars cliOpts@AnalyzeCliOpts{..} = do revisionData = collectRevisionData' basedir maybeConfig WriteOnly $ OverrideProject (optProjectName commons) (optProjectRevision commons) (analyzeBranch) - modeOpts = collectModeOptions cliOpts + vsiModeOpts = collectVsiModeOptions cliOpts filters = collectFilters maybeConfig cliOpts mavenScopeFilters = collectMavenScopeFilters maybeConfig experimentalCfgs = collectExperimental maybeConfig cliOpts @@ -517,6 +519,10 @@ mergeStandardOpts maybeConfig envvars cliOpts@AnalyzeCliOpts{..} = do then StaticOnly else Any reachabilityConfig = collectReachabilityOptions maybeConfig + mode = + if fromFlag StrictMode analyzeStrictMode + then Strict + else NonStrict firstPartyScansFlag <- case (fromFlag ForceFirstPartyScans analyzeForceFirstPartyScans, fromFlag ForceNoFirstPartyScans analyzeForceNoFirstPartyScans) of @@ -530,7 +536,7 @@ mergeStandardOpts maybeConfig envvars cliOpts@AnalyzeCliOpts{..} = do <*> pure logSeverity <*> scanDestination <*> revisionData - <*> modeOpts + <*> vsiModeOpts <*> filters <*> mavenScopeFilters <*> pure experimentalCfgs @@ -546,7 +552,7 @@ mergeStandardOpts maybeConfig envvars cliOpts@AnalyzeCliOpts{..} = do <*> pure allowedTacticType <*> resolveReachabilityOptions reachabilityConfig <*> pure analyzeWithoutDefaultFilters - <*> pure analyzeStrictMode + <*> pure mode collectMavenScopeFilters :: ( Has Diagnostics sig m @@ -649,14 +655,14 @@ collectScanDestination maybeCfgFile envvars AnalyzeCliOpts{..} = when (length (projectLabel metaMerged) > 5) $ fatalText "Projects are only allowed to have 5 associated project labels" pure $ UploadScan apiOpts metaMerged -collectModeOptions :: +collectVsiModeOptions :: ( Has Diagnostics sig m , Has (Lift IO) sig m , Has ReadFS sig m ) => AnalyzeCliOpts -> m VSIModeOptions -collectModeOptions AnalyzeCliOpts{..} = do +collectVsiModeOptions AnalyzeCliOpts{..} = do assertionDir <- traverse validateDir analyzeAssertMode resolvedDynamicLinkTarget <- traverse validateExists analyzeDynamicLinkTarget pure diff --git a/src/App/Fossa/Container/Sources/DockerArchive.hs b/src/App/Fossa/Container/Sources/DockerArchive.hs index ab43fdbcde..b413f5c985 100644 --- a/src/App/Fossa/Container/Sources/DockerArchive.hs +++ b/src/App/Fossa/Container/Sources/DockerArchive.hs @@ -16,9 +16,10 @@ import App.Fossa.Analyze.Types ( DiscoveredProjectIdentifier (..), DiscoveredProjectScan (..), ) -import App.Fossa.Config.Analyze (ExperimentalAnalyzeConfig (ExperimentalAnalyzeConfig), GoDynamicTactic (GoModulesBasedTactic), StrictMode (..), WithoutDefaultFilters (..)) +import App.Fossa.Config.Analyze (ExperimentalAnalyzeConfig (ExperimentalAnalyzeConfig), GoDynamicTactic (GoModulesBasedTactic), WithoutDefaultFilters (..)) import App.Fossa.Container.Sources.Discovery (layerAnalyzers, renderLayerTarget) import App.Fossa.Container.Sources.JarAnalysis (analyzeContainerJars) +import App.Types (Mode (..)) import Codec.Archive.Tar.Index (TarEntryOffset) import Container.Docker.Manifest (getImageDigest, getRepoTags) import Container.OsRelease (OsInfo (nameId, version), getOsInfo) @@ -59,7 +60,7 @@ import Control.Monad (join, void, when) import Data.Bifunctor (bimap) import Data.ByteString.Lazy qualified as BS import Data.FileTree.IndexFileTree (SomeFileTree, fixedVfsRoot) -import Data.Flag (Flag, fromFlag, toFlag) +import Data.Flag (Flag, fromFlag) import Data.Foldable (traverse_) import Data.Map qualified as Map import Data.Maybe (isNothing, listToMaybe, mapMaybe) @@ -202,7 +203,7 @@ analyzeLayer systemDepsOnly filters withoutDefaultFilters capabilities osInfo la runTarballReadFSIO layerFs tarball . runReader noExperimental . runReader noMavenScopeFilters - . runReader testFlag + . runReader NonStrict . Diag.context "discovery/analysis tasks" . runOutput @DiscoveredProjectScan . runStickyLogger SevInfo @@ -214,8 +215,6 @@ analyzeLayer systemDepsOnly filters withoutDefaultFilters capabilities osInfo la pure projectResults ) where - testFlag :: Flag StrictMode - testFlag = toFlag StrictMode False noMavenScopeFilters :: MavenScopeFilters noMavenScopeFilters = MavenScopeIncludeFilters mempty noExperimental :: ExperimentalAnalyzeConfig @@ -259,7 +258,7 @@ runDependencyAnalysis :: , Has (Output DiscoveredProjectScan) sig m , Has (Reader ExperimentalAnalyzeConfig) sig m , Has (Reader MavenScopeFilters) sig m - , Has (Reader (Flag StrictMode)) sig m + , Has (Reader Mode) sig m , Has (Reader AllFilters) sig m , Has Stack sig m , Has Telemetry sig m @@ -377,7 +376,7 @@ listTargetLayer capabilities osInfo layerFs tarball layerType = do False -- Targets are not impacted by path dependencies. ) . runReader (MavenScopeIncludeFilters mempty) - . runReader (toFlag StrictMode False) + . runReader (NonStrict :: Mode) . runReader (mempty :: AllFilters) $ run where diff --git a/src/App/Fossa/ListTargets.hs b/src/App/Fossa/ListTargets.hs index 35ba7cdfe3..ebd3c2449c 100644 --- a/src/App/Fossa/ListTargets.hs +++ b/src/App/Fossa/ListTargets.hs @@ -6,7 +6,7 @@ module App.Fossa.ListTargets ( ) where import App.Fossa.Analyze.Discover (DiscoverFunc (DiscoverFunc), discoverFuncs) -import App.Fossa.Config.Analyze (ExperimentalAnalyzeConfig, StrictMode (..)) +import App.Fossa.Config.Analyze (ExperimentalAnalyzeConfig) import App.Fossa.Config.ListTargets ( ListTargetOutputFormat (..), ListTargetsCliOpts, @@ -14,7 +14,7 @@ import App.Fossa.Config.ListTargets ( mkSubCommand, ) import App.Fossa.Subcommand (SubCommand) -import App.Types (BaseDir (..)) +import App.Types (BaseDir (..), Mode (..)) import Control.Carrier.AtomicCounter ( AtomicCounter, Has, @@ -36,7 +36,6 @@ import Control.Effect.Stack (Stack) import Control.Effect.Telemetry (Telemetry) import Data.Aeson (ToJSON, encode, object, (.=)) import Data.Aeson.Extra (encodeJSONToText) -import Data.Flag (Flag (..), toFlag) import Data.Foldable (for_, traverse_) import Data.Set.NonEmpty (toSet) import Data.String.Conversion (decodeUtf8, toText) @@ -84,14 +83,11 @@ listTargetsMain ListTargetsConfig{..} = do . runReader experimental -- `fossa list-targets` does not support maven scope filters. . runReader (MavenScopeIncludeFilters mempty) - . runReader (testFlag) + . runReader (NonStrict) -- The current version of `fossa list-targets` does not support filters. -- TODO: support both discovery and post-discovery filtering. . runReader (mempty :: AllFilters) $ runAll listTargetOutputFormat (unBaseDir baseDir) - where - testFlag :: Flag StrictMode - testFlag = toFlag StrictMode False runAll :: ( Has ReadFS sig m @@ -104,7 +100,7 @@ runAll :: , Has Stack sig m , Has (Reader ExperimentalAnalyzeConfig) sig m , Has (Reader MavenScopeFilters) sig m - , Has (Reader (Flag StrictMode)) sig m + , Has (Reader Mode) sig m , Has (Reader AllFilters) sig m , Has Telemetry sig m ) => diff --git a/src/App/Types.hs b/src/App/Types.hs index 821f9e39a1..0d907574a5 100644 --- a/src/App/Types.hs +++ b/src/App/Types.hs @@ -17,6 +17,7 @@ module App.Types ( ReleaseGroupProjectRevision (..), ReleaseGroupReleaseRevision (..), ComponentUploadFileType (..), + Mode (..), uploadFileTypeToFetcherName, ) where @@ -208,3 +209,11 @@ data FirstPartyScansFlag = FirstPartyScansOnFromFlag | FirstPartyScansOffFromFla instance ToJSON FirstPartyScansFlag where toEncoding = genericToEncoding defaultOptions + +data Mode + = Strict + | NonStrict + deriving (Eq, Ord, Show, Generic) + +instance ToJSON Mode where + toEncoding = genericToEncoding defaultOptions diff --git a/src/App/Util.hs b/src/App/Util.hs index 2c98845f28..06a10b12c1 100644 --- a/src/App/Util.hs +++ b/src/App/Util.hs @@ -5,14 +5,18 @@ module App.Util ( ancestryDirect, validateDir, validateFile, + guardStrictMode, + populateWarningsForAnalysisMode, FileAncestry (..), ) where import App.Types import Control.Algebra (Has) -import Control.Carrier.Diagnostics (Diagnostics, fatalText) +import Control.Carrier.Diagnostics (Diagnostics) +import Control.Effect.Diagnostics (fatalText, warnOnErr) import Control.Monad (unless) import Data.String.Conversion (ToText (..)) +import Diag.Common (MissingDeepDeps (..), MissingEdges (..)) import GHC.Generics (Generic) import Path (Abs, Dir, File, Path, Rel, SomeBase (..), toFilePath, ()) import Path.Extra (tryMakeRelative) @@ -51,3 +55,11 @@ ancestryDerived :: Has Diagnostics sig m => FileAncestry -> Path Abs Dir -> Path ancestryDerived parent dir file = do rel <- ancestryDirect dir file pure $ fileAncestryPath parent rel + +guardStrictMode :: Has Diagnostics sig m => Mode -> m a -> m a +guardStrictMode Strict _ = fatalText "Strict mode enabled, skipping other strategies" +guardStrictMode NonStrict action = action + +populateWarningsForAnalysisMode :: (Has Diagnostics sig m) => Mode -> m a -> m a +populateWarningsForAnalysisMode Strict = id +populateWarningsForAnalysisMode NonStrict = warnOnErr MissingEdges . warnOnErr MissingDeepDeps diff --git a/src/Strategy/Bundler.hs b/src/Strategy/Bundler.hs index a02af21f82..1933815c2f 100644 --- a/src/Strategy/Bundler.hs +++ b/src/Strategy/Bundler.hs @@ -12,12 +12,15 @@ import App.Fossa.Analyze.LicenseAnalyze ( ) import App.Fossa.Analyze.Types (AnalyzeProject (analyzeProjectStaticOnly), analyzeProject) import App.Fossa.Config.Analyze (StrictMode (..)) +import App.Types (Mode (..)) +import App.Util (guardStrictMode) import Control.Effect.Diagnostics ( Diagnostics, context, errCtx, errDoc, errHelp, + fatal, warnOnErr, (<||>), ) @@ -122,16 +125,10 @@ mkProject project = , projectData = project } -getDeps :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m, Has (Reader (Flag StrictMode)) sig m, Has Logger sig m) => BundlerProject -> m DependencyResults +getDeps :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m, Has (Reader Mode) sig m) => BundlerProject -> m DependencyResults getDeps project = do - strictMode <- ask @((Flag StrictMode)) - if fromFlag StrictMode strictMode - then do - logDebug "Ruby strict mode" - analyzeGemfileLock project - else do - logDebug "Ruby NOT IN STRICT MODE ===========" - analyzeGemfileLock project <||> context "Bundler" (analyzeBundleShow project) + mode <- ask + analyzeGemfileLock project <||> guardStrictMode mode (context "Bundler" (analyzeBundleShow project)) analyzeBundleShow :: (Has Exec sig m, Has Diagnostics sig m) => BundlerProject -> m DependencyResults analyzeBundleShow project = do @@ -143,11 +140,18 @@ analyzeBundleShow project = do , dependencyManifestFiles = maybe [bundlerGemfile project] pure (bundlerGemfileLock project) } -analyzeGemfileLock :: (Has ReadFS sig m, Has Diagnostics sig m, Has (Reader (Flag StrictMode)) sig m) => BundlerProject -> m DependencyResults +analyzeGemfileLock :: (Has ReadFS sig m, Has Diagnostics sig m, Has (Reader Mode) sig m) => BundlerProject -> m DependencyResults analyzeGemfileLock project = do - strictMode <- ask @((Flag StrictMode)) - if fromFlag StrictMode strictMode - then do + mode <- ask + let errorInfo = case mode of + Strict -> id + NonStrict -> warnOnErr AllDirectDeps . warnOnErr MissingEdges + errorInfo + . errCtx (BundlerMissingLockFileCtx $ bundlerGemfile project) + . errHelp BundlerMissingLockFileHelp + . errDoc bundlerLockFileRationaleUrl + . errDoc rubyFossaDocUrl + $ do lockFile <- context "Retrieve Gemfile.lock" (Diag.fromMaybeText "No Gemfile.lock present in the project" (bundlerGemfileLock project)) graph <- context "Gemfile.lock analysis" . GemfileLock.analyze' $ lockFile pure $ @@ -156,35 +160,3 @@ analyzeGemfileLock project = do , dependencyGraphBreadth = Complete , dependencyManifestFiles = [lockFile] } - else do - warnOnErr AllDirectDeps - . warnOnErr MissingEdges - . errCtx (BundlerMissingLockFileCtx $ bundlerGemfile project) - . errHelp BundlerMissingLockFileHelp - . errDoc bundlerLockFileRationaleUrl - . errDoc rubyFossaDocUrl - $ do - lockFile <- context "Retrieve Gemfile.lock" (Diag.fromMaybeText "No Gemfile.lock present in the project" (bundlerGemfileLock project)) - graph <- context "Gemfile.lock analysis" . GemfileLock.analyze' $ lockFile - pure $ - DependencyResults - { dependencyGraph = graph - , dependencyGraphBreadth = Complete - , dependencyManifestFiles = [lockFile] - } - --- warnOnErr AllDirectDeps --- . warnOnErr MissingEdges --- . errCtx (BundlerMissingLockFileCtx $ bundlerGemfile project) --- . errHelp BundlerMissingLockFileHelp --- . errDoc bundlerLockFileRationaleUrl --- . errDoc rubyFossaDocUrl --- $ do --- lockFile <- context "Retrieve Gemfile.lock" (Diag.fromMaybeText "No Gemfile.lock present in the project" (bundlerGemfileLock project)) --- graph <- context "Gemfile.lock analysis" . GemfileLock.analyze' $ lockFile --- pure $ --- DependencyResults --- { dependencyGraph = graph --- , dependencyGraphBreadth = Complete --- , dependencyManifestFiles = [lockFile] --- } diff --git a/src/Strategy/Cocoapods.hs b/src/Strategy/Cocoapods.hs index 85efc6117d..40a8de1b6b 100644 --- a/src/Strategy/Cocoapods.hs +++ b/src/Strategy/Cocoapods.hs @@ -8,19 +8,18 @@ module Strategy.Cocoapods ( import App.Fossa.Analyze.LicenseAnalyze (LicenseAnalyzeProject, licenseAnalyzeProject) import App.Fossa.Analyze.Types (AnalyzeProject (analyzeProjectStaticOnly), analyzeProject) -import App.Fossa.Config.Analyze (StrictMode (..)) +import App.Types (Mode (..)) +import App.Util (guardStrictMode, populateWarningsForAnalysisMode) import Control.Applicative ((<|>)) import Control.Carrier.Diagnostics (errHelp) -import Control.Effect.Diagnostics (Diagnostics, context, errCtx, errDoc, warnOnErr, (<||>)) +import Control.Effect.Diagnostics (Diagnostics, context, errCtx, errDoc, (<||>)) import Control.Effect.Diagnostics qualified as Diag import Control.Effect.Reader (Reader, ask) import Data.Aeson (ToJSON) -import Data.Flag (Flag, fromFlag) import Data.Glob qualified as Glob import Data.List (find) import Data.List.Extra (singleton) import Data.Text (isSuffixOf) -import Diag.Common (MissingDeepDeps (MissingDeepDeps), MissingEdges (MissingEdges)) import Discovery.Filters (AllFilters) import Discovery.Simple (simpleDiscover) import Discovery.Walk ( @@ -112,57 +111,33 @@ mkProject project = , projectData = project } -getDeps :: (Has ReadFS sig m, Has Diagnostics sig m, Has Exec sig m, Has Logger sig m, Has (Reader (Flag StrictMode)) sig m) => CocoapodsProject -> m DependencyResults +getDeps :: (Has ReadFS sig m, Has Diagnostics sig m, Has Exec sig m, Has Logger sig m, Has (Reader Mode) sig m) => CocoapodsProject -> m DependencyResults getDeps project = do - strictMode <- ask @((Flag StrictMode)) - if fromFlag StrictMode strictMode - then - context "Cocoapods" $ - context - "Podfile.lock analysis" - ( errCtx MissingPodLockFileCtx - . errHelp MissingPodLockFileHelp - . errDoc refPodDocUrl - $ (analyzePodfileLock project) - ) - else - context "Cocoapods" $ - context - "Podfile.lock analysis" - ( warnOnErr MissingEdges - . warnOnErr MissingDeepDeps - . errCtx MissingPodLockFileCtx - . errHelp MissingPodLockFileHelp - . errDoc refPodDocUrl - $ (analyzePodfileLock project) - ) - <||> context "Podfile analysis" (analyzePodfile project) - -getDeps' :: (Has ReadFS sig m, Has Diagnostics sig m, Has (Reader (Flag StrictMode)) sig m) => CocoapodsProject -> m DependencyResults + mode <- ask + context "Cocoapods" $ + context + "Podfile.lock analysis" + ( populateWarningsForAnalysisMode mode + . errCtx MissingPodLockFileCtx + . errHelp MissingPodLockFileHelp + . errDoc refPodDocUrl + $ (analyzePodfileLock project) + ) + <||> guardStrictMode mode (context "Podfile analysis" (analyzePodfile project)) + +getDeps' :: (Has ReadFS sig m, Has Diagnostics sig m, Has (Reader Mode) sig m) => CocoapodsProject -> m DependencyResults getDeps' project = do - strictMode <- ask @((Flag StrictMode)) - if fromFlag StrictMode strictMode - then - context "Cocoapods" $ - context - "Strict mode Podfile.lock analysis" - ( errCtx MissingPodLockFileCtx - . errHelp MissingPodLockFileHelp - . errDoc refPodDocUrl - $ (analyzePodfileLockStatically project) - ) - else - context "Cocoapods" $ - context - "Podfile.lock analysis" - ( warnOnErr MissingEdges - . warnOnErr MissingDeepDeps - . errCtx MissingPodLockFileCtx - . errHelp MissingPodLockFileHelp - . errDoc refPodDocUrl - $ (analyzePodfileLockStatically project) - ) - <||> context "Podfile analysis" (analyzePodfile project) + mode <- ask + context "Cocoapods" $ + context + "Podfile.lock analysis" + ( populateWarningsForAnalysisMode mode + . errCtx MissingPodLockFileCtx + . errHelp MissingPodLockFileHelp + . errDoc refPodDocUrl + $ (analyzePodfileLockStatically project) + ) + <||> guardStrictMode mode (context "Podfile analysis" (analyzePodfile project)) analyzePodfile :: (Has ReadFS sig m, Has Diagnostics sig m) => CocoapodsProject -> m DependencyResults analyzePodfile project = do diff --git a/src/Strategy/Conda.hs b/src/Strategy/Conda.hs index 581564a1b1..0904173e0b 100644 --- a/src/Strategy/Conda.hs +++ b/src/Strategy/Conda.hs @@ -5,7 +5,8 @@ module Strategy.Conda ( ) where import App.Fossa.Analyze.Types (AnalyzeProject (analyzeProjectStaticOnly), analyzeProject) -import App.Fossa.Config.Analyze (StrictMode (..)) +import App.Types (Mode (..)) +import App.Util (guardStrictMode) import Control.Effect.Diagnostics ( Diagnostics, Has, @@ -14,7 +15,6 @@ import Control.Effect.Diagnostics ( ) import Control.Effect.Reader (Reader, ask) import Data.Aeson (ToJSON) -import Data.Flag (Flag, fromFlag) import Discovery.Filters (AllFilters) import Discovery.Simple (simpleDiscover) import Discovery.Walk ( @@ -75,12 +75,10 @@ mkProject project = -- There might be a dep with a version spec in an environment.yml file: i.e. conda+foo$1.2.*, and perhaps -- the same dep resolved to a known version in the users virtual environment: i.e. conda+'conda-forge':foo$1.2.4 (we get that from conda env create). -- If we combined the results then we would include both of those deps in the result, which is not correct behavior. -getDeps :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m, Has (Reader (Flag StrictMode)) sig m) => CondaProject -> m DependencyResults +getDeps :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m, Has (Reader Mode) sig m) => CondaProject -> m DependencyResults getDeps project = do - strictMode <- ask @((Flag StrictMode)) - if fromFlag StrictMode strictMode - then analyzeCondaEnvCreate project - else analyzeCondaEnvCreate project <||> analyzeEnvironmentYml project + mode <- ask + analyzeCondaEnvCreate project <||> guardStrictMode mode (analyzeEnvironmentYml project) analyzeCondaEnvCreate :: (Has Exec sig m, Has Diagnostics sig m) => CondaProject -> m DependencyResults analyzeCondaEnvCreate CondaProject{..} = do diff --git a/src/Strategy/Godep.hs b/src/Strategy/Godep.hs index eb76119186..e73d1be095 100644 --- a/src/Strategy/Godep.hs +++ b/src/Strategy/Godep.hs @@ -3,13 +3,13 @@ module Strategy.Godep ( ) where import App.Fossa.Analyze.Types (AnalyzeProject (analyzeProjectStaticOnly), analyzeProject) -import App.Fossa.Config.Analyze (StrictMode (..)) +import App.Types (Mode (..)) +import App.Util (guardStrictMode) import Control.Applicative ((<|>)) import Control.Effect.Diagnostics (Diagnostics, context, fatalText, (<||>)) import Control.Effect.Diagnostics qualified as Diag import Control.Effect.Reader (Reader, ask) import Data.Aeson (ToJSON) -import Data.Flag (Flag, fromFlag) import Discovery.Filters (AllFilters) import Discovery.Simple (simpleDiscover) import Discovery.Walk ( @@ -71,15 +71,12 @@ mkProject project = , projectData = project } -getDeps :: (Has ReadFS sig m, Has Exec sig m, Has Diagnostics sig m, Has (Reader (Flag StrictMode)) sig m) => GodepProject -> m DependencyResults +getDeps :: (Has ReadFS sig m, Has Exec sig m, Has Diagnostics sig m, Has (Reader Mode) sig m) => GodepProject -> m DependencyResults getDeps project = do - strictMode <- ask @((Flag StrictMode)) - if fromFlag StrictMode strictMode - then context "Strict mode Godep" $ context "Gopkg.lock analysis" (analyzeGopkgLock project) - else - context "Godep" $ - context "Gopkg.lock analysis" (analyzeGopkgLock project) - <||> context "Gopkg.toml analysis" (analyzeGopkgToml project) + mode <- ask + context "Godep" $ + context "Gopkg.lock analysis" (analyzeGopkgLock project) + <||> guardStrictMode mode (context "Gopkg.toml analysis" (analyzeGopkgToml project)) analyzeGopkgLock :: (Has ReadFS sig m, Has Exec sig m, Has Diagnostics sig m) => GodepProject -> m DependencyResults analyzeGopkgLock project = do diff --git a/src/Strategy/Gomodules.hs b/src/Strategy/Gomodules.hs index 7b14e87edb..cb889bf3cc 100644 --- a/src/Strategy/Gomodules.hs +++ b/src/Strategy/Gomodules.hs @@ -7,13 +7,14 @@ module Strategy.Gomodules ( ) where import App.Fossa.Analyze.Types (AnalyzeProject (analyzeProjectStaticOnly), analyzeProject) -import App.Fossa.Config.Analyze (ExperimentalAnalyzeConfig (useV3GoResolver), GoDynamicTactic (..), StrictMode (..)) +import App.Fossa.Config.Analyze (ExperimentalAnalyzeConfig (useV3GoResolver), GoDynamicTactic (..)) +import App.Types (Mode (..)) +import App.Util (guardStrictMode) import Control.Carrier.Diagnostics (warn) import Control.Effect.Diagnostics (Diagnostics, context, fatalText, recover, (<||>)) import Control.Effect.Reader (Reader, ask, asks) import Control.Monad (when) import Data.Aeson (ToJSON) -import Data.Flag (Flag, fromFlag) import Data.Text (Text) import Discovery.Filters (AllFilters) import Discovery.Simple (simpleDiscover) @@ -68,13 +69,10 @@ mkProject project = , projectData = project } -getDeps :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m, Has (Reader (Flag StrictMode)) sig m) => GomodulesProject -> GoDynamicTactic -> m DependencyResults +getDeps :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m, Has (Reader Mode) sig m) => GomodulesProject -> GoDynamicTactic -> m DependencyResults getDeps project goDynamicTactic = do - strictMode <- ask @((Flag StrictMode)) - (graph, graphBreadth) <- - if fromFlag StrictMode strictMode - then context "Strict mode Gomodules" dynamicAnalysis - else context "Gomodules" $ dynamicAnalysis <||> staticAnalysis + mode <- ask + (graph, graphBreadth) <- context "Gomodules" $ dynamicAnalysis <||> guardStrictMode mode staticAnalysis stdlib <- recover . context "Collect go standard library information" . listGoStdlibPackages $ gomodulesDir project pure $ DependencyResults diff --git a/src/Strategy/Maven.hs b/src/Strategy/Maven.hs index 87585cc6e1..5055c26c7e 100644 --- a/src/Strategy/Maven.hs +++ b/src/Strategy/Maven.hs @@ -7,14 +7,13 @@ module Strategy.Maven ( import App.Fossa.Analyze.LicenseAnalyze (LicenseAnalyzeProject, licenseAnalyzeProject) import App.Fossa.Analyze.Types (AnalyzeProject (analyzeProjectStaticOnly), analyzeProject) -import App.Fossa.Config.Analyze (StrictMode (..)) +import App.Types (Mode (..)) +import App.Util (guardStrictMode) import Control.Algebra (Has) -import Control.Carrier.Diagnostics qualified as Diag -import Control.Effect.Diagnostics (Diag (..), Diagnostics, context, warnOnErr, (<||>)) +import Control.Effect.Diagnostics (Diagnostics, context, warnOnErr, (<||>)) import Control.Effect.Lift (Lift) import Control.Effect.Reader (Reader, ask) import Data.Aeson (ToJSON) -import Data.Flag (Flag, fromFlag) import Data.Set (Set) import Data.Set qualified as Set import Data.Set.NonEmpty (nonEmpty, toSet) @@ -24,7 +23,6 @@ import Diag.Common (MissingDeepDeps (MissingDeepDeps), MissingEdges (MissingEdge import Discovery.Filters (AllFilters, MavenScopeFilters, mavenScopeFilterSet) import Discovery.Simple (simpleDiscover) import Effect.Exec (CandidateCommandEffs) -import Effect.Logger (Logger, logDebug, pretty) import Effect.ReadFS (ReadFS) import GHC.Generics (Generic) import Graphing (Graphing, gmap, shrinkRoots) @@ -35,7 +33,6 @@ import Strategy.Maven.PluginStrategy qualified as Plugin import Strategy.Maven.Pom qualified as Pom import Strategy.Maven.Pom.Closure (MavenProjectClosure (..)) import Strategy.Maven.Pom.Closure qualified as PomClosure -import Text.Pretty.Simple (pShow) import Types (BuildTarget (..), DependencyResults (..), DiscoveredProject (..), DiscoveredProjectType (MavenProjectType), FoundTargets (..), GraphBreadth (..)) discover :: @@ -80,22 +77,18 @@ getDeps :: , Has ReadFS sig m , CandidateCommandEffs sig m , Has (Reader MavenScopeFilters) sig m - , Has (Reader (Flag StrictMode)) sig m - , Has Logger sig m + , Has (Reader Mode) sig m ) => FoundTargets -> MavenProject -> m DependencyResults getDeps foundTargets (MavenProject closure) = do let submoduleTargets = submoduleTargetSet foundTargets - strictMode <- ask @((Flag StrictMode)) + mode <- ask (graph, graphBreadth) <- - if fromFlag StrictMode strictMode - then do - logDebug "In switch case for strict mode (getDeps) &&&&&&&&&" - context "Maven" $ getDepsDynamicAnalysis submoduleTargets closure - else context "Maven" $ getDepsDynamicAnalysis submoduleTargets closure <||> getStaticAnalysis submoduleTargets closure - -- (graph, graphBreadth) <- context "Maven" $ getDepsDynamicAnalysis submoduleTargets closure <||> getStaticAnalysis submoduleTargets closure + context "Maven" $ + getDepsDynamicAnalysis submoduleTargets closure + <||> guardStrictMode mode (getStaticAnalysis submoduleTargets closure) pure $ DependencyResults { dependencyGraph = graph @@ -107,8 +100,6 @@ getDeps' :: ( Has (Lift IO) sig m , Has Diagnostics sig m , Has (Reader MavenScopeFilters) sig m - , Has (Reader (Flag StrictMode)) sig m - , Has Logger sig m ) => FoundTargets -> MavenProject -> @@ -129,34 +120,21 @@ getDepsDynamicAnalysis :: , Has ReadFS sig m , CandidateCommandEffs sig m , Has (Reader MavenScopeFilters) sig m - , Has (Reader (Flag StrictMode)) sig m - , Has Logger sig m + , Has (Reader Mode) sig m ) => Set Text -> MavenProjectClosure -> m (Graphing Dependency, GraphBreadth) getDepsDynamicAnalysis submoduleTargets closure = do let allSubmodules = PomClosure.closureSubmodules closure - strictMode <- ask @((Flag StrictMode)) - logDebug $ "Is in strict mode for maven strategy ------------- " <> pretty (pShow strictMode) + mode <- ask + let errorInfo = case mode of + Strict -> id + NonStrict -> warnOnErr MissingEdges . warnOnErr MissingDeepDeps (graph, graphBreadth) <- - if fromFlag StrictMode strictMode - then do - logDebug "In switch case for strict mode ===========" - context "Dynamic Analysis" $ getDepsPlugin closure <||> getDepsTreeCmd closure <||> getDepsPluginLegacy closure - else do - logDebug "NOT IN STRICT MODE ===========" - context "Dynamic Analysis" - $ warnOnErr MissingEdges - . warnOnErr MissingDeepDeps - $ (getDepsPlugin closure <||> getDepsTreeCmd closure <||> getDepsPluginLegacy closure) - - logDebug $ "Is in strict mode for maven strategy ------------- " <> pretty (pShow strictMode) - -- (graph, graphBreadth) <- - -- context "Dynamic Analysis" - -- $ warnOnErr MissingEdges - -- . warnOnErr MissingDeepDeps - -- $ (getDepsPlugin closure <||> getDepsTreeCmd closure <||> getDepsPluginLegacy closure) + context "Dynamic Analysis" $ + errorInfo (getDepsPlugin closure <||> guardStrictMode mode (getDepsTreeCmd closure <||> getDepsPluginLegacy closure)) + filteredGraph <- applyMavenFilters submoduleTargets allSubmodules graph pure (withoutProjectAsDep filteredGraph, graphBreadth) where @@ -170,42 +148,28 @@ getDepsPlugin :: ( CandidateCommandEffs sig m , Has (Lift IO) sig m , Has ReadFS sig m - , Has Diagnostics sig m ) => MavenProjectClosure -> m (Graphing MavenDependency, GraphBreadth) -getDepsPlugin closure = do - Diag.fatal MissingDeepDeps - --- context "Plugin analysis" (Plugin.analyze' . parent $ PomClosure.closurePath closure) +getDepsPlugin closure = context "Plugin analysis" (Plugin.analyze' . parent $ PomClosure.closurePath closure) getDepsPluginLegacy :: ( CandidateCommandEffs sig m , Has (Lift IO) sig m , Has ReadFS sig m - , Has Diagnostics sig m - , Has Logger sig m ) => MavenProjectClosure -> m (Graphing MavenDependency, GraphBreadth) -getDepsPluginLegacy closure = do - logDebug $ "In plugin command ++++++++ " - -- Diag.fatal MissingDeepDeps - context "Legacy Plugin analysis" (Plugin.analyzeLegacy' . parent $ PomClosure.closurePath closure) +getDepsPluginLegacy closure = context "Legacy Plugin analysis" (Plugin.analyzeLegacy' . parent $ PomClosure.closurePath closure) getDepsTreeCmd :: ( Has (Lift IO) sig m , Has ReadFS sig m , CandidateCommandEffs sig m - , Has Diagnostics sig m - , Has Logger sig m ) => MavenProjectClosure -> m (Graphing MavenDependency, GraphBreadth) -getDepsTreeCmd closure = do - logDebug $ "In deps tree comand ++++++++ " - Diag.fatal MissingDeepDeps - +getDepsTreeCmd closure = context "Dynamic analysis" $ DepTreeCmd.analyze . parent $ PomClosure.closurePath closure diff --git a/src/Strategy/Pub.hs b/src/Strategy/Pub.hs index 2d9b991319..63522a3e8d 100644 --- a/src/Strategy/Pub.hs +++ b/src/Strategy/Pub.hs @@ -2,13 +2,13 @@ module Strategy.Pub (discover) where import App.Fossa.Analyze.Types (AnalyzeProject (analyzeProjectStaticOnly), analyzeProject) -import App.Fossa.Config.Analyze (StrictMode (..), strictMode) +import App.Types (Mode (..)) +import App.Util (guardStrictMode) import Control.Carrier.Diagnostics (errDoc, errHelp) import Control.Effect.Diagnostics (Diagnostics, errCtx, fatalText, recover, warnOnErr, (<||>)) import Control.Effect.Reader (Reader, ask) import Control.Monad (void) import Data.Aeson (ToJSON) -import Data.Flag (Flag, fromFlag) import Diag.Common ( MissingDeepDeps (MissingDeepDeps), MissingEdges (MissingEdges), @@ -17,10 +17,10 @@ import Discovery.Filters (AllFilters) import Discovery.Simple (simpleDiscover) import Discovery.Walk (WalkStep (WalkContinue), findFileNamed, walkWithFilters') import Effect.Exec (Exec, Has) -import Effect.Logger (Logger, logDebug) +import Effect.Logger (Logger) import Effect.ReadFS (ReadFS) import GHC.Generics (Generic) -import Path (Abs, Dir, File, Path) +import Path import Strategy.Dart.Errors (PubspecLimitation (..), refPubDocUrl) import Strategy.Dart.PubDeps (analyzeDepsCmd) import Strategy.Dart.PubSpec (analyzePubSpecFile) @@ -66,39 +66,14 @@ mkProject project = , projectData = project } -getDeps :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m, Has Logger sig m, Has (Reader (Flag StrictMode)) sig m) => PubProject -> m DependencyResults +getDeps :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m, Has Logger sig m, Has (Reader Mode) sig m) => PubProject -> m DependencyResults getDeps project = do - strictMode <- ask @((Flag StrictMode)) + mode <- ask (graph, graphBreadth) <- case pubLock project of - Just lockFile -> do - if fromFlag StrictMode strictMode - then do - logDebug "Pub Analysis in strict mode" - analyzeDepsCmd lockFile (pubSpecDir project) - else do - logDebug "Pub Analysis NOT in strict mode" - analyzeDepsCmd lockFile (pubSpecDir project) <||> analyzePubLockFile lockFile + Just lockFile -> analyzeDepsCmd lockFile (pubSpecDir project) <||> guardStrictMode mode (analyzePubLockFile lockFile) Nothing -> do - if fromFlag StrictMode strictMode - then do - logDebug "Pub Analysis (NOTHING) in strict mode" - analyzePubSpecFile $ pubSpec project - else do - logDebug "Pub Analysis (NOTHING) NOT in strict mode" - void . recover - $ warnOnErr MissingDeepDeps - . warnOnErr MissingEdges - $ errCtx PubspecLimitationCtx - $ errHelp PubspecLimitationHelp - $ errDoc refPubDocUrl (fatalText "Missing pubspec.lock file") - analyzePubSpecFile $ pubSpec project - -- void . recover - -- $ warnOnErr MissingDeepDeps - -- . warnOnErr MissingEdges - -- $ errCtx PubspecLimitationCtx - -- $ errHelp PubspecLimitationHelp - -- $ errDoc refPubDocUrl (fatalText "Missing pubspec.lock file") - -- analyzePubSpecFile $ pubSpec project + _ <- applyMissingPubSpecWarnings + analyzePubSpecFile (pubSpec project) pure $ DependencyResults { dependencyGraph = graph @@ -106,30 +81,25 @@ getDeps project = do , dependencyManifestFiles = [pubSpec project] } -getDeps' :: (Has ReadFS sig m, Has Diagnostics sig m, Has Logger sig m, Has (Reader (Flag StrictMode)) sig m) => PubProject -> m DependencyResults +getDeps' :: (Has ReadFS sig m, Has Diagnostics sig m, Has Logger sig m) => PubProject -> m DependencyResults getDeps' project = do - strictMode <- ask @((Flag StrictMode)) (graph, graphBreadth) <- case pubLock project of Just lockFile -> analyzePubLockFile lockFile Nothing -> do - if fromFlag StrictMode strictMode - then do - void $ - errCtx PubspecLimitationCtx $ - errHelp PubspecLimitationHelp $ - errDoc refPubDocUrl (fatalText "Missing pubspec.lock file") - analyzePubSpecFile $ pubSpec project - else do - void . recover - $ warnOnErr MissingDeepDeps - . warnOnErr MissingEdges - $ errCtx PubspecLimitationCtx - $ errHelp PubspecLimitationHelp - $ errDoc refPubDocUrl (fatalText "Missing pubspec.lock file") - analyzePubSpecFile $ pubSpec project + _ <- applyMissingPubSpecWarnings + analyzePubSpecFile (pubSpec project) pure $ DependencyResults { dependencyGraph = graph , dependencyGraphBreadth = graphBreadth , dependencyManifestFiles = [pubSpec project] } + +applyMissingPubSpecWarnings :: (Has Diagnostics sig m) => m () +applyMissingPubSpecWarnings = + void . recover $ + warnOnErr MissingDeepDeps $ + warnOnErr MissingEdges $ + errCtx PubspecLimitationCtx $ + errHelp PubspecLimitationHelp $ + errDoc refPubDocUrl (fatalText "Missing pubspec.lock file") diff --git a/src/Strategy/Scala.hs b/src/Strategy/Scala.hs index 5ffb816924..52faac7b2e 100644 --- a/src/Strategy/Scala.hs +++ b/src/Strategy/Scala.hs @@ -13,14 +13,13 @@ module Strategy.Scala ( ) where import App.Fossa.Analyze.Types (AnalyzeProject (analyzeProjectStaticOnly), analyzeProject) -import App.Fossa.Config.Analyze (StrictMode (..)) +import App.Types (Mode (..)) import Control.Carrier.Diagnostics (errDoc) import Control.Effect.Diagnostics (Diagnostics, errCtx, errHelp, fatalText, fromMaybeText, recover, warnOnErr, (<||>)) import Control.Effect.Reader (Reader, ask) import Control.Effect.Stack (context) import Data.Aeson (KeyValue ((.=)), ToJSON (toJSON), object) import Data.ByteString.Lazy (ByteString) -import Data.Flag (Flag, fromFlag) import Data.Maybe (fromMaybe, listToMaybe, mapMaybe) import Data.String.Conversion (ConvertUtf8 (decodeUtf8), toString, toText) import Data.Text (Text) @@ -129,12 +128,12 @@ mkProject (ScalaProject sbtBuildDir sbtTreeJson closure) = , projectData = ScalaProject sbtBuildDir sbtTreeJson closure } -getDeps :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m, Has Logger sig m, Has (Reader (Flag StrictMode)) sig m) => ScalaProject -> m DependencyResults +getDeps :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m, Has Logger sig m, Has (Reader Mode) sig m) => ScalaProject -> m DependencyResults getDeps project = do - strictMode <- ask @((Flag StrictMode)) - if fromFlag StrictMode strictMode - then analyzeWithDepTreeJson project - else + mode <- ask + case mode of + Strict -> analyzeWithDepTreeJson project + NonStrict -> warnOnErr MissingDeepDeps (analyzeWithDepTreeJson project <||> analyzeWithSbtDepTree project) <||> analyzeWithPoms project diff --git a/test/App/Fossa/AnalyzeSpec.hs b/test/App/Fossa/AnalyzeSpec.hs index f10620b51a..2cc5387e18 100644 --- a/test/App/Fossa/AnalyzeSpec.hs +++ b/test/App/Fossa/AnalyzeSpec.hs @@ -2,7 +2,7 @@ module App.Fossa.AnalyzeSpec (spec) where import App.Fossa.Analyze.Discover (DiscoverFunc, discoverFuncs) import App.Fossa.Config.Analyze (ExperimentalAnalyzeConfig) -import App.Types (OverrideDynamicAnalysisBinary) +import App.Types (Mode, OverrideDynamicAnalysisBinary) import Control.Carrier.Debug (DebugC) import Control.Carrier.Diagnostics (DiagnosticsC) import Control.Carrier.Reader (ReaderC) @@ -15,7 +15,7 @@ import Effect.ReadFS (ReadFSIOC) import Test.Hspec (Spec, describe, it, shouldBe) import Type.Operator (type ($)) -type SomeMonad = TelemetryC $ ReaderC OverrideDynamicAnalysisBinary $ ReaderC ExperimentalAnalyzeConfig $ ReaderC MavenScopeFilters $ ReaderC AllFilters $ DebugC $ DiagnosticsC $ LoggerC $ ExecIOC $ ReadFSIOC $ StackC IO +type SomeMonad = TelemetryC $ ReaderC OverrideDynamicAnalysisBinary $ ReaderC ExperimentalAnalyzeConfig $ ReaderC MavenScopeFilters $ ReaderC Mode $ ReaderC AllFilters $ DebugC $ DiagnosticsC $ LoggerC $ ExecIOC $ ReadFSIOC $ StackC IO spec :: Spec spec = diff --git a/test/Test/Fixtures.hs b/test/Test/Fixtures.hs index 300f9f81c3..abbcf31bff 100644 --- a/test/Test/Fixtures.hs +++ b/test/Test/Fixtures.hs @@ -67,7 +67,7 @@ import App.Fossa.Lernie.Types (GrepOptions (..), OrgWideCustomLicenseConfigPolic import App.Fossa.Reachability.Types (CallGraphAnalysis (NoCallGraphAnalysis), SourceUnitReachability (..)) import App.Fossa.VSI.Types qualified as VSI import App.Fossa.VendoredDependency (VendoredDependency (..)) -import App.Types (OverrideDynamicAnalysisBinary (..)) +import App.Types (Mode (..), OverrideDynamicAnalysisBinary (..)) import App.Types qualified as App import Control.Effect.FossaApiClient qualified as App import Control.Timeout (Duration (MilliSeconds)) @@ -582,6 +582,7 @@ standardAnalyzeConfig = , ANZ.allowedTacticTypes = Any , ANZ.reachabilityConfig = mempty , ANZ.withoutDefaultFilters = toFlag WithoutDefaultFilters False + , ANZ.mode = NonStrict } sampleJarParsedContent :: Text From 2fdea926e3481e6e534cf866ae48214f8247584c Mon Sep 17 00:00:00 2001 From: Jeffrey Huynh Date: Fri, 30 Aug 2024 15:10:12 -0700 Subject: [PATCH 3/8] Update docs --- Changelog.md | 4 ++++ docs/references/subcommands/analyze.md | 11 ++++++----- src/App/Fossa/Analyze.hs | 5 ----- src/App/Fossa/Config/Analyze.hs | 4 ++-- src/App/Util.hs | 10 +++------- src/Strategy/Bundler.hs | 4 ---- src/Strategy/Cocoapods.hs | 13 +++++++++---- src/Strategy/Pub.hs | 2 +- 8 files changed, 25 insertions(+), 28 deletions(-) diff --git a/Changelog.md b/Changelog.md index 4989fe88c8..b8ed153c8a 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,9 @@ # FOSSA CLI Changelog +## 3.9.32 + +- `--strict`: Users can now enable strict mode for analysis. ([#1463](https://github.com/fossas/fossa-cli/pull/1463)) + ## 3.9.31 - Resolve an issue parsing toml configuration files. ([#1459](https://github.com/fossas/fossa-cli/pull/1459)) diff --git a/docs/references/subcommands/analyze.md b/docs/references/subcommands/analyze.md index 41c4582943..5823683a6d 100644 --- a/docs/references/subcommands/analyze.md +++ b/docs/references/subcommands/analyze.md @@ -127,11 +127,12 @@ We support the following archive formats: In addition to the [standard flags](#specifying-fossa-project-details), the analyze command supports the following additional strategy flags: -| Name | Description | -| --------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| [`--detect-vendored`](./analyze/detect-vendored.md) | Enable the vendored source identification engine. For more information, see the [C and C++ overview](../strategies/languages/c-cpp/c-cpp.md). | -| [`--detect-dynamic './some-binary`](./analyze/detect-dynamic.md) | Analyze the binary at the provided path for dynamically linked dependencies. For more information, see the [C and C++ overview](../strategies/languages/c-cpp/c-cpp.md). | -| [`--static-only-analysis`](../strategies/README.md#static-and-dynamic-strategies) | Do not use third-party tools when analyzing projects. | +| Name | Description | +| --------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| [`--detect-vendored`](./analyze/detect-vendored.md) | Enable the vendored source identification engine. For more information, see the [C and C++ overview](../strategies/languages/c-cpp/c-cpp.md). | +| [`--detect-dynamic './some-binary`](./analyze/detect-dynamic.md) | Analyze the binary at the provided path for dynamically linked dependencies. For more information, see the [C and C++ overview](../strategies/languages/c-cpp/c-cpp.md). | +| [`--static-only-analysis`](../strategies/README.md#static-and-dynamic-strategies) | Do not use third-party tools when analyzing projects. | +| `--strict` | Enable strict analysis to enforce that the first analysis strategy within a [strategy type](../strategies/README.md#strategies-by-type) passes. Fallback strategies are not allowed in strict mode. | ### Experimental Options diff --git a/src/App/Fossa/Analyze.hs b/src/App/Fossa/Analyze.hs index aeebc76397..643f8e29fd 100644 --- a/src/App/Fossa/Analyze.hs +++ b/src/App/Fossa/Analyze.hs @@ -364,7 +364,6 @@ analyze cfg = Diag.context "fossa-analyze" $ do pure Nothing else Diag.context "first-party-scans" . runStickyLogger SevInfo $ runFirstPartyScan basedir maybeApiOpts cfg let firstPartyScanResults = join . resultToMaybe $ maybeFirstPartyScanResults - -- logDebug $ "Is in strict mode ------------- " <> pretty (show strictMode) let discoveryFilters = if fromFlag NoDiscoveryExclusion noDiscoveryExclusion then mempty else filters (projectScans, ()) <- Diag.context "discovery/analysis tasks" @@ -599,7 +598,3 @@ updateProgress Progress{..} = <> " Completed" <> " ]" ) - --- analyzeStrictModeGuard :: Has Diagnostics sig m => Mode -> m a -> m a --- analyzeStrictModeGuard Strict _ = fatal "Strict mode enabled, skipping other strategies" --- analyzeStrictModeGuard NonStrict action = action diff --git a/src/App/Fossa/Config/Analyze.hs b/src/App/Fossa/Config/Analyze.hs index 3ab926a32a..b7a547e1c5 100644 --- a/src/App/Fossa/Config/Analyze.hs +++ b/src/App/Fossa/Config/Analyze.hs @@ -24,7 +24,6 @@ module App.Fossa.Config.Analyze ( StaticOnlyTactics (..), WithoutDefaultFilters (..), StrictMode (..), - Mode (..), mkSubCommand, loadConfig, cliParser, @@ -339,7 +338,7 @@ cliParser = <*> optional (strOption (applyFossaStyle <> long "fossa-deps-file" <> helpDoc fossaDepsFileHelp <> metavar "FILEPATH")) <*> flagOpt StaticOnlyTactics (applyFossaStyle <> long "static-only-analysis" <> stringToHelpDoc "Only analyze the project using static strategies.") <*> withoutDefaultFilterParser fossaAnalyzeDefaultFilterDocUrl - <*> flagOpt StrictMode (applyFossaStyle <> long "strict" <> stringToHelpDoc "Strict mode") + <*> flagOpt StrictMode (applyFossaStyle <> long "strict" <> stringToHelpDoc "Enables strict analysis to enforce that the first analysis strategy within a strategy type passes. Fallback strategies are not allowed in strict mode.") where fossaDepsFileHelp :: Maybe (Doc AnsiStyle) fossaDepsFileHelp = @@ -348,6 +347,7 @@ cliParser = [ "Path to fossa-deps file including filename" , boldItalicized "Default:" <> " fossa-deps.{yaml|yml|json}" ] + branchHelp :: Maybe (Doc AnsiStyle) branchHelp = Just . formatDoc $ diff --git a/src/App/Util.hs b/src/App/Util.hs index 06a10b12c1..9460d26c1e 100644 --- a/src/App/Util.hs +++ b/src/App/Util.hs @@ -6,17 +6,15 @@ module App.Util ( validateDir, validateFile, guardStrictMode, - populateWarningsForAnalysisMode, FileAncestry (..), ) where import App.Types import Control.Algebra (Has) import Control.Carrier.Diagnostics (Diagnostics) -import Control.Effect.Diagnostics (fatalText, warnOnErr) +import Control.Effect.Diagnostics (fatalText) import Control.Monad (unless) import Data.String.Conversion (ToText (..)) -import Diag.Common (MissingDeepDeps (..), MissingEdges (..)) import GHC.Generics (Generic) import Path (Abs, Dir, File, Path, Rel, SomeBase (..), toFilePath, ()) import Path.Extra (tryMakeRelative) @@ -56,10 +54,8 @@ ancestryDerived parent dir file = do rel <- ancestryDirect dir file pure $ fileAncestryPath parent rel +-- | Guards analysis strategies depending on the mode. On strict mode, emit an error to end the chain of diagnostics to prevent +-- other strategies to be executed. On non-strict mode, allow fallback strategies to be executed. guardStrictMode :: Has Diagnostics sig m => Mode -> m a -> m a guardStrictMode Strict _ = fatalText "Strict mode enabled, skipping other strategies" guardStrictMode NonStrict action = action - -populateWarningsForAnalysisMode :: (Has Diagnostics sig m) => Mode -> m a -> m a -populateWarningsForAnalysisMode Strict = id -populateWarningsForAnalysisMode NonStrict = warnOnErr MissingEdges . warnOnErr MissingDeepDeps diff --git a/src/Strategy/Bundler.hs b/src/Strategy/Bundler.hs index 1933815c2f..afbe65057f 100644 --- a/src/Strategy/Bundler.hs +++ b/src/Strategy/Bundler.hs @@ -11,7 +11,6 @@ import App.Fossa.Analyze.LicenseAnalyze ( LicenseAnalyzeProject (licenseAnalyzeProject), ) import App.Fossa.Analyze.Types (AnalyzeProject (analyzeProjectStaticOnly), analyzeProject) -import App.Fossa.Config.Analyze (StrictMode (..)) import App.Types (Mode (..)) import App.Util (guardStrictMode) import Control.Effect.Diagnostics ( @@ -20,14 +19,12 @@ import Control.Effect.Diagnostics ( errCtx, errDoc, errHelp, - fatal, warnOnErr, (<||>), ) import Control.Effect.Diagnostics qualified as Diag import Control.Effect.Reader (Reader, ask) import Data.Aeson (ToJSON) -import Data.Flag (Flag, fromFlag) import Data.Glob as Glob (toGlob, ()) import Data.Text (isSuffixOf) import Diag.Common (AllDirectDeps (AllDirectDeps), MissingEdges (MissingEdges)) @@ -40,7 +37,6 @@ import Discovery.Walk ( walkWithFilters', ) import Effect.Exec (Exec, Has) -import Effect.Logger (Logger, logDebug) import Effect.ReadFS (ReadFS, readContentsParser) import GHC.Generics (Generic) import Path (Abs, Dir, File, Path, toFilePath) diff --git a/src/Strategy/Cocoapods.hs b/src/Strategy/Cocoapods.hs index 40a8de1b6b..2bc01d28ed 100644 --- a/src/Strategy/Cocoapods.hs +++ b/src/Strategy/Cocoapods.hs @@ -9,10 +9,10 @@ module Strategy.Cocoapods ( import App.Fossa.Analyze.LicenseAnalyze (LicenseAnalyzeProject, licenseAnalyzeProject) import App.Fossa.Analyze.Types (AnalyzeProject (analyzeProjectStaticOnly), analyzeProject) import App.Types (Mode (..)) -import App.Util (guardStrictMode, populateWarningsForAnalysisMode) +import App.Util (guardStrictMode) import Control.Applicative ((<|>)) import Control.Carrier.Diagnostics (errHelp) -import Control.Effect.Diagnostics (Diagnostics, context, errCtx, errDoc, (<||>)) +import Control.Effect.Diagnostics (Diagnostics, context, errCtx, errDoc, warnOnErr, (<||>)) import Control.Effect.Diagnostics qualified as Diag import Control.Effect.Reader (Reader, ask) import Data.Aeson (ToJSON) @@ -20,6 +20,7 @@ import Data.Glob qualified as Glob import Data.List (find) import Data.List.Extra (singleton) import Data.Text (isSuffixOf) +import Diag.Common (MissingDeepDeps (..), MissingEdges (..)) import Discovery.Filters (AllFilters) import Discovery.Simple (simpleDiscover) import Discovery.Walk ( @@ -117,7 +118,7 @@ getDeps project = do context "Cocoapods" $ context "Podfile.lock analysis" - ( populateWarningsForAnalysisMode mode + ( populateErrorInfo mode . errCtx MissingPodLockFileCtx . errHelp MissingPodLockFileHelp . errDoc refPodDocUrl @@ -131,7 +132,7 @@ getDeps' project = do context "Cocoapods" $ context "Podfile.lock analysis" - ( populateWarningsForAnalysisMode mode + ( populateErrorInfo mode . errCtx MissingPodLockFileCtx . errHelp MissingPodLockFileHelp . errDoc refPodDocUrl @@ -171,3 +172,7 @@ analyzePodfileLockStatically project = do , dependencyGraphBreadth = Complete , dependencyManifestFiles = [lockFile] } + +populateErrorInfo :: (Has Diagnostics sig m) => Mode -> m a -> m a +populateErrorInfo Strict = id +populateErrorInfo NonStrict = warnOnErr MissingEdges . warnOnErr MissingDeepDeps diff --git a/src/Strategy/Pub.hs b/src/Strategy/Pub.hs index 63522a3e8d..fc82d512cd 100644 --- a/src/Strategy/Pub.hs +++ b/src/Strategy/Pub.hs @@ -20,7 +20,7 @@ import Effect.Exec (Exec, Has) import Effect.Logger (Logger) import Effect.ReadFS (ReadFS) import GHC.Generics (Generic) -import Path +import Path (Abs, Dir, File, Path) import Strategy.Dart.Errors (PubspecLimitation (..), refPubDocUrl) import Strategy.Dart.PubDeps (analyzeDepsCmd) import Strategy.Dart.PubSpec (analyzePubSpecFile) From e8c9444f794722361dae1ebad7c03ce5243986f7 Mon Sep 17 00:00:00 2001 From: Jeffrey Huynh Date: Fri, 30 Aug 2024 15:23:04 -0700 Subject: [PATCH 4/8] fix tests --- integration-test/Analysis/FixtureUtils.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/integration-test/Analysis/FixtureUtils.hs b/integration-test/Analysis/FixtureUtils.hs index ee83834496..aef6dc2dd8 100644 --- a/integration-test/Analysis/FixtureUtils.hs +++ b/integration-test/Analysis/FixtureUtils.hs @@ -16,7 +16,7 @@ module Analysis.FixtureUtils ( import App.Fossa.Analyze.Types (AnalyzeProject (analyzeProject)) import App.Fossa.Config.Analyze (ExperimentalAnalyzeConfig (ExperimentalAnalyzeConfig), GoDynamicTactic (GoModulesBasedTactic)) -import App.Types (OverrideDynamicAnalysisBinary) +import App.Types (Mode (..), OverrideDynamicAnalysisBinary) import Control.Carrier.Debug (IgnoreDebugC, ignoreDebug) import Control.Carrier.Diagnostics (DiagnosticsC, runDiagnostics) import Control.Carrier.Finally (FinallyC, runFinally) @@ -118,6 +118,7 @@ type TestC m = $ ReaderC AllFilters $ ReaderC MavenScopeFilters $ ReaderC ExperimentalAnalyzeConfig + $ ReaderC Mode $ FinallyC $ StackC $ IgnoreTelemetryC m @@ -134,6 +135,7 @@ testRunner f env = & runReader (mempty :: AllFilters) & runReader (MavenScopeIncludeFilters mempty) & runReader (ExperimentalAnalyzeConfig Nothing GoModulesBasedTactic False) + & runReader (NonStrict :: Mode) & runFinally & runStack & withoutTelemetry From 502768a0c76ab1fd73f6548d9e06f49e2a9b26ba Mon Sep 17 00:00:00 2001 From: Jeffrey Huynh Date: Fri, 6 Sep 2024 12:33:30 -0700 Subject: [PATCH 5/8] address pr comments --- docs/references/subcommands/analyze.md | 46 +++++++++---------- integration-test/Analysis/FixtureUtils.hs | 2 +- src/App/Fossa/Analyze.hs | 1 - src/App/Fossa/Config/Analyze.hs | 2 +- .../Fossa/Container/Sources/DockerArchive.hs | 2 +- src/App/Types.hs | 3 ++ src/Effect/Exec.hs | 6 ++- src/Strategy/Bundler.hs | 6 +-- src/Strategy/Cocoapods.hs | 10 ++-- src/Strategy/Conda.hs | 4 +- src/Strategy/Godep.hs | 4 +- src/Strategy/Gomodules.hs | 5 +- src/Strategy/Maven.hs | 14 +++--- src/Strategy/Nim.hs | 6 +-- src/Strategy/Pub.hs | 14 +++--- src/Strategy/Python/Pipenv.hs | 6 +-- src/Strategy/Python/Setuptools.hs | 6 +-- src/Strategy/Scala.hs | 3 +- 18 files changed, 72 insertions(+), 68 deletions(-) diff --git a/docs/references/subcommands/analyze.md b/docs/references/subcommands/analyze.md index 5823683a6d..a1c9de6161 100644 --- a/docs/references/subcommands/analyze.md +++ b/docs/references/subcommands/analyze.md @@ -13,26 +13,26 @@ For supported command-line flags, use `fossa analyze --help` In addition to the [usual FOSSA project flags](#common-fossa-project-flags) supported by all commands, the analyze command supports the following FOSSA-project-related flags: -| Name | Short | Description | -| ------------------------------------- | ----- | ----------------------------------------------------------------------------------- | -| `--title 'some title'` | `-t` | Set the title of the FOSSA project | -| `--branch 'some branch'` | `-b` | Override the detected FOSSA project branch | -| `--project-url 'https://example.com'` | `-P` | Add a URL to the FOSSA project | -| `--jira-project-key 'some-key'` | `-j` | Add a Jira project key to the FOSSA project | -| `--link 'https://example.com'` | `-L` | Attach a link to the current FOSSA build | -| `--team 'some team'` | `-T` | Specify a team within your FOSSA organization | -| `--policy 'some policy'` | | Assign a specific FOSSA policy to this project. Mutually excludes `--policy-id`. | -| `--policy-id 'some policy id'` | | Assign a specific FOSSA policy to this project by id. Mutually excludes `--policy`. | -| `--project-label` | | assign up to 5 labels to the project | -| `--release-group-name 'MY_RG'` | | add the project to this release group (also requires `--release-group-release`) | -| `--release-group-release 'MY_RELEASE'`| | add the project to this release version within the release group | +| Name | Short | Description | +|----------------------------------------|-------|-------------------------------------------------------------------------------------| +| `--title 'some title'` | `-t` | Set the title of the FOSSA project | +| `--branch 'some branch'` | `-b` | Override the detected FOSSA project branch | +| `--project-url 'https://example.com'` | `-P` | Add a URL to the FOSSA project | +| `--jira-project-key 'some-key'` | `-j` | Add a Jira project key to the FOSSA project | +| `--link 'https://example.com'` | `-L` | Attach a link to the current FOSSA build | +| `--team 'some team'` | `-T` | Specify a team within your FOSSA organization | +| `--policy 'some policy'` | | Assign a specific FOSSA policy to this project. Mutually excludes `--policy-id`. | +| `--policy-id 'some policy id'` | | Assign a specific FOSSA policy to this project by id. Mutually excludes `--policy`. | +| `--project-label` | | assign up to 5 labels to the project | +| `--release-group-name 'MY_RG'` | | add the project to this release group (also requires `--release-group-release`) | +| `--release-group-release 'MY_RELEASE'` | | add the project to this release version within the release group | ### Filtering Paths and Targets The paths and targets filtering options allow you to specify the exact targets which be should be scanned. | Name | Description | -| -------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | +|----------------------------------|--------------------------------------------------------------------------------------------------------------------------| | `--only-target` | Only scan these targets. See [targets.only](../files/fossa-yml.md#targets.only) in the fossa.yml spec. | | `--exclude-target` | Exclude these targets from scanning. See [targets.exclude](../files/fossa-yml.md#targets.exclude) in the fossa.yml spec. | | `--only-path` | Only scan these paths. See [paths.only](../files/fossa-yml.md#paths.only) in the fossa.yml spec. | @@ -76,7 +76,7 @@ fossa analyze --fossa-deps-file /path/to/file The Vendored Dependencies feature allows you to scan for licenses directly in your code. For more information, please see the [Vendored Dependencies documentation](../../features/vendored-dependencies.md). | Name | Description | -| ----------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +|-------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `--force-vendored-dependency-scan-method` | Force the vendored dependency scan method. The options are 'CLILicenseScan' or 'ArchiveUpload'. 'CLILicenseScan' is usually the default unless your organization has overridden this. | | `--force-vendored-dependency-rescans` | Force vendored dependencies to be rescanned even if the revision has been previously analyzed by FOSSA. This currently only works for CLI-side license scans. | @@ -127,12 +127,12 @@ We support the following archive formats: In addition to the [standard flags](#specifying-fossa-project-details), the analyze command supports the following additional strategy flags: -| Name | Description | -| --------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| [`--detect-vendored`](./analyze/detect-vendored.md) | Enable the vendored source identification engine. For more information, see the [C and C++ overview](../strategies/languages/c-cpp/c-cpp.md). | -| [`--detect-dynamic './some-binary`](./analyze/detect-dynamic.md) | Analyze the binary at the provided path for dynamically linked dependencies. For more information, see the [C and C++ overview](../strategies/languages/c-cpp/c-cpp.md). | -| [`--static-only-analysis`](../strategies/README.md#static-and-dynamic-strategies) | Do not use third-party tools when analyzing projects. | -| `--strict` | Enable strict analysis to enforce that the first analysis strategy within a [strategy type](../strategies/README.md#strategies-by-type) passes. Fallback strategies are not allowed in strict mode. | +| Name | Description | +|-----------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [`--detect-vendored`](./analyze/detect-vendored.md) | Enable the vendored source identification engine. For more information, see the [C and C++ overview](../strategies/languages/c-cpp/c-cpp.md). | +| [`--detect-dynamic './some-binary`](./analyze/detect-dynamic.md) | Analyze the binary at the provided path for dynamically linked dependencies. For more information, see the [C and C++ overview](../strategies/languages/c-cpp/c-cpp.md). | +| [`--static-only-analysis`](../strategies/README.md#static-and-dynamic-strategies) | Do not use third-party tools when analyzing projects. | +| `--strict` | Enforces strict analysis to ensure the most accurate results for a strategy without resorting to fallbacks. | ### Experimental Options @@ -142,7 +142,7 @@ _Important: For support and other general information, refer to the [experimenta In addition to the [standard flags](#specifying-fossa-project-details), the analyze command supports the following experimental flags: | Name | Description | -| ---------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +|------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | [`--experimental-enable-binary-discovery`](../experimental/binary-discovery/README.md) | Enable reporting binary files as unlicensed dependencies. For more information, see the [binary discovery overview](../experimental/binary-discovery/README.md). | | [`--experimental-link-project-binary './some-dir'`](../experimental/msb/README.md) | Link the provided binary files to the project being analyzed. For more information, see the [multi stage builds overview](../experimental/msb/README.md). | | [`--experimental-skip-vsi-graph 'custom+1/some$locator'`](../experimental/msb/README.md) | Skip resolving the dependencies of the given project that was previously linked via `--experimental-link-project-binary`. | @@ -258,7 +258,7 @@ touch reqs.txt && fossa analyze && rm reqs.txt && fossa test All `fossa` commands support the following FOSSA-project-related flags: | Name | Short | Description | -| ---------------------------------- | ----- | ---------------------------------------------------------------------------------------------------------------------------------------- | +|------------------------------------|-------|------------------------------------------------------------------------------------------------------------------------------------------| | `--project 'some project'` | `-p` | Override the detected project name | | `--revision 'some revision'` | `-r` | -Override the detected project revision | | `--fossa-api-key 'my-api-key'` | | An alternative to using the `FOSSA_API_KEY` environment variable to specify a FOSSA API key | diff --git a/integration-test/Analysis/FixtureUtils.hs b/integration-test/Analysis/FixtureUtils.hs index aef6dc2dd8..edb5781667 100644 --- a/integration-test/Analysis/FixtureUtils.hs +++ b/integration-test/Analysis/FixtureUtils.hs @@ -135,7 +135,7 @@ testRunner f env = & runReader (mempty :: AllFilters) & runReader (MavenScopeIncludeFilters mempty) & runReader (ExperimentalAnalyzeConfig Nothing GoModulesBasedTactic False) - & runReader (NonStrict :: Mode) + & runReader NonStrict & runFinally & runStack & withoutTelemetry diff --git a/src/App/Fossa/Analyze.hs b/src/App/Fossa/Analyze.hs index 643f8e29fd..206ebc2b7f 100644 --- a/src/App/Fossa/Analyze.hs +++ b/src/App/Fossa/Analyze.hs @@ -296,7 +296,6 @@ analyze cfg = Diag.context "fossa-analyze" $ do shouldAnalyzePathDependencies = resolvePathDependencies $ Config.experimental cfg allowedTactics = Config.allowedTacticTypes cfg withoutDefaultFilters = Config.withoutDefaultFilters cfg - -- strictMode = Config.strictMode cfg manualSrcUnits <- Diag.errorBoundaryIO . diagToDebug $ diff --git a/src/App/Fossa/Config/Analyze.hs b/src/App/Fossa/Config/Analyze.hs index b7a547e1c5..fe55f1765f 100644 --- a/src/App/Fossa/Config/Analyze.hs +++ b/src/App/Fossa/Config/Analyze.hs @@ -338,7 +338,7 @@ cliParser = <*> optional (strOption (applyFossaStyle <> long "fossa-deps-file" <> helpDoc fossaDepsFileHelp <> metavar "FILEPATH")) <*> flagOpt StaticOnlyTactics (applyFossaStyle <> long "static-only-analysis" <> stringToHelpDoc "Only analyze the project using static strategies.") <*> withoutDefaultFilterParser fossaAnalyzeDefaultFilterDocUrl - <*> flagOpt StrictMode (applyFossaStyle <> long "strict" <> stringToHelpDoc "Enables strict analysis to enforce that the first analysis strategy within a strategy type passes. Fallback strategies are not allowed in strict mode.") + <*> flagOpt StrictMode (applyFossaStyle <> long "strict" <> stringToHelpDoc "Enforces strict analysis to ensure the most accurate results for a strategy without resorting to fallbacks.") where fossaDepsFileHelp :: Maybe (Doc AnsiStyle) fossaDepsFileHelp = diff --git a/src/App/Fossa/Container/Sources/DockerArchive.hs b/src/App/Fossa/Container/Sources/DockerArchive.hs index b413f5c985..49d1d5fa7d 100644 --- a/src/App/Fossa/Container/Sources/DockerArchive.hs +++ b/src/App/Fossa/Container/Sources/DockerArchive.hs @@ -376,7 +376,7 @@ listTargetLayer capabilities osInfo layerFs tarball layerType = do False -- Targets are not impacted by path dependencies. ) . runReader (MavenScopeIncludeFilters mempty) - . runReader (NonStrict :: Mode) + . runReader NonStrict . runReader (mempty :: AllFilters) $ run where diff --git a/src/App/Types.hs b/src/App/Types.hs index 0d907574a5..469fdd8460 100644 --- a/src/App/Types.hs +++ b/src/App/Types.hs @@ -210,6 +210,9 @@ data FirstPartyScansFlag = FirstPartyScansOnFromFlag | FirstPartyScansOffFromFla instance ToJSON FirstPartyScansFlag where toEncoding = genericToEncoding defaultOptions +-- | Represents the different modes of operation during the analysis process. +-- 'Strict' mode enforces the most accurate results by rejecting fallback strategies. +-- 'NonStrict' mode allows for fallback strategies. data Mode = Strict | NonStrict diff --git a/src/Effect/Exec.hs b/src/Effect/Exec.hs index 86497d2b9f..949c456d3c 100644 --- a/src/Effect/Exec.hs +++ b/src/Effect/Exec.hs @@ -33,10 +33,11 @@ module Effect.Exec ( CandidateAnalysisCommands (..), mkAnalysisCommand, mkSingleCandidateAnalysisCommand, + GetDepsEffs, ) where import App.Support (reportDefectMsg) -import App.Types (OverrideDynamicAnalysisBinary (..)) +import App.Types (Mode, OverrideDynamicAnalysisBinary (..)) import Control.Algebra (Has) import Control.Carrier.Reader (Reader, ask) import Control.Carrier.Simple ( @@ -411,6 +412,9 @@ execCurrentDirStdinThrow cmd stdin = do Left failure -> fatal (CommandFailed failure) Right stdout -> pure stdout +-- | Shorthand for the effects needed to retrieve dependencies in analysis command. +type GetDepsEffs sig m = (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m, Has (Reader Mode) sig m) + -- | Shorthand for the effects needed to select a candidate analysis command. type CandidateCommandEffs sig m = (Has Diagnostics sig m, Has Exec sig m, Has (Reader OverrideDynamicAnalysisBinary) sig m) diff --git a/src/Strategy/Bundler.hs b/src/Strategy/Bundler.hs index afbe65057f..7df930f18a 100644 --- a/src/Strategy/Bundler.hs +++ b/src/Strategy/Bundler.hs @@ -36,7 +36,7 @@ import Discovery.Walk ( findFilesMatchingGlob, walkWithFilters', ) -import Effect.Exec (Exec, Has) +import Effect.Exec (Exec, GetDepsEffs, Has) import Effect.ReadFS (ReadFS, readContentsParser) import GHC.Generics (Generic) import Path (Abs, Dir, File, Path, toFilePath) @@ -121,7 +121,7 @@ mkProject project = , projectData = project } -getDeps :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m, Has (Reader Mode) sig m) => BundlerProject -> m DependencyResults +getDeps :: (GetDepsEffs sig m) => BundlerProject -> m DependencyResults getDeps project = do mode <- ask analyzeGemfileLock project <||> guardStrictMode mode (context "Bundler" (analyzeBundleShow project)) @@ -136,7 +136,7 @@ analyzeBundleShow project = do , dependencyManifestFiles = maybe [bundlerGemfile project] pure (bundlerGemfileLock project) } -analyzeGemfileLock :: (Has ReadFS sig m, Has Diagnostics sig m, Has (Reader Mode) sig m) => BundlerProject -> m DependencyResults +analyzeGemfileLock :: (GetDepsEffs sig m) => BundlerProject -> m DependencyResults analyzeGemfileLock project = do mode <- ask let errorInfo = case mode of diff --git a/src/Strategy/Cocoapods.hs b/src/Strategy/Cocoapods.hs index 2bc01d28ed..9cb6de02a0 100644 --- a/src/Strategy/Cocoapods.hs +++ b/src/Strategy/Cocoapods.hs @@ -29,7 +29,7 @@ import Discovery.Walk ( findFilesMatchingGlob, walkWithFilters', ) -import Effect.Exec (Exec) +import Effect.Exec (Exec, GetDepsEffs) import Effect.Logger (Logger) import Effect.ReadFS (Has, ReadFS, readContentsParser) import GHC.Generics (Generic) @@ -81,7 +81,7 @@ instance ToJSON CocoapodsProject instance AnalyzeProject CocoapodsProject where analyzeProject _ = getDeps - analyzeProjectStaticOnly _ = getDeps' + analyzeProjectStaticOnly _ = getDepsStatically instance LicenseAnalyzeProject CocoapodsProject where licenseAnalyzeProject = traverse readLicense . cocoapodsSpecFiles @@ -112,7 +112,7 @@ mkProject project = , projectData = project } -getDeps :: (Has ReadFS sig m, Has Diagnostics sig m, Has Exec sig m, Has Logger sig m, Has (Reader Mode) sig m) => CocoapodsProject -> m DependencyResults +getDeps :: (GetDepsEffs sig m, Has Logger sig m) => CocoapodsProject -> m DependencyResults getDeps project = do mode <- ask context "Cocoapods" $ @@ -126,8 +126,8 @@ getDeps project = do ) <||> guardStrictMode mode (context "Podfile analysis" (analyzePodfile project)) -getDeps' :: (Has ReadFS sig m, Has Diagnostics sig m, Has (Reader Mode) sig m) => CocoapodsProject -> m DependencyResults -getDeps' project = do +getDepsStatically :: (GetDepsEffs sig m) => CocoapodsProject -> m DependencyResults +getDepsStatically project = do mode <- ask context "Cocoapods" $ context diff --git a/src/Strategy/Conda.hs b/src/Strategy/Conda.hs index 0904173e0b..374341e249 100644 --- a/src/Strategy/Conda.hs +++ b/src/Strategy/Conda.hs @@ -22,7 +22,7 @@ import Discovery.Walk ( findFileNamed, walkWithFilters', ) -import Effect.Exec (Exec) +import Effect.Exec (Exec, GetDepsEffs) import Effect.ReadFS (ReadFS) import GHC.Generics (Generic) import Path (Abs, Dir, File, Path) @@ -75,7 +75,7 @@ mkProject project = -- There might be a dep with a version spec in an environment.yml file: i.e. conda+foo$1.2.*, and perhaps -- the same dep resolved to a known version in the users virtual environment: i.e. conda+'conda-forge':foo$1.2.4 (we get that from conda env create). -- If we combined the results then we would include both of those deps in the result, which is not correct behavior. -getDeps :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m, Has (Reader Mode) sig m) => CondaProject -> m DependencyResults +getDeps :: (GetDepsEffs sig m) => CondaProject -> m DependencyResults getDeps project = do mode <- ask analyzeCondaEnvCreate project <||> guardStrictMode mode (analyzeEnvironmentYml project) diff --git a/src/Strategy/Godep.hs b/src/Strategy/Godep.hs index e73d1be095..d9016d2d8b 100644 --- a/src/Strategy/Godep.hs +++ b/src/Strategy/Godep.hs @@ -17,7 +17,7 @@ import Discovery.Walk ( findFileNamed, walkWithFilters', ) -import Effect.Exec (Exec, Has) +import Effect.Exec (Exec, GetDepsEffs, Has) import Effect.ReadFS (ReadFS) import GHC.Generics (Generic) import Path (Abs, Dir, File, Path) @@ -71,7 +71,7 @@ mkProject project = , projectData = project } -getDeps :: (Has ReadFS sig m, Has Exec sig m, Has Diagnostics sig m, Has (Reader Mode) sig m) => GodepProject -> m DependencyResults +getDeps :: (GetDepsEffs sig m) => GodepProject -> m DependencyResults getDeps project = do mode <- ask context "Godep" $ diff --git a/src/Strategy/Gomodules.hs b/src/Strategy/Gomodules.hs index cb889bf3cc..7f3d9133e5 100644 --- a/src/Strategy/Gomodules.hs +++ b/src/Strategy/Gomodules.hs @@ -8,7 +8,6 @@ module Strategy.Gomodules ( import App.Fossa.Analyze.Types (AnalyzeProject (analyzeProjectStaticOnly), analyzeProject) import App.Fossa.Config.Analyze (ExperimentalAnalyzeConfig (useV3GoResolver), GoDynamicTactic (..)) -import App.Types (Mode (..)) import App.Util (guardStrictMode) import Control.Carrier.Diagnostics (warn) import Control.Effect.Diagnostics (Diagnostics, context, fatalText, recover, (<||>)) @@ -23,7 +22,7 @@ import Discovery.Walk ( findFileNamed, walkWithFilters', ) -import Effect.Exec (Exec, Has) +import Effect.Exec (Exec, GetDepsEffs, Has) import Effect.ReadFS (ReadFS) import GHC.Generics (Generic) import Graphing (Graphing) @@ -69,7 +68,7 @@ mkProject project = , projectData = project } -getDeps :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m, Has (Reader Mode) sig m) => GomodulesProject -> GoDynamicTactic -> m DependencyResults +getDeps :: (GetDepsEffs sig m) => GomodulesProject -> GoDynamicTactic -> m DependencyResults getDeps project goDynamicTactic = do mode <- ask (graph, graphBreadth) <- context "Gomodules" $ dynamicAnalysis <||> guardStrictMode mode staticAnalysis diff --git a/src/Strategy/Maven.hs b/src/Strategy/Maven.hs index 5055c26c7e..0d88adc0b0 100644 --- a/src/Strategy/Maven.hs +++ b/src/Strategy/Maven.hs @@ -22,7 +22,7 @@ import DepTypes (Dependency) import Diag.Common (MissingDeepDeps (MissingDeepDeps), MissingEdges (MissingEdges)) import Discovery.Filters (AllFilters, MavenScopeFilters, mavenScopeFilterSet) import Discovery.Simple (simpleDiscover) -import Effect.Exec (CandidateCommandEffs) +import Effect.Exec (CandidateCommandEffs, GetDepsEffs) import Effect.ReadFS (ReadFS) import GHC.Generics (Generic) import Graphing (Graphing, gmap, shrinkRoots) @@ -67,17 +67,16 @@ instance ToJSON MavenProject instance AnalyzeProject MavenProject where analyzeProject = getDeps - analyzeProjectStaticOnly = getDeps' + analyzeProjectStaticOnly = getDepsStatically instance LicenseAnalyzeProject MavenProject where licenseAnalyzeProject = pure . Pom.getLicenses . unMavenProject getDeps :: ( Has (Lift IO) sig m - , Has ReadFS sig m + , GetDepsEffs sig m , CandidateCommandEffs sig m , Has (Reader MavenScopeFilters) sig m - , Has (Reader Mode) sig m ) => FoundTargets -> MavenProject -> @@ -96,7 +95,7 @@ getDeps foundTargets (MavenProject closure) = do , dependencyManifestFiles = [PomClosure.closurePath closure] } -getDeps' :: +getDepsStatically :: ( Has (Lift IO) sig m , Has Diagnostics sig m , Has (Reader MavenScopeFilters) sig m @@ -104,7 +103,7 @@ getDeps' :: FoundTargets -> MavenProject -> m DependencyResults -getDeps' foundTargets (MavenProject closure) = do +getDepsStatically foundTargets (MavenProject closure) = do let submoduleTargets = submoduleTargetSet foundTargets (graph, graphBreadth) <- context "Maven" $ getStaticAnalysis submoduleTargets closure @@ -117,10 +116,9 @@ getDeps' foundTargets (MavenProject closure) = do getDepsDynamicAnalysis :: ( Has (Lift IO) sig m - , Has ReadFS sig m + , GetDepsEffs sig m , CandidateCommandEffs sig m , Has (Reader MavenScopeFilters) sig m - , Has (Reader Mode) sig m ) => Set Text -> MavenProjectClosure -> diff --git a/src/Strategy/Nim.hs b/src/Strategy/Nim.hs index d48832710f..9c0f75b83c 100644 --- a/src/Strategy/Nim.hs +++ b/src/Strategy/Nim.hs @@ -32,7 +32,7 @@ instance ToJSON NimbleProject instance AnalyzeProject NimbleProject where analyzeProject _ = getDeps - analyzeProjectStaticOnly _ = getDeps' + analyzeProjectStaticOnly _ = getDepsStatically discover :: (Has ReadFS sig m, Has Diagnostics sig m, Has (Reader AllFilters) sig m) => Path Abs Dir -> m [DiscoveredProject NimbleProject] discover = simpleDiscover findProjects mkProject NimbleProjectType @@ -62,8 +62,8 @@ getDeps project = do , dependencyManifestFiles = [nimbleLockFile project] } -getDeps' :: (Has ReadFS sig m, Has Diagnostics sig m) => NimbleProject -> m DependencyResults -getDeps' project = do +getDepsStatically :: (Has ReadFS sig m, Has Diagnostics sig m) => NimbleProject -> m DependencyResults +getDepsStatically project = do (graph, graphBreadth) <- analyze' (nimDir project) (nimbleLockFile project) pure $ DependencyResults diff --git a/src/Strategy/Pub.hs b/src/Strategy/Pub.hs index fc82d512cd..1ef07badea 100644 --- a/src/Strategy/Pub.hs +++ b/src/Strategy/Pub.hs @@ -16,7 +16,7 @@ import Diag.Common ( import Discovery.Filters (AllFilters) import Discovery.Simple (simpleDiscover) import Discovery.Walk (WalkStep (WalkContinue), findFileNamed, walkWithFilters') -import Effect.Exec (Exec, Has) +import Effect.Exec (Exec, GetDepsEffs, Has) import Effect.Logger (Logger) import Effect.ReadFS (ReadFS) import GHC.Generics (Generic) @@ -55,7 +55,7 @@ instance ToJSON PubProject instance AnalyzeProject PubProject where analyzeProject _ = getDeps - analyzeProjectStaticOnly _ = getDeps' + analyzeProjectStaticOnly _ = getDepsStatically mkProject :: PubProject -> DiscoveredProject PubProject mkProject project = @@ -66,13 +66,13 @@ mkProject project = , projectData = project } -getDeps :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m, Has Logger sig m, Has (Reader Mode) sig m) => PubProject -> m DependencyResults +getDeps :: (GetDepsEffs sig m, Has Logger sig m) => PubProject -> m DependencyResults getDeps project = do mode <- ask (graph, graphBreadth) <- case pubLock project of Just lockFile -> analyzeDepsCmd lockFile (pubSpecDir project) <||> guardStrictMode mode (analyzePubLockFile lockFile) Nothing -> do - _ <- applyMissingPubSpecWarnings + applyMissingPubSpecWarnings analyzePubSpecFile (pubSpec project) pure $ DependencyResults @@ -81,12 +81,12 @@ getDeps project = do , dependencyManifestFiles = [pubSpec project] } -getDeps' :: (Has ReadFS sig m, Has Diagnostics sig m, Has Logger sig m) => PubProject -> m DependencyResults -getDeps' project = do +getDepsStatically :: (Has ReadFS sig m, Has Diagnostics sig m, Has Logger sig m) => PubProject -> m DependencyResults +getDepsStatically project = do (graph, graphBreadth) <- case pubLock project of Just lockFile -> analyzePubLockFile lockFile Nothing -> do - _ <- applyMissingPubSpecWarnings + applyMissingPubSpecWarnings analyzePubSpecFile (pubSpec project) pure $ DependencyResults diff --git a/src/Strategy/Python/Pipenv.hs b/src/Strategy/Python/Pipenv.hs index bc7837cedf..03ca3f52e0 100644 --- a/src/Strategy/Python/Pipenv.hs +++ b/src/Strategy/Python/Pipenv.hs @@ -114,13 +114,13 @@ getDeps project = context "Pipenv" $ do , dependencyManifestFiles = [pipenvLockfile project] } -getDeps' :: +getDepsStatically :: ( Has ReadFS sig m , Has Diagnostics sig m ) => PipenvProject -> m DependencyResults -getDeps' project = context "Pipenv" $ do +getDepsStatically project = context "Pipenv" $ do lock <- context "Getting direct dependencies" $ readContentsJson (pipenvLockfile project) graph <- context "Building dependency graph" $ pure (buildGraph lock Nothing) pure $ @@ -148,7 +148,7 @@ instance ToJSON PipenvProject instance AnalyzeProject PipenvProject where analyzeProject _ = getDeps - analyzeProjectStaticOnly _ = getDeps' + analyzeProjectStaticOnly _ = getDepsStatically pipenvGraphCmd :: Command pipenvGraphCmd = diff --git a/src/Strategy/Python/Setuptools.hs b/src/Strategy/Python/Setuptools.hs index 9e76fdce0f..c6c3e4db18 100644 --- a/src/Strategy/Python/Setuptools.hs +++ b/src/Strategy/Python/Setuptools.hs @@ -79,8 +79,8 @@ getDeps project = do , dependencyManifestFiles = maybeToList (setuptoolsSetupPy project) ++ setuptoolsReqTxt project } -getDeps' :: (Has ReadFS sig m, Has Diagnostics sig m) => SetuptoolsProject -> m DependencyResults -getDeps' project = do +getDepsStatically :: (Has ReadFS sig m, Has Diagnostics sig m) => SetuptoolsProject -> m DependencyResults +getDepsStatically project = do graph <- context "Setuptools" $ Diag.combineSuccessful @Text @Text @@ -115,7 +115,7 @@ instance ToJSON SetuptoolsProject instance AnalyzeProject SetuptoolsProject where analyzeProject _ = getDeps - analyzeProjectStaticOnly _ = getDeps' + analyzeProjectStaticOnly _ = getDepsStatically mkProject :: SetuptoolsProject -> DiscoveredProject SetuptoolsProject mkProject project = diff --git a/src/Strategy/Scala.hs b/src/Strategy/Scala.hs index 52faac7b2e..b0bf447780 100644 --- a/src/Strategy/Scala.hs +++ b/src/Strategy/Scala.hs @@ -36,6 +36,7 @@ import Discovery.Walk ( import Effect.Exec ( Command (..), Exec, + GetDepsEffs, Has, execThrow, ) @@ -128,7 +129,7 @@ mkProject (ScalaProject sbtBuildDir sbtTreeJson closure) = , projectData = ScalaProject sbtBuildDir sbtTreeJson closure } -getDeps :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m, Has Logger sig m, Has (Reader Mode) sig m) => ScalaProject -> m DependencyResults +getDeps :: (GetDepsEffs sig m, Has Logger sig m) => ScalaProject -> m DependencyResults getDeps project = do mode <- ask case mode of From b72f30543bd5bb64f8f079ef504d1c5d37d4470d Mon Sep 17 00:00:00 2001 From: Jeffrey Huynh Date: Fri, 6 Sep 2024 14:31:10 -0700 Subject: [PATCH 6/8] update tests --- docs/references/subcommands/analyze.md | 12 +++++----- integration-test/Analysis/CarthageSpec.hs | 3 ++- integration-test/Analysis/ClojureSpec.hs | 5 ++-- integration-test/Analysis/CocoapodsSpec.hs | 6 +++-- integration-test/Analysis/ElixirSpec.hs | 3 ++- integration-test/Analysis/ErlangSpec.hs | 5 ++-- .../Analysis/FixtureExpectationUtils.hs | 14 ++++++----- integration-test/Analysis/FixtureUtils.hs | 6 ++--- integration-test/Analysis/GoSpec.hs | 3 ++- integration-test/Analysis/GradleSpec.hs | 5 ++-- integration-test/Analysis/MavenSpec.hs | 23 +++++++++++++++---- integration-test/Analysis/NugetSpec.hs | 5 ++-- .../Analysis/Python/PipenvSpec.hs | 3 ++- .../Analysis/Python/PoetrySpec.hs | 3 ++- .../Analysis/Python/SetuptoolsSpec.hs | 5 ++-- integration-test/Analysis/RubySpec.hs | 3 ++- integration-test/Analysis/ScalaSpec.hs | 4 +++- integration-test/Analysis/SwiftSpec.hs | 3 ++- src/Strategy/Conda.hs | 1 - src/Strategy/Godep.hs | 1 - 20 files changed, 71 insertions(+), 42 deletions(-) diff --git a/docs/references/subcommands/analyze.md b/docs/references/subcommands/analyze.md index 0d61e3442b..8caf3f5a84 100644 --- a/docs/references/subcommands/analyze.md +++ b/docs/references/subcommands/analyze.md @@ -128,12 +128,12 @@ We support the following archive formats: In addition to the [standard flags](#specifying-fossa-project-details), the analyze command supports the following additional strategy flags: -| Name | Description | -|-----------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [`--detect-vendored`](./analyze/detect-vendored.md) | Enable the vendored source identification engine. For more information, see the [C and C++ overview](../strategies/languages/c-cpp/c-cpp.md). | -| [`--detect-dynamic './some-binary`](./analyze/detect-dynamic.md) | Analyze the binary at the provided path for dynamically linked dependencies. For more information, see the [C and C++ overview](../strategies/languages/c-cpp/c-cpp.md). | -| [`--static-only-analysis`](../strategies/README.md#static-and-dynamic-strategies) | Do not use third-party tools when analyzing projects. | -| `--strict` | Enforces strict analysis to ensure the most accurate results for a strategy without resorting to fallbacks. | +| Name | Description | +|-----------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [`--detect-vendored`](./analyze/detect-vendored.md) | Enable the vendored source identification engine. For more information, see the [C and C++ overview](../strategies/languages/c-cpp/c-cpp.md). | +| [`--detect-dynamic './some-binary`](./analyze/detect-dynamic.md) | Analyze the binary at the provided path for dynamically linked dependencies. For more information, see the [C and C++ overview](../strategies/languages/c-cpp/c-cpp.md). | +| [`--static-only-analysis`](../strategies/README.md#static-and-dynamic-strategies) | Do not use third-party tools when analyzing projects. | +| `--strict` | Enforces strict analysis to ensure the most accurate results for a strategy by rejecting fallbacks. When ran with `--statitic-only-analysis`, the most optimal static strategy will be applied without fallbacks. | ### Experimental Options diff --git a/integration-test/Analysis/CarthageSpec.hs b/integration-test/Analysis/CarthageSpec.hs index bc4fd6d8e9..178ac14918 100644 --- a/integration-test/Analysis/CarthageSpec.hs +++ b/integration-test/Analysis/CarthageSpec.hs @@ -5,6 +5,7 @@ module Analysis.CarthageSpec (spec) where import Analysis.FixtureExpectationUtils import Analysis.FixtureUtils +import App.Types (Mode (NonStrict)) import Path import Strategy.Carthage qualified as Carthage import Test.Hspec @@ -24,4 +25,4 @@ swiftQueue = spec :: Spec spec = do - testSuiteDepResultSummary swiftQueue CarthageProjectType (DependencyResultsSummary 1 1 0 1 Complete) + testSuiteDepResultSummary NonStrict swiftQueue CarthageProjectType (DependencyResultsSummary 1 1 0 1 Complete) diff --git a/integration-test/Analysis/ClojureSpec.hs b/integration-test/Analysis/ClojureSpec.hs index 16f217a985..199f892822 100644 --- a/integration-test/Analysis/ClojureSpec.hs +++ b/integration-test/Analysis/ClojureSpec.hs @@ -5,6 +5,7 @@ module Analysis.ClojureSpec (spec) where import Analysis.FixtureExpectationUtils import Analysis.FixtureUtils +import App.Types (Mode (NonStrict)) import Path import Strategy.Leiningen qualified as Leiningen import Test.Hspec @@ -39,5 +40,5 @@ ring = spec :: Spec spec = do - testSuiteDepResultSummary eastwood LeiningenProjectType (DependencyResultsSummary 10 7 3 1 Complete) - testSuiteDepResultSummary ring LeiningenProjectType (DependencyResultsSummary 23 6 17 1 Complete) + testSuiteDepResultSummary NonStrict eastwood LeiningenProjectType (DependencyResultsSummary 10 7 3 1 Complete) + testSuiteDepResultSummary NonStrict ring LeiningenProjectType (DependencyResultsSummary 23 6 17 1 Complete) diff --git a/integration-test/Analysis/CocoapodsSpec.hs b/integration-test/Analysis/CocoapodsSpec.hs index 41dd10240d..a84ff4f1da 100644 --- a/integration-test/Analysis/CocoapodsSpec.hs +++ b/integration-test/Analysis/CocoapodsSpec.hs @@ -5,6 +5,7 @@ module Analysis.CocoapodsSpec (spec) where import Analysis.FixtureExpectationUtils import Analysis.FixtureUtils +import App.Types (Mode (..)) import Path import Strategy.Cocoapods qualified as Cocoapods import Test.Hspec @@ -36,5 +37,6 @@ sDWebImage = spec :: Spec spec = do - testSuiteDepResultSummary shadowsocksXNG CocoapodsProjectType (DependencyResultsSummary 7 6 2 1 Complete) - testSuiteDepResultSummary sDWebImage CocoapodsProjectType (DependencyResultsSummary 4 4 0 1 Partial) + testSuiteDepResultSummary NonStrict shadowsocksXNG CocoapodsProjectType (DependencyResultsSummary 7 6 2 1 Complete) + testSuiteDepResultSummary Strict shadowsocksXNG CocoapodsProjectType (DependencyResultsSummary 7 6 2 1 Complete) + testSuiteDepResultSummary NonStrict sDWebImage CocoapodsProjectType (DependencyResultsSummary 4 4 0 1 Partial) diff --git a/integration-test/Analysis/ElixirSpec.hs b/integration-test/Analysis/ElixirSpec.hs index 9b7b74821a..5a2616c395 100644 --- a/integration-test/Analysis/ElixirSpec.hs +++ b/integration-test/Analysis/ElixirSpec.hs @@ -5,6 +5,7 @@ module Analysis.ElixirSpec (spec) where import Analysis.FixtureExpectationUtils import Analysis.FixtureUtils +import App.Types (Mode (NonStrict)) import Effect.Exec (AllowErr (Never), Command (Command)) import Path import Strategy.Mix qualified as Mix @@ -42,4 +43,4 @@ absinthe = spec :: Spec spec = do - testSuiteDepResultSummary absinthe MixProjectType (DependencyResultsSummary 4 4 1 1 Complete) + testSuiteDepResultSummary NonStrict absinthe MixProjectType (DependencyResultsSummary 4 4 1 1 Complete) diff --git a/integration-test/Analysis/ErlangSpec.hs b/integration-test/Analysis/ErlangSpec.hs index 288c1ea9f3..c4dc52a3a7 100644 --- a/integration-test/Analysis/ErlangSpec.hs +++ b/integration-test/Analysis/ErlangSpec.hs @@ -5,6 +5,7 @@ module Analysis.ErlangSpec (spec) where import Analysis.FixtureExpectationUtils import Analysis.FixtureUtils +import App.Types (Mode (NonStrict)) import Path import Strategy.Rebar3 qualified as Rebar3 import Test.Hspec @@ -39,5 +40,5 @@ emqx = spec :: Spec spec = do - testSuiteDepResultSummary cowboy Rebar3ProjectType (DependencyResultsSummary 2 2 0 1 Complete) - testSuiteDepResultSummary emqx Rebar3ProjectType (DependencyResultsSummary 0 0 0 1 Complete) + testSuiteDepResultSummary NonStrict cowboy Rebar3ProjectType (DependencyResultsSummary 2 2 0 1 Complete) + testSuiteDepResultSummary NonStrict emqx Rebar3ProjectType (DependencyResultsSummary 0 0 0 1 Complete) diff --git a/integration-test/Analysis/FixtureExpectationUtils.hs b/integration-test/Analysis/FixtureExpectationUtils.hs index 394ae23406..0c5c38dbd1 100644 --- a/integration-test/Analysis/FixtureExpectationUtils.hs +++ b/integration-test/Analysis/FixtureExpectationUtils.hs @@ -16,6 +16,7 @@ module Analysis.FixtureExpectationUtils ( import Analysis.FixtureUtils (AnalysisTestFixture (..), FixtureArtifact (..), getArtifact, performDiscoveryAndAnalyses) import App.Fossa.Analyze.Types (AnalyzeProject) +import App.Types (Mode (..)) import Control.Algebra (Has) import Control.Effect.Lift (Lift, sendIO) import Data.List (find) @@ -51,12 +52,13 @@ summarize dr = -- | Performs discovery and analysis for all discovered project for provided analysis integration fixture. withAnalysisOf :: (Has (Lift IO) sig m, AnalyzeProject a, MonadFail m) => + Mode -> AnalysisTestFixture a -> (([(DiscoveredProject a, DependencyResults)], Path Abs Dir) -> m b) -> m () -withAnalysisOf testFixture runTest = do +withAnalysisOf mode testFixture runTest = do extractedDir <- getArtifact (artifact testFixture) - res <- performDiscoveryAndAnalyses extractedDir testFixture + res <- performDiscoveryAndAnalyses extractedDir testFixture mode _ <- runTest (res, extractedDir (scopedDir . artifact $ testFixture)) sendIO $ PIO.removeDirRecur extractedDir @@ -91,7 +93,7 @@ withProjectOfType result (projType, projPath) = find (\(dr, _) -> projectType dr == projType && projectPath dr == projPath) result testSuiteHasSomeDepResults :: (AnalyzeProject a, Show a, Eq a) => AnalysisTestFixture a -> DiscoveredProjectType -> Spec -testSuiteHasSomeDepResults fixture projType = aroundAll (withAnalysisOf fixture) $ +testSuiteHasSomeDepResults fixture projType = aroundAll (withAnalysisOf NonStrict fixture) $ describe (toString $ testName fixture) $ do it "should find targets" $ \(result, extractedDir) -> expectProject (projType, extractedDir) result @@ -99,9 +101,9 @@ testSuiteHasSomeDepResults fixture projType = aroundAll (withAnalysisOf fixture) it "should have some dependency results" $ \(result, extractedDir) -> isJust (getDepResultsOf result (projType, extractedDir)) `shouldBe` True -testSuiteDepResultSummary :: (AnalyzeProject a, Show a, Eq a) => AnalysisTestFixture a -> DiscoveredProjectType -> DependencyResultsSummary -> Spec -testSuiteDepResultSummary fixture projType depResultSummary = - aroundAll (withAnalysisOf fixture) $ +testSuiteDepResultSummary :: (AnalyzeProject a, Show a, Eq a) => Mode -> AnalysisTestFixture a -> DiscoveredProjectType -> DependencyResultsSummary -> Spec +testSuiteDepResultSummary mode fixture projType depResultSummary = + aroundAll (withAnalysisOf mode fixture) $ describe (toString $ testName fixture) $ do it "should find targets" $ \(result, extractedDir) -> expectProject (projType, extractedDir) result diff --git a/integration-test/Analysis/FixtureUtils.hs b/integration-test/Analysis/FixtureUtils.hs index edb5781667..7ff970ea4f 100644 --- a/integration-test/Analysis/FixtureUtils.hs +++ b/integration-test/Analysis/FixtureUtils.hs @@ -156,8 +156,8 @@ decorateCmdWith (NixEnv pkgs) cmd = -- -------------------------------- -- Analysis fixture test runner -performDiscoveryAndAnalyses :: (Has (Lift IO) sig m, AnalyzeProject a, MonadFail m) => Path Abs Dir -> AnalysisTestFixture a -> m [(DiscoveredProject a, DependencyResults)] -performDiscoveryAndAnalyses targetDir AnalysisTestFixture{..} = do +performDiscoveryAndAnalyses :: (Has (Lift IO) sig m, AnalyzeProject a, MonadFail m) => Path Abs Dir -> AnalysisTestFixture a -> Mode -> m [(DiscoveredProject a, DependencyResults)] +performDiscoveryAndAnalyses targetDir AnalysisTestFixture{..} mode = do -- Perform any project builds _ <- sendIO $ runCmd environment buildCmd @@ -165,7 +165,7 @@ performDiscoveryAndAnalyses targetDir AnalysisTestFixture{..} = do discoveryResult <- sendIO $ testRunner (discover targetDir) environment withResult discoveryResult $ \_ dps -> for dps $ \dp -> do - analysisResult <- sendIO $ testRunner (ignoreDebug $ analyzeProject (projectBuildTargets dp) (projectData dp)) environment + analysisResult <- sendIO $ testRunner (ignoreDebug $ runReader mode $ analyzeProject (projectBuildTargets dp) (projectData dp)) environment withResult analysisResult $ \_ dr -> pure (dp, dr) where runCmd :: FixtureEnvironment -> Maybe (Command) -> IO () diff --git a/integration-test/Analysis/GoSpec.hs b/integration-test/Analysis/GoSpec.hs index c888cc8286..53fa8bcae0 100644 --- a/integration-test/Analysis/GoSpec.hs +++ b/integration-test/Analysis/GoSpec.hs @@ -5,6 +5,7 @@ module Analysis.GoSpec (spec) where import Analysis.FixtureExpectationUtils import Analysis.FixtureUtils +import App.Types (Mode (NonStrict)) import Path import Strategy.Gomodules qualified as Gomodules import Test.Hspec @@ -27,7 +28,7 @@ vault = testVault :: Spec testVault = - aroundAll (withAnalysisOf vault) $ do + aroundAll (withAnalysisOf NonStrict vault) $ do describe "vault" $ do it "should find targets" $ \(result, extractedDir) -> do expectProject (GomodProjectType, extractedDir) result diff --git a/integration-test/Analysis/GradleSpec.hs b/integration-test/Analysis/GradleSpec.hs index cb3f7a89e8..c50cd0d406 100644 --- a/integration-test/Analysis/GradleSpec.hs +++ b/integration-test/Analysis/GradleSpec.hs @@ -12,6 +12,7 @@ import Analysis.FixtureUtils ( FixtureArtifact (FixtureArtifact), FixtureEnvironment (NixEnv), ) +import App.Types (Mode (NonStrict)) import Path (reldir) import Strategy.Gradle qualified as Gradle import Test.Hspec (Spec, aroundAll, describe, it, shouldBe) @@ -46,7 +47,7 @@ gradleSettingsOnly = testSpringBoot :: Spec testSpringBoot = - aroundAll (withAnalysisOf springBoot) $ do + aroundAll (withAnalysisOf NonStrict springBoot) $ do describe "gradle-java springboot" $ do it "should find targets" $ \(result, extractedDir) -> do expectProject (GradleProjectType, extractedDir) result @@ -54,7 +55,7 @@ testSpringBoot = testGradleSettingsOnly :: Spec testGradleSettingsOnly = - aroundAll (withAnalysisOf gradleSettingsOnly) $ do + aroundAll (withAnalysisOf NonStrict gradleSettingsOnly) $ do describe "gradle-java gradle settings only" $ do it "should find targets" $ \(result, extractedDir) -> do expectProject (GradleProjectType, extractedDir) result diff --git a/integration-test/Analysis/MavenSpec.hs b/integration-test/Analysis/MavenSpec.hs index 91a6d6fc80..31a7be2a33 100644 --- a/integration-test/Analysis/MavenSpec.hs +++ b/integration-test/Analysis/MavenSpec.hs @@ -21,6 +21,8 @@ import Analysis.FixtureUtils ( FixtureArtifact (FixtureArtifact), FixtureEnvironment (NixEnv), ) +import App.Types (Mode (..)) + import Path (reldir) import Strategy.Maven qualified as Maven import Test.Hspec (Spec, aroundAll, describe, it) @@ -80,16 +82,24 @@ pomFileWithBuildDirOverride = [reldir|maven/build_dir_override/|] [reldir|example-pom-file-override-build-directory|] -testKeycloak :: Spec -testKeycloak = do - aroundAll (withAnalysisOf keycloak) $ do - describe "keycloak" $ do +testKeycloakNonStrict :: Spec +testKeycloakNonStrict = do + aroundAll (withAnalysisOf NonStrict keycloak) $ do + describe "keycloak non strict mode" $ do + it "should find targets" $ \(result, extractedDir) -> + expectProject (MavenProjectType, extractedDir) result + +testKeycloakStrict :: Spec +testKeycloakStrict = do + aroundAll (withAnalysisOf Strict keycloak) $ do + describe "keycloak strict mode" $ do it "should find targets" $ \(result, extractedDir) -> expectProject (MavenProjectType, extractedDir) result testGuava :: Spec testGuava = testSuiteDepResultSummary + NonStrict guava Types.MavenProjectType DependencyResultsSummary @@ -103,6 +113,7 @@ testGuava = testSimplePomFile :: Spec testSimplePomFile = do testSuiteDepResultSummary + NonStrict simplePomFile Types.MavenProjectType DependencyResultsSummary @@ -116,6 +127,7 @@ testSimplePomFile = do testBuildDirOverride :: Spec testBuildDirOverride = do testSuiteDepResultSummary + NonStrict pomFileWithBuildDirOverride Types.MavenProjectType DependencyResultsSummary @@ -128,7 +140,8 @@ testBuildDirOverride = do spec :: Spec spec = do - testKeycloak + testKeycloakNonStrict + testKeycloakStrict testGuava testBuildDirOverride testSimplePomFile diff --git a/integration-test/Analysis/NugetSpec.hs b/integration-test/Analysis/NugetSpec.hs index cc7a8d3b8a..1e3737ed38 100644 --- a/integration-test/Analysis/NugetSpec.hs +++ b/integration-test/Analysis/NugetSpec.hs @@ -5,6 +5,7 @@ module Analysis.NugetSpec (spec) where import Analysis.FixtureExpectationUtils import Analysis.FixtureUtils +import App.Types (Mode (NonStrict)) import Path import Strategy.NuGet.PackageReference qualified as PackageReference import Strategy.NuGet.PackagesConfig qualified as PackagesConfig @@ -25,14 +26,14 @@ serviceStack discoveryFunc = testServiceStackForPkgReferences :: Spec testServiceStackForPkgReferences = - aroundAll (withAnalysisOf $ serviceStack PackageReference.discover) $ do + aroundAll (withAnalysisOf NonStrict $ serviceStack PackageReference.discover) $ do describe "ServiceStack" $ do it "should find targets" $ \(result, _) -> do length result `shouldBe` 64 testServiceStackForPkgConfig :: Spec testServiceStackForPkgConfig = - aroundAll (withAnalysisOf $ serviceStack PackagesConfig.discover) $ do + aroundAll (withAnalysisOf NonStrict $ serviceStack PackagesConfig.discover) $ do describe "ServiceStack" $ do it "should find targets" $ \(result, _) -> do length result `shouldBe` 4 diff --git a/integration-test/Analysis/Python/PipenvSpec.hs b/integration-test/Analysis/Python/PipenvSpec.hs index 28da51b495..93e5cc101b 100644 --- a/integration-test/Analysis/Python/PipenvSpec.hs +++ b/integration-test/Analysis/Python/PipenvSpec.hs @@ -5,6 +5,7 @@ module Analysis.Python.PipenvSpec (spec) where import Analysis.FixtureExpectationUtils import Analysis.FixtureUtils +import App.Types (Mode (NonStrict)) import Path import Strategy.Python.Pipenv qualified as Pipenv import Test.Hspec @@ -24,4 +25,4 @@ pipenv = spec :: Spec spec = do - testSuiteDepResultSummary pipenv PipenvProjectType (DependencyResultsSummary 90 90 0 1 Complete) + testSuiteDepResultSummary NonStrict pipenv PipenvProjectType (DependencyResultsSummary 90 90 0 1 Complete) diff --git a/integration-test/Analysis/Python/PoetrySpec.hs b/integration-test/Analysis/Python/PoetrySpec.hs index 40a3da1a02..fb3968b7f9 100644 --- a/integration-test/Analysis/Python/PoetrySpec.hs +++ b/integration-test/Analysis/Python/PoetrySpec.hs @@ -5,6 +5,7 @@ module Analysis.Python.PoetrySpec (spec) where import Analysis.FixtureExpectationUtils import Analysis.FixtureUtils +import App.Types (Mode (NonStrict)) import Path import Strategy.Python.Poetry qualified as Poetry import Test.Hspec @@ -24,4 +25,4 @@ poetry = spec :: Spec spec = do - testSuiteDepResultSummary poetry PoetryProjectType (DependencyResultsSummary 65 29 69 1 Complete) + testSuiteDepResultSummary NonStrict poetry PoetryProjectType (DependencyResultsSummary 65 29 69 1 Complete) diff --git a/integration-test/Analysis/Python/SetuptoolsSpec.hs b/integration-test/Analysis/Python/SetuptoolsSpec.hs index 2c071048f7..276bf5fc56 100644 --- a/integration-test/Analysis/Python/SetuptoolsSpec.hs +++ b/integration-test/Analysis/Python/SetuptoolsSpec.hs @@ -5,6 +5,7 @@ module Analysis.Python.SetuptoolsSpec (spec) where import Analysis.FixtureExpectationUtils import Analysis.FixtureUtils +import App.Types (Mode (NonStrict)) import Path import Strategy.Python.Setuptools qualified as Setuptools import Test.Hspec @@ -36,5 +37,5 @@ flask = spec :: Spec spec = do - testSuiteDepResultSummary theFuck SetuptoolsProjectType (DependencyResultsSummary 16 16 0 2 Partial) - testSuiteDepResultSummary flask SetuptoolsProjectType (DependencyResultsSummary 4 4 0 1 Partial) + testSuiteDepResultSummary NonStrict theFuck SetuptoolsProjectType (DependencyResultsSummary 16 16 0 2 Partial) + testSuiteDepResultSummary NonStrict flask SetuptoolsProjectType (DependencyResultsSummary 4 4 0 1 Partial) diff --git a/integration-test/Analysis/RubySpec.hs b/integration-test/Analysis/RubySpec.hs index 5906942ae3..317acf34ef 100644 --- a/integration-test/Analysis/RubySpec.hs +++ b/integration-test/Analysis/RubySpec.hs @@ -5,6 +5,7 @@ module Analysis.RubySpec (spec) where import Analysis.FixtureExpectationUtils import Analysis.FixtureUtils +import App.Types (Mode (NonStrict)) import Path import Strategy.Bundler qualified as Bundler import Test.Hspec @@ -24,4 +25,4 @@ rails = spec :: Spec spec = do - testSuiteDepResultSummary rails BundlerProjectType (DependencyResultsSummary 206 70 293 1 Complete) + testSuiteDepResultSummary NonStrict rails BundlerProjectType (DependencyResultsSummary 206 70 293 1 Complete) diff --git a/integration-test/Analysis/ScalaSpec.hs b/integration-test/Analysis/ScalaSpec.hs index 6fb5ed0c1a..d7b9906628 100644 --- a/integration-test/Analysis/ScalaSpec.hs +++ b/integration-test/Analysis/ScalaSpec.hs @@ -12,6 +12,7 @@ import Analysis.FixtureUtils ( FixtureArtifact (..), FixtureEnvironment (NixEnv), ) +import App.Types (Mode (..)) import Path (reldir) import Strategy.Scala qualified as Scala import Test.Hspec (Spec) @@ -34,4 +35,5 @@ scalaExampleProject = spec :: Spec spec = do - testSuiteDepResultSummary scalaExampleProject ScalaProjectType (DependencyResultsSummary 3 2 1 1 Complete) + testSuiteDepResultSummary NonStrict scalaExampleProject ScalaProjectType (DependencyResultsSummary 3 2 1 1 Complete) + testSuiteDepResultSummary Strict scalaExampleProject ScalaProjectType (DependencyResultsSummary 3 2 1 1 Complete) diff --git a/integration-test/Analysis/SwiftSpec.hs b/integration-test/Analysis/SwiftSpec.hs index 4edd47c5e8..5d1f6b562a 100644 --- a/integration-test/Analysis/SwiftSpec.hs +++ b/integration-test/Analysis/SwiftSpec.hs @@ -5,6 +5,7 @@ module Analysis.SwiftSpec (spec) where import Analysis.FixtureExpectationUtils import Analysis.FixtureUtils +import App.Types (Mode (NonStrict)) import Path import Strategy.SwiftPM qualified as SwiftPM import Test.Hspec @@ -24,4 +25,4 @@ exampleProject = spec :: Spec spec = do - testSuiteDepResultSummary exampleProject SwiftProjectType (DependencyResultsSummary 6 6 0 1 Partial) + testSuiteDepResultSummary NonStrict exampleProject SwiftProjectType (DependencyResultsSummary 6 6 0 1 Partial) diff --git a/src/Strategy/Conda.hs b/src/Strategy/Conda.hs index 374341e249..a9679ca4a7 100644 --- a/src/Strategy/Conda.hs +++ b/src/Strategy/Conda.hs @@ -5,7 +5,6 @@ module Strategy.Conda ( ) where import App.Fossa.Analyze.Types (AnalyzeProject (analyzeProjectStaticOnly), analyzeProject) -import App.Types (Mode (..)) import App.Util (guardStrictMode) import Control.Effect.Diagnostics ( Diagnostics, diff --git a/src/Strategy/Godep.hs b/src/Strategy/Godep.hs index d9016d2d8b..65dba72546 100644 --- a/src/Strategy/Godep.hs +++ b/src/Strategy/Godep.hs @@ -3,7 +3,6 @@ module Strategy.Godep ( ) where import App.Fossa.Analyze.Types (AnalyzeProject (analyzeProjectStaticOnly), analyzeProject) -import App.Types (Mode (..)) import App.Util (guardStrictMode) import Control.Applicative ((<|>)) import Control.Effect.Diagnostics (Diagnostics, context, fatalText, (<||>)) From d2811f9c5e7bbb21972b8fd98f453206aa331011 Mon Sep 17 00:00:00 2001 From: Jeffrey Huynh Date: Fri, 6 Sep 2024 14:42:03 -0700 Subject: [PATCH 7/8] lint --- src/App/Fossa/Config/Analyze.hs | 2 +- src/Strategy/Pub.hs | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/App/Fossa/Config/Analyze.hs b/src/App/Fossa/Config/Analyze.hs index fe55f1765f..8f8acf07df 100644 --- a/src/App/Fossa/Config/Analyze.hs +++ b/src/App/Fossa/Config/Analyze.hs @@ -338,7 +338,7 @@ cliParser = <*> optional (strOption (applyFossaStyle <> long "fossa-deps-file" <> helpDoc fossaDepsFileHelp <> metavar "FILEPATH")) <*> flagOpt StaticOnlyTactics (applyFossaStyle <> long "static-only-analysis" <> stringToHelpDoc "Only analyze the project using static strategies.") <*> withoutDefaultFilterParser fossaAnalyzeDefaultFilterDocUrl - <*> flagOpt StrictMode (applyFossaStyle <> long "strict" <> stringToHelpDoc "Enforces strict analysis to ensure the most accurate results for a strategy without resorting to fallbacks.") + <*> flagOpt StrictMode (applyFossaStyle <> long "strict" <> stringToHelpDoc "Enforces strict analysis to ensure the most accurate results for a strategy by rejecting fallbacks.") where fossaDepsFileHelp :: Maybe (Doc AnsiStyle) fossaDepsFileHelp = diff --git a/src/Strategy/Pub.hs b/src/Strategy/Pub.hs index 1ef07badea..935601cf6d 100644 --- a/src/Strategy/Pub.hs +++ b/src/Strategy/Pub.hs @@ -2,7 +2,6 @@ module Strategy.Pub (discover) where import App.Fossa.Analyze.Types (AnalyzeProject (analyzeProjectStaticOnly), analyzeProject) -import App.Types (Mode (..)) import App.Util (guardStrictMode) import Control.Carrier.Diagnostics (errDoc, errHelp) import Control.Effect.Diagnostics (Diagnostics, errCtx, fatalText, recover, warnOnErr, (<||>)) @@ -16,7 +15,7 @@ import Diag.Common ( import Discovery.Filters (AllFilters) import Discovery.Simple (simpleDiscover) import Discovery.Walk (WalkStep (WalkContinue), findFileNamed, walkWithFilters') -import Effect.Exec (Exec, GetDepsEffs, Has) +import Effect.Exec (GetDepsEffs, Has) import Effect.Logger (Logger) import Effect.ReadFS (ReadFS) import GHC.Generics (Generic) From 05f218d9f00a9083ccaa2ece5a90c1a6ce4564f9 Mon Sep 17 00:00:00 2001 From: Jeffrey Huynh Date: Sat, 7 Sep 2024 15:19:52 -0700 Subject: [PATCH 8/8] update test --- integration-test/Analysis/ScalaSpec.hs | 1 - 1 file changed, 1 deletion(-) diff --git a/integration-test/Analysis/ScalaSpec.hs b/integration-test/Analysis/ScalaSpec.hs index d7b9906628..c4a116033f 100644 --- a/integration-test/Analysis/ScalaSpec.hs +++ b/integration-test/Analysis/ScalaSpec.hs @@ -36,4 +36,3 @@ scalaExampleProject = spec :: Spec spec = do testSuiteDepResultSummary NonStrict scalaExampleProject ScalaProjectType (DependencyResultsSummary 3 2 1 1 Complete) - testSuiteDepResultSummary Strict scalaExampleProject ScalaProjectType (DependencyResultsSummary 3 2 1 1 Complete)