diff --git a/Changelog.md b/Changelog.md index 7d6d25bdc..01353e21b 100644 --- a/Changelog.md +++ b/Changelog.md @@ -2,6 +2,7 @@ ## 3.18.0 +- Go: `fossa analyze` can now report Go module dependencies read from the buildinfo embedded in compiled Go binaries (built with Go >= 1.18), so Go code shipped as a binary with no `go.mod` alongside it is no longer invisible to analysis. Opt in with `--enable-go-binary-analysis`; it is off by default because `fossa analyze` otherwise reports only what package managers declare. To reach binaries nested inside an archive (for example a `.so` inside an AAR or JAR), combine it with `--unpack-archives`. - Container scanning: `fossa container analyze` now reports Go module dependencies embedded in Go binaries (built with Go >= 1.18) found in container image layers as regular Go dependencies, supporting images without package manager metadata such as `scratch` and distroless images ([#1740](https://github.com/fossas/fossa-cli/pull/1740)) - Bun: Dependencies reachable only through a `devDependencies` entry are now reported as development dependencies instead of production dependencies. - Analysis: JSON manifest files with a leading UTF-8 byte order mark (commonly written by Windows tooling, e.g. in NuGet `project.json`) no longer fail to parse. diff --git a/docs/references/strategies/README.md b/docs/references/strategies/README.md index 66bed5b58..8bf66539c 100644 --- a/docs/references/strategies/README.md +++ b/docs/references/strategies/README.md @@ -46,6 +46,7 @@ See the linked documentation above for details. ### go - [gomodules (`go mod`)](languages/golang/gomodules.md) +- [gobinary (compiled Go binaries)](languages/golang/gobinary.md) - [dep](languages/golang/godep.md) - [glide](languages/golang/glide.md) @@ -176,6 +177,7 @@ Invoke strict analysis with the `--strict` flag when running `fossa analyze`. | [Erlang (rebar3)](https://github.com/fossas/fossa-cli/blob/master/docs/references/strategies/languages/erlang/erlang.md) | Dynamic | ❌ | | [Fortran](https://github.com/fossas/fossa-cli/blob/master/docs/references/strategies/languages/fortran/fortran.md) | Static | ❌ | | [Go (dep)](https://github.com/fossas/fossa-cli/blob/master/docs/references/strategies/languages/golang/godep.md) | Static | ❌ | +| [Go (gobinary)](https://github.com/fossas/fossa-cli/blob/master/docs/references/strategies/languages/golang/gobinary.md) | Static | ❌ | | [Go (glide)](https://github.com/fossas/fossa-cli/blob/master/docs/references/strategies/languages/golang/glide.md) | Static | ❌ | | [Go (gomodules)](https://github.com/fossas/fossa-cli/blob/master/docs/references/strategies/languages/golang/gomodules.md) | Dynamic with static fallback | ❌ | | [Gradle](https://github.com/fossas/fossa-cli/blob/master/docs/references/strategies/languages/gradle/gradle.md) | Dynamic | ❌ | diff --git a/docs/references/strategies/languages/golang/gobinary.md b/docs/references/strategies/languages/golang/gobinary.md new file mode 100644 index 000000000..1e9f0185c --- /dev/null +++ b/docs/references/strategies/languages/golang/gobinary.md @@ -0,0 +1,98 @@ +# Go Binaries (buildinfo) + +Go binaries built with module support (Go >= 1.18) embed the list of every +module linked into them. This is the same data `go version -m ` prints. + +FOSSA CLI reads that list, so Go code shipped as a compiled binary is reported +even when no `go.mod`, `go.sum`, or Go source is present next to it. + +This matters for artifacts such as: + +- a gomobile SDK shipping `jni//lib.so` inside an AAR +- a Go binary vendored into a repository that is otherwise not a Go project +- a Go binary packaged inside a JAR or other archive + +The embedded module list is generally more accurate than a hand-maintained +third-party notice file, because the linker writes it from what was actually +built into the binary. + +## Enabling + +This strategy is opt-in. `fossa analyze` otherwise reports only what package +managers declare, and reading binaries would add dependencies to existing +projects without the user asking for them. + +```bash +fossa analyze --enable-go-binary-analysis +``` + +To reach a binary nested inside an archive, pass `--unpack-archives` as well. +The two flags are independent - neither implies the other: + +```bash +fossa analyze --enable-go-binary-analysis --unpack-archives +``` + +## Project Discovery + +Walk the scan directory and sniff each file for an embedded buildinfo section. +A file is reported only if buildinfo is found and it yields at least one +usable module version. + +Binaries nested inside archives are found when `--unpack-archives` is passed: +discovery runs again over the extracted contents, so a binary inside an AAR or +JAR is reached the same way a manifest inside one would be. + +Only ELF, Mach-O, and PE files at least 4 KiB in size are examined, so the +walk is cheap on repositories that contain unrelated binary files. + +All Go binaries found in one directory are reported as a single project, because +a source unit is named after its directory. Each contributing binary appears as +an origin path, and their module lists are combined. + +Default path filters still apply: a binary under `vendor/` is skipped unless you +pass `--include-path vendor`. + +## Analysis + +The module list is read directly out of the binary; no Go toolchain is invoked +and nothing is executed. Every module found is reported as a direct `go` +dependency. + +Versions are normalized the same way `go.mod` analysis normalizes them: +pseudo-versions are reduced to their commit hash, and semantic versions keep +their `v` prefix. + +The main module is skipped when it is unversioned (the linker records `(devel)` +for a locally built binary), and reported when it carries a real version, which +happens for binaries built via `go install @`. + +## Limitations + +- Binaries built by Go < 1.18 use an older pointer-based buildinfo encoding and + are skipped. +- Binaries built without module support (`GOPATH` mode, or `CGO`-only objects) + carry no module list. +- Buildinfo records modules, not the dependency edges between them, so the + resulting graph is flat. The set of modules is complete. +- Stripping a binary does not remove buildinfo, but rewriting or packing it + (for example with UPX) can. + +## FAQ + +### How do I only perform analysis for Go binaries? + +Pass `--only-target gobinary` alongside the enabling flag: + +```bash +fossa analyze --enable-go-binary-analysis --only-target gobinary +``` + +`--only-target gobinary` on its own reports nothing, because the strategy is +still disabled. + +### How do I inspect the same data by hand? + +```bash +go version -m path/to/binary +``` diff --git a/docs/references/subcommands/analyze.md b/docs/references/subcommands/analyze.md index b59763bd9..7b774656d 100644 --- a/docs/references/subcommands/analyze.md +++ b/docs/references/subcommands/analyze.md @@ -138,6 +138,7 @@ In addition to the [standard flags](#specifying-fossa-project-details), the anal |-----------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | [`--detect-vendored`](./analyze/detect-vendored.md) | Enable the vendored source identification engine. For more information, see the [C and C++ overview](../strategies/languages/c-cpp/c-cpp.md). | | [`--detect-dynamic './some-binary`](./analyze/detect-dynamic.md) | Analyze the binary at the provided path for dynamically linked dependencies. For more information, see the [C and C++ overview](../strategies/languages/c-cpp/c-cpp.md). | +| [`--enable-go-binary-analysis`](../strategies/languages/golang/gobinary.md) | Report Go modules read from the buildinfo embedded in compiled Go binaries. Opt-in. Combine with `--unpack-archives` to reach binaries nested inside archives. | | [`--static-only-analysis`](../strategies/README.md#static-and-dynamic-strategies) | Do not use third-party tools when analyzing projects. | | `--strict` | Enforces strict analysis to ensure the most accurate results by rejecting fallbacks. When run with `--static-only-analysis`, the most optimal static strategy will be applied without fallbacks. | diff --git a/extlib/millhone/src/cmd/analyze_container.rs b/extlib/millhone/src/cmd/analyze_container.rs index fde077853..6de2d8cf6 100644 --- a/extlib/millhone/src/cmd/analyze_container.rs +++ b/extlib/millhone/src/cmd/analyze_container.rs @@ -14,7 +14,10 @@ use tar::{Archive, Entry}; use tracing::{debug, info, info_span, warn}; use typed_builder::TypedBuilder; -use super::go_buildinfo::{is_candidate_binary, scan_go_buildinfo, GoBuildInfo, GoModule}; +use super::go_buildinfo::{ + is_candidate_binary, scan_go_buildinfo, DiscoveredGoBinary, BINARY_PREFIX_LEN, + MIN_GO_BINARY_SIZE, +}; #[derive(Debug, Parser, Getters)] #[getset(get = "pub")] @@ -25,11 +28,6 @@ pub struct Subcommand { } const JAR_OBSERVATION: &str = "v1.discover.binary.jar"; -const GO_BINARY_OBSERVATION: &str = "v1.discover.binary.go"; - -/// Only sniff regular files at least this large; Go binaries are never tiny. -/// (u64 because that's what tar header sizes are.) -const MIN_GO_BINARY_SIZE: u64 = 4096; /// Magic bytes identifying a gzip stream. const GZIP_MAGIC: [u8; 2] = [0x1f, 0x8b]; @@ -61,29 +59,6 @@ struct OciManifest { #[derive(Debug, PartialEq, Eq, Serialize, Hash)] struct LayerPath(PathBuf); -/// A Go binary discovered in a layer, with the module list parsed from its -/// embedded buildinfo. -#[derive(Debug, PartialEq, Eq, Serialize, Clone)] -struct DiscoveredGoBinary { - kind: &'static str, - path: PathBuf, - go_version: String, - main_module: Option, - modules: Vec, -} - -impl DiscoveredGoBinary { - fn new(path: PathBuf, info: GoBuildInfo) -> Self { - DiscoveredGoBinary { - kind: GO_BINARY_OBSERVATION, - path, - go_version: info.go_version, - main_module: info.main_module, - modules: info.modules, - } - } -} - #[derive(Debug, PartialEq, Eq, Serialize, TypedBuilder)] struct ContainerAnalysis { /// Jars and fingerprints associated with each layer in a jar file. @@ -232,7 +207,7 @@ fn maybe_go_binary(entry: &mut Entry<'_, impl Read>, path: &Path) -> Option (Option, Vec) { (main_module, modules) } +/// Observation kind reported for a Go binary, shared by every discovery path. +pub const GO_BINARY_OBSERVATION: &str = "v1.discover.binary.go"; + +/// Only sniff regular files at least this large; Go binaries are never tiny. +pub const MIN_GO_BINARY_SIZE: u64 = 4096; + +/// How many leading bytes are read to decide whether a file is a candidate +/// binary. `is_candidate_binary` needs only 4, but reading a slightly larger +/// prefix lets tar callers stitch it back onto a forward-only stream cheaply. +pub const BINARY_PREFIX_LEN: usize = 64; + +/// A Go binary discovered somewhere (a container layer, or the filesystem), +/// with the module list parsed from its embedded buildinfo. +#[derive(Debug, PartialEq, Eq, Serialize, Clone)] +pub struct DiscoveredGoBinary { + pub kind: &'static str, + pub path: PathBuf, + pub go_version: String, + pub main_module: Option, + pub modules: Vec, +} + +impl DiscoveredGoBinary { + pub fn new(path: PathBuf, info: GoBuildInfo) -> Self { + DiscoveredGoBinary { + kind: GO_BINARY_OBSERVATION, + path, + go_version: info.go_version, + main_module: info.main_module, + modules: info.modules, + } + } +} + +/// Sniff a file on disk for embedded Go buildinfo. +/// +/// Anything that isn't a parseable Go binary - too small, wrong magic, +/// unreadable, or built by Go < 1.18 - returns `None` rather than failing, so +/// one bad file can't sink a whole scan. +pub fn scan_file(path: &Path) -> Option { + let metadata = match path.metadata() { + Ok(metadata) => metadata, + Err(e) => { + debug!(?path, "skipped: failed to stat: {e:?}"); + return None; + } + }; + if !metadata.is_file() || metadata.len() < MIN_GO_BINARY_SIZE { + return None; + } + + let mut file = match File::open(path) { + Ok(file) => file, + Err(e) => { + debug!(?path, "skipped: failed to open: {e:?}"); + return None; + } + }; + + let mut prefix = [0u8; BINARY_PREFIX_LEN]; + if let Err(e) = file.read_exact(&mut prefix) { + debug!(?path, "skipped: failed to read file prefix: {e:?}"); + return None; + } + if !is_candidate_binary(&prefix) { + return None; + } + + debug!(?path, "candidate binary; scanning for Go buildinfo"); + // The scan needs offsets relative to the start of the file, so put the + // already-consumed prefix back in front of the remaining stream. + let reader = std::io::Cursor::new(prefix).chain(file); + scan_go_buildinfo(reader).map(|info| DiscoveredGoBinary::new(path.to_path_buf(), info)) +} + +/// Scan the given paths, keeping only those that are Go binaries carrying +/// buildinfo. Order follows the input. +pub fn scan_files<'a>(paths: impl IntoIterator) -> Vec { + paths.into_iter().filter_map(scan_file).collect() +} + +#[derive(Debug, Parser)] +pub struct Subcommand { + /// Files to scan. When empty, paths are read from stdin, one per line. + /// + /// Callers with many candidates should prefer stdin: a large repository can + /// blow past the platform argument-length limit. + files: Vec, +} + +#[tracing::instrument] +pub fn main(opts: Subcommand) -> Result<()> { + let paths = if opts.files.is_empty() { + read_paths_from_stdin().context("read paths from stdin")? + } else { + opts.files + }; + + let discovered = scan_files(paths.iter().map(PathBuf::as_path)); + let mut stdout = BufWriter::new(std::io::stdout()); + serde_json::to_writer(&mut stdout, &discovered).context("serialize results") +} + +/// Read newline-delimited paths from stdin, ignoring blank lines. +fn read_paths_from_stdin() -> std::io::Result> { + let mut buf = String::new(); + std::io::stdin().read_to_string(&mut buf)?; + Ok(buf + .lines() + .map(str::trim_end) + .filter(|line| !line.is_empty()) + .map(PathBuf::from) + .collect()) +} + #[cfg(test)] mod tests { use super::*; + /// The real buildinfo region carved out of a Go binary. + const SCAN_FILE_FIXTURE: &[u8] = include_bytes!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/testdata/go-buildinfo/gh-2.86.0-darwin-arm64.bin" + )); + + /// A fake ELF: candidate magic up front, the real buildinfo fixture at a + /// 16-byte-aligned offset, padded past MIN_GO_BINARY_SIZE. + fn fake_elf_with_buildinfo() -> Vec { + let mut buf = vec![0u8; BUILDINFO_ALIGN]; + buf[..4].copy_from_slice(b"\x7fELF"); + buf.extend_from_slice(SCAN_FILE_FIXTURE); + if buf.len() < MIN_GO_BINARY_SIZE as usize { + buf.resize(MIN_GO_BINARY_SIZE as usize, 0); + } + buf + } + + /// Write bytes to a uniquely-named file under the system temp dir. + fn write_temp(name: &str, bytes: &[u8]) -> PathBuf { + let dir = std::env::temp_dir().join(format!("millhone-scan-file-{name}")); + std::fs::create_dir_all(&dir).expect("create temp dir"); + let path = dir.join(name); + std::fs::write(&path, bytes).expect("write temp file"); + path + } + + #[test] + fn scan_file_reads_go_binary_on_disk() { + let path = write_temp("go-binary", &fake_elf_with_buildinfo()); + let discovered = scan_file(&path).expect("scan fake Go binary"); + + assert_eq!(discovered.kind, GO_BINARY_OBSERVATION); + assert_eq!(discovered.path, path); + assert_eq!(discovered.go_version, "go1.25.6"); + assert!( + !discovered.modules.is_empty(), + "expected dependency modules, got none" + ); + } + + #[test] + fn scan_file_skips_files_that_are_not_go_binaries() { + // Big enough, but no candidate-binary magic. + let text = write_temp("not-a-binary", &vec![b'a'; MIN_GO_BINARY_SIZE as usize]); + assert_eq!(scan_file(&text), None, "plain text should be skipped"); + + // Right magic, but below the size floor. + let tiny = write_temp("tiny-elf", b"\x7fELF and nothing else"); + assert_eq!(scan_file(&tiny), None, "tiny file should be skipped"); + + // A directory is not a file. + let dir = std::env::temp_dir().join("millhone-scan-file-dir"); + std::fs::create_dir_all(&dir).expect("create temp dir"); + assert_eq!(scan_file(&dir), None, "directory should be skipped"); + + // A path that does not exist at all. + assert_eq!(scan_file(&dir.join("missing")), None, "missing file"); + } + + #[test] + fn scan_files_keeps_only_go_binaries() { + let go_binary = write_temp("batch-go-binary", &fake_elf_with_buildinfo()); + let text = write_temp("batch-text", &vec![b'a'; MIN_GO_BINARY_SIZE as usize]); + + let discovered = scan_files([go_binary.as_path(), text.as_path()]); + assert_eq!(discovered.len(), 1, "only the Go binary should be reported"); + assert_eq!(discovered[0].path, go_binary); + } + /// Build a synthetic buildinfo blob: padding to 16-byte alignment, magic, /// ptrSize+flags, then inline go version + sentinel-wrapped modinfo. fn synthetic_buildinfo(pad: usize, flags: u8, go_version: &str, modinfo_body: &str) -> Vec { diff --git a/extlib/millhone/src/main.rs b/extlib/millhone/src/main.rs index 942eb91d2..553b56609 100644 --- a/extlib/millhone/src/main.rs +++ b/extlib/millhone/src/main.rs @@ -82,6 +82,11 @@ impl Application { enum Commands { /// Find and fingerprint JAR files. AnalyzeContainer(cmd::analyze_container::Subcommand), + + /// Extract Go module lists from the buildinfo embedded in Go binaries. + /// + /// Paths are read from stdin (one per line) unless given as arguments. + AnalyzeGoBinaries(cmd::go_buildinfo::Subcommand), } fn main() -> stable_eyre::Result<()> { @@ -121,6 +126,7 @@ fn main() -> stable_eyre::Result<()> { // And then dispatch to the subcommand. match app.commands { Commands::AnalyzeContainer(opts) => cmd::analyze_container::main(opts), + Commands::AnalyzeGoBinaries(opts) => cmd::go_buildinfo::main(opts), } } diff --git a/spectrometer.cabal b/spectrometer.cabal index 497492057..c5c3bfae8 100644 --- a/spectrometer.cabal +++ b/spectrometer.cabal @@ -449,6 +449,7 @@ library Strategy.Fpm Strategy.Glide Strategy.Go.GlideLock + Strategy.Go.GoBinary Strategy.Go.GoListPackages Strategy.Go.Gomod Strategy.Go.GopkgLock @@ -672,6 +673,7 @@ test-suite unit-tests Fossa.API.CoreTypesSpec Fossa.API.TypesSpec Go.GlideLockSpec + Go.GoBinarySpec Go.GoListPackagesSpec Go.GomodSpec Go.GopkgLockSpec diff --git a/src/App/Fossa/Analyze/Discover.hs b/src/App/Fossa/Analyze/Discover.hs index a715fec77..5b5b3dad0 100644 --- a/src/App/Fossa/Analyze/Discover.hs +++ b/src/App/Fossa/Analyze/Discover.hs @@ -16,6 +16,7 @@ import Strategy.Composer qualified as Composer import Strategy.Conda qualified as Conda import Strategy.Fpm qualified as Fpm import Strategy.Glide qualified as Glide +import Strategy.Go.GoBinary qualified as GoBinary import Strategy.Godep qualified as Godep import Strategy.Gomodules qualified as Gomodules import Strategy.Googlesource.RepoManifest qualified as RepoManifest @@ -58,6 +59,7 @@ discoverFuncs = , DiscoverFunc Fpm.discover , DiscoverFunc Glide.discover , DiscoverFunc Godep.discover + , DiscoverFunc GoBinary.discover , DiscoverFunc Gomodules.discover , DiscoverFunc Gradle.discover , DiscoverFunc Leiningen.discover diff --git a/src/App/Fossa/Config/Analyze.hs b/src/App/Fossa/Config/Analyze.hs index e946ab04c..b7d855768 100644 --- a/src/App/Fossa/Config/Analyze.hs +++ b/src/App/Fossa/Config/Analyze.hs @@ -243,6 +243,7 @@ data AnalyzeCliOpts = AnalyzeCliOpts , analyzeBaseDir :: FilePath , analyzeDeprecatedUseV3GoResolver :: Flag DeprecatedUseV3GoResolver , analyzePathDependencies :: Bool + , analyzeGoBinaryAnalysis :: Bool , analyzeForceFirstPartyScans :: Flag ForceFirstPartyScans , analyzeForceNoFirstPartyScans :: Flag ForceNoFirstPartyScans , analyzeIgnoreOrgWideCustomLicenseScanConfigs :: Flag IgnoreOrgWideCustomLicenseScanConfigs @@ -310,6 +311,10 @@ data StrategyConfig = StrategyConfig { allowedGradleConfigs :: Maybe (Set Text) , resolvePathDependencies :: Bool , useGitBackedCargoLocators :: UseGitBackedCargoLocators + , enableGoBinaryAnalysis :: Bool + -- ^ Read Go module lists from the buildinfo embedded in compiled Go + -- binaries. Opt-in: @fossa analyze@ otherwise reports only what package + -- managers declare. } deriving (Eq, Ord, Show, Generic) @@ -364,6 +369,7 @@ cliParser = <*> baseDirArg <*> experimentalUseV3GoResolver <*> experimentalAnalyzePathDependencies + <*> enableGoBinaryAnalysisParser <*> flagOpt ForceFirstPartyScans (applyFossaStyle <> long "experimental-force-first-party-scans" <> stringToHelpDoc "Force first party scans") <*> flagOpt ForceNoFirstPartyScans (applyFossaStyle <> long "experimental-block-first-party-scans" <> stringToHelpDoc "Block first party scans. This can be used to forcibly turn off first-party scans if your organization defaults to first-party scans.") <*> flagOpt IgnoreOrgWideCustomLicenseScanConfigs (applyFossaStyle <> long "ignore-org-wide-custom-license-scan-configs" <> stringToHelpDoc "Ignore custom-license scan configurations for your organization. These configurations are defined in the `Integrations` section of the Admin settings in the FOSSA web app") @@ -405,6 +411,13 @@ withoutDefaultFilterParser docsUrl = flagOpt WithoutDefaultFilters (applyFossaSt experimentalUseV3GoResolver :: Parser (Flag DeprecatedUseV3GoResolver) experimentalUseV3GoResolver = flagOpt DeprecatedUseV3GoResolver (applyFossaStyle <> long "experimental-use-v3-go-resolver" <> hidden) +enableGoBinaryAnalysisParser :: Parser Bool +enableGoBinaryAnalysisParser = + switch $ + long "enable-go-binary-analysis" + <> applyFossaStyle + <> stringToHelpDoc "Report Go modules read from the buildinfo embedded in compiled Go binaries. Combine with --unpack-archives to reach binaries inside archives." + experimentalAnalyzePathDependencies :: Parser Bool experimentalAnalyzePathDependencies = switch $ @@ -684,7 +697,7 @@ collectCLIFilters AnalyzeCliOpts{..} = (comboExclude analyzeExcludeTargets analyzeExcludePaths) collectStrategyConfig :: Maybe ConfigFile -> AnalyzeCliOpts -> StrategyConfig -collectStrategyConfig maybeCfg AnalyzeCliOpts{analyzePathDependencies = shouldAnalyzePathDependencies} = +collectStrategyConfig maybeCfg AnalyzeCliOpts{analyzePathDependencies = shouldAnalyzePathDependencies, analyzeGoBinaryAnalysis} = StrategyConfig ( fmap gradleConfigsOnly @@ -692,6 +705,7 @@ collectStrategyConfig maybeCfg AnalyzeCliOpts{analyzePathDependencies = shouldAn ) shouldAnalyzePathDependencies (UseGitBackedCargoLocators True) + analyzeGoBinaryAnalysis collectVendoredDeps :: (Has Diagnostics sig m) => diff --git a/src/App/Fossa/Config/ListTargets.hs b/src/App/Fossa/Config/ListTargets.hs index 25eb797e9..ad372f85d 100644 --- a/src/App/Fossa/Config/ListTargets.hs +++ b/src/App/Fossa/Config/ListTargets.hs @@ -124,6 +124,7 @@ collectStrategyConfig maybeCfg = ) False -- This should be ok because discovery has no impact on whether, analysis includes path dependency or not! (UseGitBackedCargoLocators True) -- Default to git-backed cargo locators when no org info is available + False -- Go binary analysis is opt-in; listing its targets would imply it runs by default data ListTargetsCliOpts = ListTargetsCliOpts { commons :: CommonOpts diff --git a/src/App/Fossa/Container/Sources/DockerArchive.hs b/src/App/Fossa/Container/Sources/DockerArchive.hs index 5619239bc..59e48ccb4 100644 --- a/src/App/Fossa/Container/Sources/DockerArchive.hs +++ b/src/App/Fossa/Container/Sources/DockerArchive.hs @@ -237,6 +237,7 @@ analyzeLayer useGitBackedCargo systemDepsOnly filters withoutDefaultFilters capa Nothing False -- Discovery has no consequence from path dependency analysis config useGitBackedCargo + False -- Layers get Go binaries from millhone's own layer scan; running the filesystem strategy here would double-report toSourceUnit :: [DiscoveredProjectScan] -> [SourceUnit] toSourceUnit = map (Srclib.projectToSourceUnit False) @@ -391,6 +392,7 @@ listTargetLayer capabilities osInfo layerFs tarball layerType = do Nothing False -- Targets are not impacted by path dependencies. (UseGitBackedCargoLocators True) -- Default to git-backed cargo locators when no org info is available + False -- Go binaries in layers come from millhone's layer scan, not this strategy ) . runReader (MavenScopeIncludeFilters mempty) . runReader NonStrict diff --git a/src/App/Fossa/Container/Sources/GoBinary.hs b/src/App/Fossa/Container/Sources/GoBinary.hs index 42dfc659c..7c5d83b92 100644 --- a/src/App/Fossa/Container/Sources/GoBinary.hs +++ b/src/App/Fossa/Container/Sources/GoBinary.hs @@ -11,13 +11,8 @@ import Container.Types (DiscoveredGoBinary (..), GoModule (..)) import Data.Aeson qualified as Aeson import Data.List (nub) import Data.Maybe (mapMaybe, maybeToList) -import Data.SemVer qualified as SemVer -import Data.SemVer.Internal (Version (..)) -import Data.Text (Text) -import Data.Text qualified as Text import Srclib.Types (Locator (..), SourceUnit (..), SourceUnitBuild (..), SourceUnitDependency (..), textToOriginPath) -import Strategy.Go.Gomod (PackageVersion (..), parsePackageVersion) -import Text.Megaparsec (parseMaybe) +import Strategy.Go.GoBinary (normalizeVersion) import Types (GraphBreadth (..)) -- | One source unit per discovered binary. Binaries with no usable module @@ -75,13 +70,3 @@ goModuleToLocator (GoModule path version) = do , locatorProject = path , locatorRevision = Just normalized } - -normalizeVersion :: Text -> Maybe Text -normalizeVersion version = - if Text.null version || version == "(devel)" - then Nothing - else case parseMaybe (parsePackageVersion id) version of - Just (Pseudo commitHash) -> Just commitHash - Just (Semantic semver) -> Just ("v" <> SemVer.toText semver{_versionMeta = []}) - Just (NonCanonical v) -> Just v - Nothing -> Just version diff --git a/src/Container/Types.hs b/src/Container/Types.hs index 81803c2fe..fd93a7e4f 100644 --- a/src/Container/Types.hs +++ b/src/Container/Types.hs @@ -21,6 +21,8 @@ module Container.Types ( DiscoveredBinaries (..), -- * Go Binary Analysis Related Types + + -- Defined alongside the filesystem discovery strategy that shares them. GoModule (..), DiscoveredGoBinary (..), @@ -44,6 +46,7 @@ import Data.String.Conversion (ToText, toText) import Data.Text (Text) import GHC.Generics (Generic) import Srclib.Types (SourceUnit) +import Strategy.Go.GoBinary (DiscoveredGoBinary (..), GoModule (..)) data ContainerImageRaw = ContainerImageRaw { layers :: NonEmpty.NonEmpty ContainerLayer @@ -149,37 +152,6 @@ newtype JarObservation = JarObservation deriving (Eq, Ord, Show, Generic) deriving (ToJSON, FromJSON) via Value --- | A Go module parsed from a binary's embedded buildinfo. -data GoModule = GoModule - { goModulePath :: Text - , goModuleVersion :: Text - } - deriving (Eq, Ord, Show) - -instance FromJSON GoModule where - parseJSON = withObject "GoModule" $ \o -> - GoModule - <$> o .: "path" - <*> o .: "version" - --- | A Go binary millhone discovered in a layer, with the module list parsed --- from its embedded buildinfo (the data @go version -m@ reads). -data DiscoveredGoBinary = DiscoveredGoBinary - { goBinaryPath :: Text - , goBinaryGoVersion :: Text - , goBinaryMainModule :: Maybe GoModule - , goBinaryModules :: [GoModule] - } - deriving (Eq, Ord, Show) - -instance FromJSON DiscoveredGoBinary where - parseJSON = withObject "DiscoveredGoBinary" $ \o -> - DiscoveredGoBinary - <$> o .: "path" - <*> o .: "go_version" - <*> o .:? "main_module" - <*> o .: "modules" - -- | Output parse type for millhone: everything (jars and Go binaries) it -- discovered per layer. data DiscoveredBinaries = DiscoveredBinaries diff --git a/src/Strategy/Go/GoBinary.hs b/src/Strategy/Go/GoBinary.hs new file mode 100644 index 000000000..1195e07e1 --- /dev/null +++ b/src/Strategy/Go/GoBinary.hs @@ -0,0 +1,284 @@ +-- | Discover compiled Go binaries on the filesystem and read the module list +-- from the buildinfo the Go toolchain embeds in them (the data +-- @go version -m@ prints). +-- +-- This covers Go code shipped as a binary with no manifest alongside it: +-- a gomobile @.so@ inside an AAR, a vendored CLI, a binary nested in a JAR. +-- Manifest-based analysis ('Strategy.Gomodules') cannot see those, because +-- there is no @go.mod@ to find. +-- +-- Opt-in via @--enable-go-binary-analysis@: @fossa analyze@ otherwise reports +-- only what package managers declare, and reading binaries would add +-- dependencies to existing projects without the user asking for them. +-- +-- Because this is a normal discovery strategy, it inherits path filters and, +-- under @--unpack-archives@, runs again over extracted archive contents - which +-- is what reaches binaries nested inside AARs and JARs. The two flags are +-- independent: archives need both. +-- +-- The buildinfo parsing itself lives in millhone (Rust), shared with the +-- container analysis path; this module shells out to it and converts the +-- result into a dependency graph. +module Strategy.Go.GoBinary ( + discover, + findProjects, + toProjects, + mkProject, + getDeps, + GoBinaryProject (..), + GoModule (..), + DiscoveredGoBinary (..), + goBinaryDependencies, + goModuleToDependency, + normalizeVersion, +) where + +import App.Fossa.Analyze.Types (AnalyzeProject (analyzeProject, analyzeProjectStaticOnly)) +import App.Fossa.Config.Analyze (StrategyConfig (enableGoBinaryAnalysis)) +import App.Fossa.EmbeddedBinary (BinaryPaths, toPath, withMillhoneBinary) +import Control.Effect.Diagnostics (Diagnostics, context, warnThenRecover) +import Control.Effect.Lift (Lift) +import Control.Effect.Reader (Reader, ask) +import Control.Monad (filterM) +import Data.Aeson (FromJSON, ToJSON, parseJSON, withObject, (.:), (.:?)) +import Data.List (nub, sortOn) +import Data.List.NonEmpty qualified as NE +import Data.Map qualified as Map +import Data.Maybe (mapMaybe, maybeToList) +import Data.SemVer qualified as SemVer +import Data.SemVer.Internal (Version (..)) +import Data.String.Conversion (toText) +import Data.Text (Text) +import Data.Text qualified as Text +import DepTypes ( + DepType (GoType), + Dependency (..), + VerConstraint (CEq), + ) +import Discovery.Filters (AllFilters) +import Discovery.Simple (simpleDiscover) +import Discovery.Walk (WalkStep (WalkContinue), walkWithFilters') +import Effect.Exec (AllowErr (Never), Command (..), Exec, Has, execJson') +import Effect.ReadFS (ReadFS, contentIsBinary) +import GHC.Generics (Generic) +import Graphing qualified +import Path (Abs, Dir, File, Path, parent) +import Strategy.Go.Gomod (PackageVersion (..), parsePackageVersion) +import Text.Megaparsec (parseMaybe) +import Types ( + DependencyResults (..), + DiscoveredProject (..), + DiscoveredProjectType (GoBinaryProjectType), + GraphBreadth (Complete), + ) + +-- | A Go module (path + version) parsed from a binary's embedded buildinfo. +data GoModule = GoModule + { goModulePath :: Text + , goModuleVersion :: Text + } + deriving (Eq, Ord, Show) + +instance FromJSON GoModule where + parseJSON = withObject "GoModule" $ \o -> + GoModule + <$> o .: "path" + <*> o .: "version" + +-- | A Go binary millhone found, with the module list parsed from its embedded +-- buildinfo. Shared with the container analysis path, which discovers the same +-- shape inside image layers. +data DiscoveredGoBinary = DiscoveredGoBinary + { goBinaryPath :: Text + , goBinaryGoVersion :: Text + , goBinaryMainModule :: Maybe GoModule + , goBinaryModules :: [GoModule] + } + deriving (Eq, Ord, Show) + +instance FromJSON DiscoveredGoBinary where + parseJSON = withObject "DiscoveredGoBinary" $ \o -> + DiscoveredGoBinary + <$> o .: "path" + <*> o .: "go_version" + <*> o .:? "main_module" + <*> o .: "modules" + +-- | The Go binaries found in one directory, with their dependencies already +-- read at discovery time. Analysis is therefore pure: millhone is invoked once +-- per scanned directory tree, not once per binary. +-- +-- A project is a directory rather than a single binary because a source unit +-- is named after its directory ('Srclib.Converter.toSourceUnit'); one project +-- per binary would emit units that collide whenever a directory holds more +-- than one Go binary. Each contributing binary stays visible as an origin path. +data GoBinaryProject = GoBinaryProject + { goBinaryProjectDir :: Path Abs Dir + , goBinaryProjectFiles :: [Path Abs File] + , goBinaryProjectDeps :: [Dependency] + } + deriving (Eq, Ord, Show, Generic) + +instance ToJSON GoBinaryProject + +instance AnalyzeProject GoBinaryProject where + analyzeProject _ = getDeps + + -- Reading bytes already on disk; no build tool is invoked. + analyzeProjectStaticOnly _ = getDeps + +discover :: + ( Has ReadFS sig m + , Has Exec sig m + , Has Diagnostics sig m + , Has (Lift IO) sig m + , Has (Reader AllFilters) sig m + , Has (Reader StrategyConfig) sig m + ) => + Path Abs Dir -> + m [DiscoveredProject GoBinaryProject] +discover = simpleDiscover findProjects mkProject GoBinaryProjectType + +-- | Walk for candidate files, then hand the whole batch to millhone in a single +-- invocation. 'contentIsBinary' is a cheap pre-filter (it only reads a prefix); +-- millhone applies the precise magic/size checks and the buildinfo parse. +findProjects :: + ( Has ReadFS sig m + , Has Exec sig m + , Has Diagnostics sig m + , Has (Lift IO) sig m + , Has (Reader AllFilters) sig m + , Has (Reader StrategyConfig) sig m + ) => + Path Abs Dir -> + m [GoBinaryProject] +findProjects dir = do + enabled <- enableGoBinaryAnalysis <$> ask + if not enabled + then pure [] + else do + candidates <- walkWithFilters' collectBinaries dir + if null candidates + then pure [] + else do + -- A failure here (millhone missing, unparseable output) should not + -- sink the whole scan, but it must not pass silently either: the + -- user asked for this analysis and would otherwise get zero Go + -- dependencies with no explanation. + discovered <- + warnThenRecover @Text "Error reading Go buildinfo (millhone)" $ + context "Reading Go buildinfo" $ + analyzeGoBinaries dir candidates + pure . toProjects candidates $ concat discovered + where + collectBinaries _ _ files = do + binaries <- filterM contentIsBinary files + pure (binaries, WalkContinue) + +-- | Match millhone's results back to the paths we handed it and group them by +-- containing directory, dropping any binary whose buildinfo yields no usable +-- dependency (for example a binary whose only module is the unversioned +-- @(devel)@ main module). +toProjects :: [Path Abs File] -> [DiscoveredGoBinary] -> [GoBinaryProject] +toProjects candidates discovered = map toProject . Map.toAscList $ Map.fromListWith merge byDir + where + byPath = Map.fromList [(toText path, path) | path <- candidates] + + byDir = + [ (parent path, (NE.singleton path, deps)) + | binary <- discovered + , let deps = goBinaryDependencies binary + , not (null deps) + , Just path <- [Map.lookup (goBinaryPath binary) byPath] + ] + + -- 'Map.fromListWith' applies the later entry first; flip so paths and + -- dependencies stay in the order the binaries were discovered. + merge newer older = older <> newer + + toProject (dir, (paths, deps)) = + GoBinaryProject + { goBinaryProjectDir = dir + , goBinaryProjectFiles = sortOn toText $ NE.toList paths + , goBinaryProjectDeps = nub deps + } + +mkProject :: GoBinaryProject -> DiscoveredProject GoBinaryProject +mkProject project = + DiscoveredProject + { projectType = GoBinaryProjectType + , projectBuildTargets = mempty + , projectPath = goBinaryProjectDir project + , projectData = project + } + +getDeps :: (Applicative m) => GoBinaryProject -> m DependencyResults +getDeps project = + pure + DependencyResults + { dependencyGraph = Graphing.directs $ goBinaryProjectDeps project + , -- Buildinfo records every module linked into the binary, but carries no + -- edges between them, so the graph is a flat complete set. + dependencyGraphBreadth = Complete + , dependencyManifestFiles = goBinaryProjectFiles project + } + +-- | Every usable dependency in a binary's buildinfo. +-- +-- The main module is normally versioned @(devel)@ and dropped, but binaries +-- built via @go install module\@version@ carry a real version. +goBinaryDependencies :: DiscoveredGoBinary -> [Dependency] +goBinaryDependencies binary = + nub . mapMaybe goModuleToDependency $ + goBinaryModules binary <> maybeToList (goBinaryMainModule binary) + +goModuleToDependency :: GoModule -> Maybe Dependency +goModuleToDependency (GoModule path version) = do + normalized <- normalizeVersion version + Just + Dependency + { dependencyType = GoType + , dependencyName = path + , dependencyVersion = Just $ CEq normalized + , dependencyLocations = [] + , dependencyEnvironments = mempty + , dependencyTags = mempty + } + +-- | Normalize a buildinfo version the same way go.mod analysis does +-- ('Strategy.Go.GoListPackages.toVerConstraint'): pseudo-versions become their +-- commit hash, semantic versions keep their "v" prefix. Unusable versions +-- (empty, @(devel)@) yield 'Nothing'. +normalizeVersion :: Text -> Maybe Text +normalizeVersion version = + if Text.null version || version == "(devel)" + then Nothing + else case parseMaybe (parsePackageVersion id) version of + Just (Pseudo commitHash) -> Just commitHash + Just (Semantic semver) -> Just ("v" <> SemVer.toText semver{_versionMeta = []}) + Just (NonCanonical v) -> Just v + Nothing -> Just version + +-- | Millhone reads the candidate paths from stdin, one per line: a large +-- repository can have more candidates than the platform argument limit allows. +analyzeGoBinaries :: + ( Has Exec sig m + , Has Diagnostics sig m + , Has (Lift IO) sig m + ) => + Path Abs Dir -> + [Path Abs File] -> + m [DiscoveredGoBinary] +analyzeGoBinaries dir candidates = withMillhoneBinary $ \binaryPaths -> + execJson' dir (millhoneGoBinaryCmd binaryPaths) stdin + where + stdin = Text.unlines $ map toText candidates + +millhoneGoBinaryCmd :: BinaryPaths -> Command +millhoneGoBinaryCmd binaryPaths = + Command + { cmdName = toText . toPath $ binaryPaths + , cmdArgs = ["--log-to", "stderr", "analyze-go-binaries"] + , cmdAllowErr = Never + , cmdEnvVars = Map.empty + } diff --git a/src/Types.hs b/src/Types.hs index 69c82e029..841404c4f 100644 --- a/src/Types.hs +++ b/src/Types.hs @@ -79,6 +79,7 @@ data DiscoveredProjectType | DpkgDatabaseProjectType | FpmProjectType | GlideProjectType + | GoBinaryProjectType | GodepProjectType | GomodProjectType | GradleProjectType @@ -132,6 +133,7 @@ projectTypeToText = \case DpkgDatabaseProjectType -> "dpkgdb" FpmProjectType -> "fpm" GlideProjectType -> "glide" + GoBinaryProjectType -> "gobinary" GodepProjectType -> "godep" GomodProjectType -> "gomod" GradleProjectType -> "gradle" diff --git a/test/App/Fossa/AnalyzeSpec.hs b/test/App/Fossa/AnalyzeSpec.hs index db28cf11d..d3ab698f2 100644 --- a/test/App/Fossa/AnalyzeSpec.hs +++ b/test/App/Fossa/AnalyzeSpec.hs @@ -21,5 +21,5 @@ spec :: Spec spec = -- this test only exists to prevent merging the commented out analyzers describe "Discovery function list" $ - it "should be length 36" $ - length (discoverFuncs :: [DiscoverFunc SomeMonad]) `shouldBe` 36 + it "should be length 37" $ + length (discoverFuncs :: [DiscoverFunc SomeMonad]) `shouldBe` 37 diff --git a/test/Go/GoBinarySpec.hs b/test/Go/GoBinarySpec.hs new file mode 100644 index 000000000..94adf0174 --- /dev/null +++ b/test/Go/GoBinarySpec.hs @@ -0,0 +1,138 @@ +{-# LANGUAGE TemplateHaskell #-} + +module Go.GoBinarySpec (spec) where + +import Data.Aeson (eitherDecode) +import Data.ByteString.Lazy (ByteString) +import Data.Text (Text) +import DepTypes ( + DepType (GoType), + Dependency (..), + VerConstraint (CEq), + ) +import Path (mkAbsDir, mkAbsFile) +import Strategy.Go.GoBinary ( + DiscoveredGoBinary (..), + GoBinaryProject (..), + GoModule (..), + goBinaryDependencies, + goModuleToDependency, + toProjects, + ) +import Test.Hspec (Spec, describe, it, shouldBe) + +-- Output from @millhone analyze-go-binaries@ for a single Go binary. +millhoneOutput :: ByteString +millhoneOutput = + "[{\"kind\":\"v1.discover.binary.go\",\"path\":\"/src/jni/arm64-v8a/libgojni.so\",\ + \\"go_version\":\"go1.25.6\",\ + \\"main_module\":{\"path\":\"example.com/sdk\",\"version\":\"(devel)\"},\ + \\"modules\":[\ + \{\"path\":\"github.com/google/uuid\",\"version\":\"v1.6.0\"},\ + \{\"path\":\"golang.org/x/sys\",\"version\":\"v0.0.0-20220715151400-c0bba94af5f8\"}\ + \]}]" + +expectedBinary :: DiscoveredGoBinary +expectedBinary = + DiscoveredGoBinary + { goBinaryPath = "/src/jni/arm64-v8a/libgojni.so" + , goBinaryGoVersion = "go1.25.6" + , goBinaryMainModule = Just (GoModule "example.com/sdk" "(devel)") + , goBinaryModules = + [ GoModule "github.com/google/uuid" "v1.6.0" + , GoModule "golang.org/x/sys" "v0.0.0-20220715151400-c0bba94af5f8" + ] + } + +mkDep :: Text -> Text -> Dependency +mkDep name version = + Dependency + { dependencyType = GoType + , dependencyName = name + , dependencyVersion = Just $ CEq version + , dependencyLocations = [] + , dependencyEnvironments = mempty + , dependencyTags = mempty + } + +spec :: Spec +spec = do + describe "millhone analyze-go-binaries output decoding" $ + it "decodes discovered go binaries" $ + eitherDecode millhoneOutput `shouldBe` Right [expectedBinary] + + describe "goModuleToDependency" $ do + it "renders semantic versions with the v prefix" $ + goModuleToDependency (GoModule "github.com/google/uuid" "v1.6.0") + `shouldBe` Just (mkDep "github.com/google/uuid" "v1.6.0") + + it "normalizes pseudo-versions to their commit hash" $ + goModuleToDependency (GoModule "golang.org/x/sys" "v0.0.0-20220715151400-c0bba94af5f8") + `shouldBe` Just (mkDep "golang.org/x/sys" "c0bba94af5f8") + + it "drops (devel) and empty versions" $ do + goModuleToDependency (GoModule "example.com/sdk" "(devel)") `shouldBe` Nothing + goModuleToDependency (GoModule "example.com/sdk" "") `shouldBe` Nothing + + describe "goBinaryDependencies" $ do + it "reports every dependency module, dropping an unversioned main module" $ + goBinaryDependencies expectedBinary + `shouldBe` [ mkDep "github.com/google/uuid" "v1.6.0" + , mkDep "golang.org/x/sys" "c0bba94af5f8" + ] + + it "keeps a main module that carries a real version" $ + -- Binaries built via `go install module@version` record one. + goBinaryDependencies + expectedBinary{goBinaryMainModule = Just (GoModule "example.com/sdk" "v1.2.3")} + `shouldBe` [ mkDep "github.com/google/uuid" "v1.6.0" + , mkDep "golang.org/x/sys" "c0bba94af5f8" + , mkDep "example.com/sdk" "v1.2.3" + ] + + it "reports nothing when no module carries a usable version" $ + goBinaryDependencies + expectedBinary + { goBinaryModules = [] + , goBinaryMainModule = Just (GoModule "example.com/sdk" "(devel)") + } + `shouldBe` [] + + describe "toProjects" $ do + -- Source units are named after their directory, so one project per binary + -- would emit colliding units for a directory holding several Go binaries. + it "groups binaries in one directory into a single project" $ do + let toolA = $(mkAbsFile "/src/tools/toolA") + toolB = $(mkAbsFile "/src/tools/toolB") + binA = expectedBinary{goBinaryPath = "/src/tools/toolA"} + binB = + expectedBinary + { goBinaryPath = "/src/tools/toolB" + , goBinaryModules = [GoModule "github.com/urfave/cli/v3" "v3.3.3"] + } + toProjects [toolA, toolB] [binA, binB] + `shouldBe` [ GoBinaryProject + { goBinaryProjectDir = $(mkAbsDir "/src/tools/") + , goBinaryProjectFiles = [toolA, toolB] + , goBinaryProjectDeps = + [ mkDep "github.com/google/uuid" "v1.6.0" + , mkDep "golang.org/x/sys" "c0bba94af5f8" + , mkDep "github.com/urfave/cli/v3" "v3.3.3" + ] + } + ] + + it "keeps binaries in different directories as separate projects" $ do + let toolA = $(mkAbsFile "/src/tools/toolA") + nested = $(mkAbsFile "/src/jni/libgojni.so") + binA = expectedBinary{goBinaryPath = "/src/tools/toolA"} + binNested = expectedBinary{goBinaryPath = "/src/jni/libgojni.so"} + map goBinaryProjectDir (toProjects [toolA, nested] [binA, binNested]) + `shouldBe` [$(mkAbsDir "/src/jni/"), $(mkAbsDir "/src/tools/")] + + it "drops binaries with no usable dependency and paths millhone was not given" $ do + let toolA = $(mkAbsFile "/src/tools/toolA") + devel = expectedBinary{goBinaryPath = "/src/tools/toolA", goBinaryModules = [], goBinaryMainModule = Just (GoModule "example.com/sdk" "(devel)")} + unknown = expectedBinary{goBinaryPath = "/somewhere/else"} + toProjects [toolA] [devel] `shouldBe` [] + toProjects [toolA] [unknown] `shouldBe` [] diff --git a/test/Test/Fixtures.hs b/test/Test/Fixtures.hs index 961448a62..75c053a65 100644 --- a/test/Test/Fixtures.hs +++ b/test/Test/Fixtures.hs @@ -654,6 +654,7 @@ fixtureStrategyConfig = { allowedGradleConfigs = Nothing , resolvePathDependencies = False , useGitBackedCargoLocators = ANZ.UseGitBackedCargoLocators True + , enableGoBinaryAnalysis = False } vendoredDepsOptions :: VendoredDependencyOptions