diff --git a/cli/docs/flags.go b/cli/docs/flags.go index 9bde0bf4b..baeee469b 100644 --- a/cli/docs/flags.go +++ b/cli/docs/flags.go @@ -160,6 +160,7 @@ const ( // Unique curation flags CurationOutput = "curation-format" DockerImageName = "image" + HuggingFaceModel = "hugging-face-model" SolutionPath = "solution-path" IncludeCachedPackages = "include-cached-packages" LegacyPeerDeps = "legacy-peer-deps" @@ -228,7 +229,7 @@ var commandFlags = map[string][]string{ StaticSca, XrayLibPluginBinaryCustomPath, AnalyzerManagerCustomPath, AddSastRules, }, CurationAudit: { - CurationOutput, WorkingDirs, Threads, RequirementsFile, InsecureTls, useWrapperAudit, UseIncludedBuilds, SolutionPath, DockerImageName, IncludeCachedPackages, MvnIncludePluginDeps, LegacyPeerDeps, RunNative, + CurationOutput, WorkingDirs, Threads, RequirementsFile, InsecureTls, useWrapperAudit, UseIncludedBuilds, SolutionPath, DockerImageName, HuggingFaceModel, IncludeCachedPackages, MvnIncludePluginDeps, LegacyPeerDeps, RunNative, }, GitCountContributors: { InputFile, ScmType, ScmApiUrl, Token, Owner, RepoName, Months, DetailedSummary, InsecureTls, GitThreads, CacheValidity, @@ -370,7 +371,8 @@ var flagsMap = map[string]components.Flag{ UseConfigProfile: components.NewBoolFlag(UseConfigProfile, "Set to false to override config profile for the audit.", components.WithBoolDefaultValue(true), components.SetHiddenBoolFlag()), // Docker flags - DockerImageName: components.NewStringFlag(DockerImageName, "Specifies the Docker image name to audit. Uses the same format as the Docker CLI, including Artifactory-hosted images."), + DockerImageName: components.NewStringFlag(DockerImageName, "Specifies the Docker image name to audit. Uses the same format as the Docker CLI, including Artifactory-hosted images."), + HuggingFaceModel: components.NewStringFlag(HuggingFaceModel, "Hugging Face models to audit, as '[:]' (revision defaults to 'main'). Multiple entries are comma-separated (e.g. 'mcpotato/42-eicar-street:main,bert-base-uncased'). Datasets are not audited (curation does not currently cover datasets). Requires HF_ENDPOINT set to the Artifactory Hugging Face repository URL."), // Git flags InputFile: components.NewStringFlag(InputFile, "Path to an input file in YAML format contains multiple git providers. With this option, all other scm flags will be ignored and only git servers mentioned in the file will be examined.."), diff --git a/cli/docs/scan/curation/help.go b/cli/docs/scan/curation/help.go index 182736e7e..b40b7b11a 100644 --- a/cli/docs/scan/curation/help.go +++ b/cli/docs/scan/curation/help.go @@ -14,7 +14,7 @@ When to use: Prerequisites: - A configured JFrog Platform server (jf c add) with JFrog Curation entitlement. -- Project must use a supported package manager (npm, yarn, pip, maven, gradle, nuget, go) resolved through a curation-configured remote. +- Project must use a supported package manager (npm, yarn, pip, maven, gradle, nuget, go) resolved through a curation-configured remote. Docker images and Hugging Face models are audited via dedicated flags (datasets are detected but not audited). - The package manager and its lockfile must be present in the working directory. Common patterns: @@ -23,6 +23,7 @@ Common patterns: $ jf curation-audit --format=json --threads=4 $ jf curation-audit --requirements-file=requirements-dev.txt $ jf curation-audit --docker-image=my-image:tag + $ HF_ENDPOINT=https://my.jfrog.io/artifactory/api/huggingfaceml/my-hf-repo jf curation-audit --hugging-face-model=org/model:main Gotchas: - The user/token must be entitled for Curation; otherwise the command exits with an entitlement notice. diff --git a/cli/scancommands.go b/cli/scancommands.go index 05ba99a71..c955ed709 100644 --- a/cli/scancommands.go +++ b/cli/scancommands.go @@ -34,6 +34,7 @@ import ( uploadCdxDocs "github.com/jfrog/jfrog-cli-security/cli/docs/upload" "github.com/jfrog/jfrog-cli-security/utils" + "github.com/jfrog/jfrog-client-go/utils/errorutils" "github.com/jfrog/jfrog-client-go/utils/io/fileutils" "github.com/jfrog/jfrog-client-go/utils/log" "github.com/urfave/cli" @@ -754,6 +755,15 @@ func getCurationCommand(c *components.Context) (*curation.CurationAuditCommand, SetPipRequirementsFile(c.GetStringFlagValue(flags.RequirementsFile)). SetSolutionFilePath(c.GetStringFlagValue(flags.SolutionPath)) curationAuditCommand.SetDockerImageName(c.GetStringFlagValue(flags.DockerImageName)) + if c.IsFlagSet(flags.HuggingFaceModel) { + hfModel := c.GetStringFlagValue(flags.HuggingFaceModel) + if hfModel == "" { + return nil, errorutils.CheckErrorf( + "--hugging-face-model value cannot be empty; expected '[:]' (revision defaults to 'main'), comma-separated for multiple (e.g. 'mcpotato/42-eicar-street:main,bert-base-uncased')", + ) + } + curationAuditCommand.SetHuggingFaceModel(hfModel) + } curationAuditCommand.SetIncludeCachedPackages(c.GetBoolFlagValue(flags.IncludeCachedPackages)) curationAuditCommand.SetMvnIncludePluginDeps(c.GetBoolFlagValue(flags.MvnIncludePluginDeps)) curationAuditCommand.SetLegacyPeerDeps(c.GetBoolFlagValue(flags.LegacyPeerDeps)) diff --git a/commands/curation/curation_huggingface_test.go b/commands/curation/curation_huggingface_test.go new file mode 100644 index 000000000..a9a2e50e5 --- /dev/null +++ b/commands/curation/curation_huggingface_test.go @@ -0,0 +1,389 @@ +package curation + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "testing" + + coreCommonTests "github.com/jfrog/jfrog-cli-core/v2/common/tests" + "github.com/jfrog/jfrog-cli-core/v2/utils/config" + "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" + "github.com/jfrog/jfrog-client-go/utils/io/fileutils" + clienttestutils "github.com/jfrog/jfrog-client-go/utils/tests" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// curationBlockedHFResponse is the Artifactory 403 body for a blocked HF model. +const curationBlockedHFResponse = `{"errors":[{"status":403,"message":"Package download was blocked by JFrog Packages Curation service due to the following policies violated {blocks-unknown-license, Package has no identified license, Package license is unidentified, Please replace it with an alternate package}"}]}` + +// TestHuggingFaceCurationAuditAutoDiscovery verifies HF model auto-discovery from source. +func TestHuggingFaceCurationAuditAutoDiscovery(t *testing.T) { + cleanUpFlags := setCurationFlagsForTest(t) + defer cleanUpFlags() + + projectDir := t.TempDir() + pyContent := "from transformers import AutoModel\n" + + "model = AutoModel.from_pretrained(\"org/blocked-model\", revision=\"v1.0\")\n" + require.NoError(t, os.WriteFile(filepath.Join(projectDir, "app.py"), []byte(pyContent), 0o644)) + + const ( + hfRepo = "hf-local-repo" + modelProbeURL = "/api/huggingfaceml/" + hfRepo + "/api/models/org/blocked-model/revision/v1.0" + ) + expectedRequest := map[string]bool{modelProbeURL: false} + requestToFail := map[string]bool{modelProbeURL: false} + + mockServer, serverConfig := hfMockServer(t, expectedRequest, requestToFail, hfRepo) + defer mockServer.Close() + + tempHomeDir, cleanUpHome := createHFTestHome(t, serverConfig) + defer cleanUpHome() + callbackHomeDir := clienttestutils.SetEnvWithCallbackAndAssert(t, coreutils.HomeDir, tempHomeDir) + defer callbackHomeDir() + + t.Setenv("HF_ENDPOINT", serverConfig.ArtifactoryUrl+"api/huggingfaceml/"+hfRepo) + + curationCmd := NewCurationAuditCommand() + curationCmd.SetServerDetails(serverConfig) + curationCmd.SetIsCurationCmd(true) + curationCmd.SetInsecureTls(true) + curationCmd.SetWorkingDirs([]string{projectDir}) + curationCmd.OriginPath = projectDir // scan the temp project, not "." + + results := map[string]*CurationReport{} + require.NoError(t, curationCmd.doCurateAudit(results)) + + var blocked []*PackageStatus + for _, report := range results { + for _, pkg := range report.packagesStatus { + if pkg.Action == "blocked" { + blocked = append(blocked, pkg) + } + } + } + require.Len(t, blocked, 1, "expected exactly one blocked HF model") + assert.Equal(t, "org/blocked-model", blocked[0].PackageName) + assert.Equal(t, "v1.0", blocked[0].PackageVersion) + assert.Equal(t, "huggingfaceml", blocked[0].PkgType) + assert.Equal(t, BlockingReasonPolicy, blocked[0].BlockingReason) + for k, v := range expectedRequest { + assert.Truef(t, v, "expected HEAD probe for %s", k) + } +} + +// TestHuggingFaceCurationAuditAutoDiscovery_CleanModelWithWarning is a regression test for +// a bug where a real (non-blocked) audited model, combined with any warning (here: a +// skipped, unparsable notebook), got silently dropped from the results table. packagesStatus +// only ever holds blocked packages, so an all-clean audit also has it empty — identical in +// shape to the "nothing was audited" placeholder. isWarningsOnlyReport must tell the two +// apart using totalNumberOfPackages, which itself must count the real dependency (not +// undercount to 0 by assuming every tech's FlatTree.Nodes includes a root self-entry, which +// Hugging Face's BuildDependencyTree does not add). +func TestHuggingFaceCurationAuditAutoDiscovery_CleanModelWithWarning(t *testing.T) { + cleanUpFlags := setCurationFlagsForTest(t) + defer cleanUpFlags() + + projectDir := t.TempDir() + pyContent := "from transformers import AutoModel\n" + + "model = AutoModel.from_pretrained(\"org/clean-model\")\n" + require.NoError(t, os.WriteFile(filepath.Join(projectDir, "app.py"), []byte(pyContent), 0o644)) + // Unparsable notebook triggers a skipped-file warning alongside the clean model audit. + require.NoError(t, os.WriteFile(filepath.Join(projectDir, "broken.ipynb"), []byte("not json"), 0o644)) + + const ( + hfRepo = "hf-local-repo" + modelProbeURL = "/api/huggingfaceml/" + hfRepo + "/api/models/org/clean-model/revision/main" + ) + expectedRequest := map[string]bool{modelProbeURL: false} + requestToFail := map[string]bool{} // no entry => probe returns 200 OK (not blocked) + + mockServer, serverConfig := hfMockServer(t, expectedRequest, requestToFail, hfRepo) + defer mockServer.Close() + + tempHomeDir, cleanUpHome := createHFTestHome(t, serverConfig) + defer cleanUpHome() + callbackHomeDir := clienttestutils.SetEnvWithCallbackAndAssert(t, coreutils.HomeDir, tempHomeDir) + defer callbackHomeDir() + + t.Setenv("HF_ENDPOINT", serverConfig.ArtifactoryUrl+"api/huggingfaceml/"+hfRepo) + + curationCmd := NewCurationAuditCommand() + curationCmd.SetServerDetails(serverConfig) + curationCmd.SetIsCurationCmd(true) + curationCmd.SetInsecureTls(true) + curationCmd.SetWorkingDirs([]string{projectDir}) + curationCmd.OriginPath = projectDir + + results := map[string]*CurationReport{} + require.NoError(t, curationCmd.doCurateAudit(results)) + + require.Len(t, results, 1, "expected a single report for the audited project, not the warnings-only placeholder") + var report *CurationReport + for _, r := range results { + report = r + } + assert.Empty(t, report.packagesStatus, "the clean model isn't blocked, so packagesStatus stays empty") + assert.NotEmpty(t, report.warnings, "expected the skipped-file warning to be attached") + assert.Equal(t, 1, report.totalNumberOfPackages, "the clean model must still be counted, not undercounted to 0") + assert.False(t, isWarningsOnlyReport(report), "a report with a real audited package must not be treated as warnings-only") + for k, v := range expectedRequest { + assert.Truef(t, v, "expected HEAD probe for %s", k) + } +} + +// TestHuggingFaceCurationAuditAutoDiscovery_MixedResolvedAndUnresolved: an unresolved +// (404) model must not inflate totalNumberOfPackages, and must set hfPartial, not isPartial. +func TestHuggingFaceCurationAuditAutoDiscovery_MixedResolvedAndUnresolved(t *testing.T) { + cleanUpFlags := setCurationFlagsForTest(t) + defer cleanUpFlags() + + projectDir := t.TempDir() + pyContent := "from transformers import AutoModel\n" + + "clean = AutoModel.from_pretrained(\"org/clean-model\")\n" + + "missing = AutoModel.from_pretrained(\"org/missing-model\")\n" + require.NoError(t, os.WriteFile(filepath.Join(projectDir, "app.py"), []byte(pyContent), 0o644)) + + const hfRepo = "hf-local-repo" + missingProbeURL := "/api/huggingfaceml/" + hfRepo + "/api/models/org/missing-model/revision/main" + + mockServer, serverConfig, _ := coreCommonTests.CreateRtRestsMockServer(t, func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodHead && r.RequestURI == missingProbeURL: + w.WriteHeader(http.StatusNotFound) + case r.Method == http.MethodHead: + // clean-model and any other HEAD probe: 200 OK (not blocked). + case r.RequestURI == "/api/system/version": + _, err := w.Write([]byte(`{"version": "7.82.0"}`)) + require.NoError(t, err) + case r.RequestURI == "/api/v1/system/version": + _, err := w.Write([]byte(`{"xray_version": "3.92.0"}`)) + require.NoError(t, err) + case r.RequestURI == "/api/repositories/"+hfRepo: + w.WriteHeader(http.StatusOK) + } + }) + defer mockServer.Close() + + tempHomeDir, cleanUpHome := createHFTestHome(t, serverConfig) + defer cleanUpHome() + callbackHomeDir := clienttestutils.SetEnvWithCallbackAndAssert(t, coreutils.HomeDir, tempHomeDir) + defer callbackHomeDir() + + t.Setenv("HF_ENDPOINT", serverConfig.ArtifactoryUrl+"api/huggingfaceml/"+hfRepo) + + curationCmd := NewCurationAuditCommand() + curationCmd.SetServerDetails(serverConfig) + curationCmd.SetIsCurationCmd(true) + curationCmd.SetInsecureTls(true) + curationCmd.SetWorkingDirs([]string{projectDir}) + curationCmd.OriginPath = projectDir + + results := map[string]*CurationReport{} + require.NoError(t, curationCmd.doCurateAudit(results)) + + require.Len(t, results, 1) + var projectKey string + var report *CurationReport + for k, r := range results { + projectKey, report = k, r + } + // Project key is a dir name, not repo_id:revision — must not pick up a spurious ":main". + assert.Equal(t, filepath.Base(projectDir), projectKey) + assert.Empty(t, report.packagesStatus, "the resolved model isn't blocked, so packagesStatus stays empty") + assert.Equal(t, 1, report.totalNumberOfPackages, "the unresolved model must be excluded from the total, not counted as audited") + assert.True(t, report.hfPartial, "an unresolved model must mark the report partial") + assert.False(t, report.isPartial, "hfPartial must not reuse isPartial (that also triggers the CVS-specific warning and skips waivers)") + require.NotEmpty(t, report.warnings) + assert.Contains(t, strings.Join(report.warnings, "\n"), "org/missing-model") + + summary := convertResultsToSummary(results) + require.Len(t, summary.Scans, 1) + assert.Equal(t, filepath.Base(projectDir), summary.Scans[0].Target) + curated := summary.Scans[0].CuratedPackages + require.NotNil(t, curated) + assert.Equal(t, 1, curated.GetApprovedCount()) + assert.Equal(t, 0, curated.GetBlockedCount()) + assert.True(t, curated.IsPartial) + assert.Equal(t, "hf_unresolved", curated.PartialReason) +} + +// TestHuggingFaceCurationAuditExplicitModel verifies the --hugging-face-model spot-check. +func TestHuggingFaceCurationAuditExplicitModel(t *testing.T) { + cleanUpFlags := setCurationFlagsForTest(t) + defer cleanUpFlags() + + projectDir := t.TempDir() + + const ( + hfRepo = "hf-local-repo" + modelProbeURL = "/api/huggingfaceml/" + hfRepo + "/api/models/org/explicit-model/revision/main" + ) + expectedRequest := map[string]bool{modelProbeURL: false} + requestToFail := map[string]bool{modelProbeURL: false} + + mockServer, serverConfig := hfMockServer(t, expectedRequest, requestToFail, hfRepo) + defer mockServer.Close() + + tempHomeDir, cleanUpHome := createHFTestHome(t, serverConfig) + defer cleanUpHome() + callbackHomeDir := clienttestutils.SetEnvWithCallbackAndAssert(t, coreutils.HomeDir, tempHomeDir) + defer callbackHomeDir() + + t.Setenv("HF_ENDPOINT", serverConfig.ArtifactoryUrl+"api/huggingfaceml/"+hfRepo) + + curationCmd := NewCurationAuditCommand() + curationCmd.SetServerDetails(serverConfig) + curationCmd.SetIsCurationCmd(true) + curationCmd.SetInsecureTls(true) + curationCmd.SetWorkingDirs([]string{projectDir}) + curationCmd.SetHuggingFaceModel("org/explicit-model") + + results := map[string]*CurationReport{} + require.NoError(t, curationCmd.doCurateAudit(results)) + + var blocked []*PackageStatus + for _, report := range results { + for _, pkg := range report.packagesStatus { + if pkg.Action == "blocked" { + blocked = append(blocked, pkg) + } + } + } + require.Len(t, blocked, 1, "expected exactly one blocked HF model") + assert.Equal(t, "org/explicit-model", blocked[0].PackageName) + assert.Equal(t, "main", blocked[0].PackageVersion) + assert.Equal(t, "huggingfaceml", blocked[0].PkgType) + assert.Equal(t, BlockingReasonPolicy, blocked[0].BlockingReason) + for k, v := range expectedRequest { + assert.Truef(t, v, "expected HEAD probe for %s", k) + } +} + +// TestHuggingFaceCurationAuditRun_MultipleRelativeWorkingDirs verifies that each +// entry in --working-dirs is resolved to an absolute path against the original +// root directory, not against the previous entry's cwd. Run() chdir's into each +// working dir in turn; resolving the next (relative) entry inside that loop, +// after the previous chdir, would incorrectly nest it under the prior directory +// (e.g. "b/model" resolving to ".../a/model/b/model") — a nonexistent path here, +// so it also silently drops HF root-node disambiguation for every entry past the +// first when directories share a basename. +func TestHuggingFaceCurationAuditRun_MultipleRelativeWorkingDirs(t *testing.T) { + cleanUpFlags := setCurationFlagsForTest(t) + defer cleanUpFlags() + + rootDir := t.TempDir() + dirA := filepath.Join(rootDir, "a", "model") + dirB := filepath.Join(rootDir, "b", "model") + require.NoError(t, os.MkdirAll(dirA, 0o755)) + require.NoError(t, os.MkdirAll(dirB, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dirA, "app.py"), + []byte("from transformers import AutoModel\nmodel = AutoModel.from_pretrained(\"org/model-a\")\n"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dirB, "app.py"), + []byte("from transformers import AutoModel\nmodel = AutoModel.from_pretrained(\"org/model-b\")\n"), 0o644)) + + const ( + hfRepo = "hf-local-repo" + probeA = "/api/huggingfaceml/" + hfRepo + "/api/models/org/model-a/revision/main" + probeB = "/api/huggingfaceml/" + hfRepo + "/api/models/org/model-b/revision/main" + ) + expectedRequest := map[string]bool{probeA: false, probeB: false} + requestToFail := map[string]bool{} + + mockServer, serverConfig := hfMockServer(t, expectedRequest, requestToFail, hfRepo) + defer mockServer.Close() + + tempHomeDir, cleanUpHome := createHFTestHome(t, serverConfig) + defer cleanUpHome() + callbackHomeDir := clienttestutils.SetEnvWithCallbackAndAssert(t, coreutils.HomeDir, tempHomeDir) + defer callbackHomeDir() + + t.Setenv("HF_ENDPOINT", serverConfig.ArtifactoryUrl+"api/huggingfaceml/"+hfRepo) + t.Chdir(rootDir) + + curationCmd := NewCurationAuditCommand() + curationCmd.SetServerDetails(serverConfig) + curationCmd.SetIsCurationCmd(true) + curationCmd.SetInsecureTls(true) + // Relative to rootDir (the cwd at Run() start) — this is exactly the shape that + // triggers the bug: two relative dirs sharing a basename ("model"). + curationCmd.SetWorkingDirs([]string{filepath.Join("a", "model"), filepath.Join("b", "model")}) + + require.NoError(t, curationCmd.Run()) + + for k, v := range expectedRequest { + assert.Truef(t, v, "expected HEAD probe for %s", k) + } +} + +// hfMockServer mocks Artifactory HEAD/GET probes and the repo-exists check for HF tests. +func hfMockServer(t *testing.T, expectedRequest, requestToFail map[string]bool, hfRepo string) (*httptest.Server, *config.ServerDetails) { + mapLock := sync.Mutex{} + serverMock, serverConfig, _ := coreCommonTests.CreateRtRestsMockServer(t, func(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodHead: + mapLock.Lock() + if _, ok := expectedRequest[r.RequestURI]; ok { + expectedRequest[r.RequestURI] = true + } + mapLock.Unlock() + if _, ok := requestToFail[r.RequestURI]; ok { + w.WriteHeader(http.StatusForbidden) + } + case http.MethodGet: + switch r.RequestURI { + case "/api/system/version": + _, err := w.Write([]byte(`{"version": "7.82.0"}`)) + require.NoError(t, err) + case "/api/v1/system/version": + _, err := w.Write([]byte(`{"xray_version": "3.92.0"}`)) + require.NoError(t, err) + case "/api/repositories/" + hfRepo: + w.WriteHeader(http.StatusOK) + default: + if _, ok := requestToFail[r.RequestURI]; ok { + w.WriteHeader(http.StatusForbidden) + _, err := w.Write([]byte(curationBlockedHFResponse)) + require.NoError(t, err) + } + } + } + }) + return serverMock, serverConfig +} + +// createHFTestHome writes a temp JFrog home with a default server config for the mock. +func createHFTestHome(t *testing.T, serverConfig *config.ServerDetails) (string, func()) { + tempHomeDir, err := fileutils.CreateTempDir() + require.NoError(t, err) + require.NoError(t, os.MkdirAll(filepath.Join(tempHomeDir, ".jfrog"), 0o777)) + + cfgVersion := coreutils.GetCliConfigVersion() + conf := config.ConfigV5{ + Servers: []*config.ServerDetails{ + { + ServerId: "test", + User: "admin", + Password: "password", + Url: serverConfig.ArtifactoryUrl, + ArtifactoryUrl: serverConfig.ArtifactoryUrl, + IsDefault: true, + }, + }, + Version: "v" + strconv.Itoa(cfgVersion), + } + confBytes, err := json.Marshal(conf) + require.NoError(t, err) + confPath := filepath.Join(tempHomeDir, "jfrog-cli.conf.v"+strconv.Itoa(cfgVersion)) + require.NoError(t, os.WriteFile(confPath, confBytes, 0o644)) + + return tempHomeDir, func() { + _ = fileutils.RemoveTempDir(tempHomeDir) + } +} diff --git a/commands/curation/curationaudit.go b/commands/curation/curationaudit.go index 0d9bb4101..26defbdd1 100644 --- a/commands/curation/curationaudit.go +++ b/commands/curation/curationaudit.go @@ -4,10 +4,13 @@ import ( "encoding/json" "errors" "fmt" + "io/fs" "net/http" + "net/url" "os" "path/filepath" "regexp" + "slices" "sort" "strconv" "strings" @@ -39,6 +42,8 @@ import ( "github.com/jfrog/jfrog-cli-security/sca/bom/buildinfo" "github.com/jfrog/jfrog-cli-security/sca/bom/buildinfo/technologies" "github.com/jfrog/jfrog-cli-security/sca/bom/buildinfo/technologies/docker" + "github.com/jfrog/jfrog-cli-security/sca/bom/buildinfo/technologies/huggingface" + hfdiscovery "github.com/jfrog/jfrog-cli-security/sca/bom/buildinfo/technologies/huggingface/discovery" npmtech "github.com/jfrog/jfrog-cli-security/sca/bom/buildinfo/technologies/npm" pnpmtech "github.com/jfrog/jfrog-cli-security/sca/bom/buildinfo/technologies/pnpm" "github.com/jfrog/jfrog-cli-security/sca/bom/buildinfo/technologies/python" @@ -97,6 +102,9 @@ const ( "are blocked by the curation policy. Details of the policy violations are shown in the table below.\n" + "Dependency analysis cannot proceed until these issues are addressed.\n" + "Once you switch to an approved version and re-run the audit, additional results will be available." + + // hfUnresolvedReportKey is used when an HF scan found only dynamic references — no table, just warnings. + hfUnresolvedReportKey = "huggingface (unresolved references)" ) var CurationOutputFormats = []string{string(outFormat.Table), string(outFormat.Json)} @@ -124,7 +132,8 @@ var supportedTech = map[techutils.Technology]func(ca *CurationAuditCommand) (boo techutils.Gem: func(ca *CurationAuditCommand) (bool, error) { return ca.checkSupportByVersionOrEnv(techutils.Gem, MinArtiGradleGemSupport) }, - techutils.Docker: func(ca *CurationAuditCommand) (bool, error) { return true, nil }, + techutils.Docker: func(ca *CurationAuditCommand) (bool, error) { return true, nil }, + techutils.HuggingFaceML: func(ca *CurationAuditCommand) (bool, error) { return true, nil }, techutils.Poetry: func(ca *CurationAuditCommand) (bool, error) { return ca.checkSupportByVersionOrEnv(techutils.Poetry, MinArtiPassThroughSupport) }, @@ -243,15 +252,23 @@ type treeAnalyzer struct { // Stored via atomic.Value so it can be retrieved after the parallel runner finishes // and returned once — avoiding double-printing via errorsQueue.AddError. authErr atomic.Value + // hfExplicitModel is true for --hugging-face-model (vs auto-discovery). + hfExplicitModel bool + // hfUnresolvedMu guards hfUnresolvedNodes, appended concurrently. + hfUnresolvedMu sync.Mutex + hfUnresolvedNodes []string } type CurationAuditCommand struct { - PackageManagerConfig *project.RepositoryConfig - extractPoliciesRegex *regexp.Regexp - workingDirs []string - OriginPath string - parallelRequests int - dockerImageName string + PackageManagerConfig *project.RepositoryConfig + extractPoliciesRegex *regexp.Regexp + workingDirs []string + OriginPath string + parallelRequests int + dockerImageName string + huggingFaceModel string + // hfProjectNameHint is the collision-free HF root-node name for the current working dir. + hfProjectNameHint string includeCachedPackages bool mvnIncludePluginDeps bool // pendingWarnings collects log.Warn messages that must be emitted after the @@ -268,6 +285,32 @@ type CurationReport struct { // was produced via the metadata-API fallback. The partial-report warning // is printed after the spinner stops so it is not swallowed by the spinner. isPartial bool + // hfPartial marks unresolved (404) HF nodes excluded from totalNumberOfPackages. + // Kept separate from isPartial, which also triggers the CVS-specific warning + // and skips the waiver flow. + hfPartial bool + // warnings holds user-facing messages from tree-build (e.g. unresolved HF references). + warnings []string + // huggingFaceReport marks reports produced by the Hugging Face audit path (including + // warning-only unresolved-reference placeholders), for deterministic output ordering. + huggingFaceReport bool +} + +// uniqueReportKey returns key, or a tech-suffixed variant if key is already taken by another +// tech's report — e.g. pip's CVS-fallback and HF auto-discovery can both default to the same +// directory-basename key. This keeps them as two separate tables instead of one clobbering +// (or being fused into) the other. +func uniqueReportKey(results map[string]*CurationReport, key string, tech techutils.Technology) string { + if _, exists := results[key]; !exists { + return key + } + candidate := fmt.Sprintf("%s (%s)", key, tech) + for i := 2; ; i++ { + if _, exists := results[candidate]; !exists { + return candidate + } + candidate = fmt.Sprintf("%s (%s) #%d", key, tech, i) + } } type WaiverResponse struct { @@ -307,6 +350,15 @@ func (ca *CurationAuditCommand) SetDockerImageName(dockerImageName string) *Cura return ca } +func (ca *CurationAuditCommand) HuggingFaceModel() string { + return ca.huggingFaceModel +} + +func (ca *CurationAuditCommand) SetHuggingFaceModel(huggingFaceModel string) *CurationAuditCommand { + ca.huggingFaceModel = huggingFaceModel + return ca +} + func (ca *CurationAuditCommand) SetIncludeCachedPackages(includeCachedPackages bool) *CurationAuditCommand { ca.includeCachedPackages = includeCachedPackages return ca @@ -331,20 +383,29 @@ func (ca *CurationAuditCommand) Run() (err error) { } else { ca.workingDirs = append(ca.workingDirs, rootDir) } - results := map[string]*CurationReport{} - var scanErr error - for _, workDir := range ca.workingDirs { - var absWd string - absWd, err = filepath.Abs(workDir) - if err != nil { + // Ensures dirs sharing a basename still get distinct HF root-node names. + hfProjectNames := huggingface.DisambiguateRootNodeNames(ca.workingDirs) + // Resolved up front, before any chdir below — each iteration chdir's into its + // own absWd, so resolving relative dirs one at a time inside the loop would + // resolve later entries against an earlier entry's cwd instead of rootDir. + absWorkingDirs := make([]string, len(ca.workingDirs)) + for i, workDir := range ca.workingDirs { + if absWorkingDirs[i], err = filepath.Abs(workDir); err != nil { return errorutils.CheckError(err) } + } + results := map[string]*CurationReport{} + var scanErr error + for _, absWd := range absWorkingDirs { log.Info("Running curation audit on project:", absWd) if absWd != rootDir { if err = os.Chdir(absWd); err != nil { return errorutils.CheckError(err) } } + // OriginPath scopes hasPythonFiles and params.WorkingDirectory to this working directory. + ca.OriginPath = absWd + ca.hfProjectNameHint = hfProjectNames[absWd] // If error returned, continue to print results(if any), and return error at the end. if e := ca.doCurateAudit(results); e != nil { scanErr = errors.Join(scanErr, e) @@ -369,11 +430,29 @@ func (ca *CurationAuditCommand) Run() (err error) { } } - for projectPath, packagesStatus := range results { - err = errors.Join(err, printResult(ca.OutputFormat(), projectPath, packagesStatus.packagesStatus)) + // Non-HF tables first, then HF — deterministic order regardless of map iteration. + var nonHFKeys, hfKeys []string + for k, report := range results { + if isHuggingFaceReport(report) { + hfKeys = append(hfKeys, k) + } else { + nonHFKeys = append(nonHFKeys, k) + } + } + sort.Strings(nonHFKeys) + sort.Strings(hfKeys) + projectPaths := slices.Concat(nonHFKeys, hfKeys) + + var allWarnings []string + for _, projectPath := range projectPaths { + packagesStatus := results[projectPath] + if !isWarningsOnlyReport(packagesStatus) { + err = errors.Join(err, printResult(ca.OutputFormat(), projectPath, packagesStatus.packagesStatus)) + } + allWarnings = append(allWarnings, packagesStatus.warnings...) // A partial report comes from the CVS fallback: the dependency tree could - // not be fully resolved. Never offers a waiver when the full tree wasn't built + // not be fully resolved. Never offers a waiver when the full tree wasn't built. if packagesStatus.isPartial { continue } @@ -386,6 +465,9 @@ func (ca *CurationAuditCommand) Run() (err error) { } } } + for _, w := range allWarnings { + log.Warn(w) + } err = errors.Join(err, output.RecordSecurityCommandSummary(output.NewCurationSummary(convertResultsToSummary(results)))) return } @@ -393,11 +475,23 @@ func (ca *CurationAuditCommand) Run() (err error) { func convertResultsToSummary(results map[string]*CurationReport) formats.ResultsSummary { summaryResults := formats.ResultsSummary{} for projectPath, packagesStatus := range results { + // hfPartial reports must still be recorded, not treated as "HF wasn't attempted". + if isWarningsOnlyReport(packagesStatus) && !packagesStatus.hfPartial { + continue + } + var partialReason string + switch { + case packagesStatus.isPartial: + partialReason = "cvs_fallback" + case packagesStatus.hfPartial: + partialReason = "hf_unresolved" + } summaryResults.Scans = append(summaryResults.Scans, formats.ScanSummary{Target: projectPath, CuratedPackages: &formats.CuratedPackages{ - PackageCount: packagesStatus.totalNumberOfPackages, - Blocked: getBlocked(packagesStatus.packagesStatus), - IsPartial: packagesStatus.isPartial, + PackageCount: packagesStatus.totalNumberOfPackages, + Blocked: getBlocked(packagesStatus.packagesStatus), + IsPartial: packagesStatus.isPartial || packagesStatus.hfPartial, + PartialReason: partialReason, }, }) } @@ -529,12 +623,26 @@ func promoteYarnWorkspaceMember(techs []string) []string { } func (ca *CurationAuditCommand) doCurateAudit(results map[string]*CurationReport) error { + if err := validateCurationAuditFlags(ca); err != nil { + return err + } techs := promotePnpmWorkspaceMember(techutils.DetectedTechnologiesListForCurationAudit()) techs = promoteYarnWorkspaceMember(techs) if ca.DockerImageName() != "" { log.Debug(fmt.Sprintf("Docker image name '%s' was provided, running Docker curation audit.", ca.DockerImageName())) techs = []string{techutils.Docker.String()} } + // --hugging-face-model: explicit spot-check — run HF only, skip other package managers. + // Auto-discovery: if HF_ENDPOINT is set and .py/.ipynb files exist, append HF to the tech list. + if ca.HuggingFaceModel() != "" { + log.Debug(fmt.Sprintf("Hugging Face models '%s' were provided explicitly — running HF-only audit.", ca.HuggingFaceModel())) + techs = []string{techutils.HuggingFaceML.String()} + } else if os.Getenv("HF_ENDPOINT") != "" && hasPythonFiles(ca.OriginPath) { + hfTech := techutils.HuggingFaceML.String() + if !slices.Contains(techs, hfTech) { + techs = append(techs, hfTech) + } + } // Resolve npm→yarn when the project was configured with 'jf yarn-config' (yarn.yaml exists) // but has no yarn.lock/.yarnrc.yml so the file-based detector picked npm instead. for i, tech := range techs { @@ -556,6 +664,13 @@ func (ca *CurationAuditCommand) doCurateAudit(results map[string]*CurationReport } if err := ca.auditTree(techutils.Technology(tech), results); err != nil { + if techutils.Technology(tech) == techutils.HuggingFaceML && ca.HuggingFaceModel() == "" { + // HF auto-discovery is additive — config/connectivity failures must not abort other techs. + log.Warn(fmt.Sprintf("Hugging Face curation audit skipped: %v", err)) + ca.setPackageManagerConfig(nil) + ca.AuditParamsInterface = ca.SetDepsRepo("") + continue + } return err } // clear the package manager config to avoid using the same config for the next tech @@ -566,6 +681,72 @@ func (ca *CurationAuditCommand) doCurateAudit(results map[string]*CurationReport return nil } +// isWarningsOnlyReport is true for HF unresolved-reference placeholders that carry +// warnings but no curation table rows (e.g. hfUnresolvedReportKey). +// isWarningsOnlyReport reports whether report represents the placeholder created when +// no packages were resolved/audited at all (e.g. every HF reference was unresolved) — +// not merely a report where nothing happened to be blocked. packagesStatus only holds +// blocked packages (see fetchNodeStatus), so an all-clean audit of N real packages also +// has an empty packagesStatus; totalNumberOfPackages distinguishes that case (N) from +// the true warnings-only placeholder (0, never set). +func isWarningsOnlyReport(report *CurationReport) bool { + return report.totalNumberOfPackages == 0 && len(report.warnings) > 0 +} + +// isHuggingFaceReport reports whether a result belongs to the Hugging Face audit path. +func isHuggingFaceReport(report *CurationReport) bool { + if report.huggingFaceReport { + return true + } + if len(report.packagesStatus) == 0 { + return false + } + for _, ps := range report.packagesStatus { + if ps.PkgType != techutils.HuggingFaceML.String() { + return false + } + } + return true +} + +// hasPythonFiles returns true if dir contains at least one .py or .ipynb file, +// indicating the project may have Hugging Face model references to discover. +func hasPythonFiles(dir string) bool { + if dir == "" { + dir = "." + } + found := false + _ = filepath.WalkDir(dir, func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + log.Debug(fmt.Sprintf("hasPythonFiles: skipping %s: %v", path, walkErr)) + return nil + } + if d.IsDir() { + if hfdiscovery.IsExcludedWalkDir(d.Name()) { + return filepath.SkipDir + } + return nil + } + ext := strings.ToLower(filepath.Ext(path)) + if ext == ".py" || ext == ".ipynb" { + found = true + return fs.SkipAll + } + return nil + }) + return found +} + +func validateCurationAuditFlags(ca *CurationAuditCommand) error { + if ca.DockerImageName() != "" && ca.HuggingFaceModel() != "" { + return errorutils.CheckErrorf( + "--docker-image and --hugging-face-model cannot be used together; run separate curation-audit commands for each", + ) + } + + return nil +} + // resolveNpmYarnTech upgrades npm→yarn when the project has yarn.yaml but no npm.yaml // (the developer ran 'jf yarn-config' but the file-system detector fell back to npm), // or when the project has a yarn indicator file (.yarnrc.yml / yarn.lock / .yarnrc / .yarn) @@ -708,11 +889,31 @@ func (ca *CurationAuditCommand) getBuildInfoParamsByTech() (technologies.BuildIn PipRequirementsFile: ca.PipRequirementsFile(), // Docker params DockerImageName: ca.DockerImageName(), + // Hugging Face params + HuggingFaceModel: ca.HuggingFaceModel(), + WorkingDirectory: ca.OriginPath, + HFProjectName: ca.hfProjectNameHint, // NuGet params SolutionFilePath: ca.SolutionFilePath(), }, err } +// countPackageNodes returns the number of real dependency nodes in flatTreeNodes, +// excluding root self-entries. FlatTree.Nodes includes each root's own self-entry +// alongside real dependencies for most techs (its ID matches a rootNodes entry), +// but Hugging Face's BuildDependencyTree never adds one (a scanned directory isn't +// itself a package) — so root entries are only subtracted when actually present, +// rather than assuming exactly one and undercounting a single-dependency HF project to 0. +func countPackageNodes(rootNodes map[string]struct{}, flatTreeNodes []*xrayUtils.GraphNode) int { + count := len(flatTreeNodes) + for _, node := range flatTreeNodes { + if _, ok := rootNodes[node.Id]; ok { + count-- + } + } + return count +} + func (ca *CurationAuditCommand) auditTree(tech techutils.Technology, results map[string]*CurationReport) error { // --run-native is only meaningful for npm/pnpm (.npmrc-based); reject it early for other techs. if err := validateRunNativeForTech(tech, ca.RunNative()); err != nil { @@ -758,6 +959,17 @@ func (ca *CurationAuditCommand) auditTree(tech techutils.Technology, results map } // Validate the graph isn't empty. if len(depTreeResult.FullDepTrees) == 0 { + // For HF auto-discovery, an empty tree is normal (no HF call sites found). + if tech == techutils.HuggingFaceML { + log.Debug("Hugging Face: no model references discovered in source — skipping HF curation probe") + if len(depTreeResult.Warnings) > 0 { + results[uniqueReportKey(results, hfUnresolvedReportKey, tech)] = &CurationReport{ + warnings: depTreeResult.Warnings, + huggingFaceReport: true, + } + } + return nil + } return errorutils.CheckErrorf("found no dependencies for the audited project using '%v' as the package manager", tech.String()) } rtManager, serverDetails, err := ca.getRtManagerAndAuth(tech) @@ -771,6 +983,12 @@ func (ca *CurationAuditCommand) auditTree(tech techutils.Technology, results map rootNode := depTreeResult.FullDepTrees[0] // Extract project name from the dependency tree _, projectName, projectScope, projectVersion := getUrlNameAndVersionByTech(tech, rootNode, nil, "", "") + if tech == techutils.HuggingFaceML { + // rootNode.Id is a directory/project name, not a "repo_id:revision" model + // reference — getHuggingFaceNameAndVersion defaults a missing revision to + // "main", which would otherwise tack on a spurious ":main" here. + projectName, projectVersion = rootNode.Id, "" + } // If the project name is not set, we use the current working directory name if projectName == "" { workPath, err := os.Getwd() @@ -783,8 +1001,13 @@ func (ca *CurationAuditCommand) auditTree(tech techutils.Technology, results map if projectVersion != "" { fullProjectName += ":" + projectVersion } + rootNodes := map[string]struct{}{} + for _, tree := range depTreeResult.FullDepTrees { + rootNodes[tree.Id] = struct{}{} + } + packageNodeCount := countPackageNodes(rootNodes, depTreeResult.FlatTree.Nodes) if ca.Progress() != nil { - ca.Progress().SetHeadlineMsg(fmt.Sprintf("Fetch curation status for %s graph with %v nodes project name: %s", tech.ToFormal(), len(depTreeResult.FlatTree.Nodes)-1, fullProjectName)) + ca.Progress().SetHeadlineMsg(fmt.Sprintf("Fetch curation status for %s graph with %v nodes project name: %s", tech.ToFormal(), packageNodeCount, fullProjectName)) } if projectScope != "" { projectName = projectScope + "/" + projectName @@ -804,12 +1027,9 @@ func (ca *CurationAuditCommand) auditTree(tech techutils.Technology, results map parallelRequests: ca.parallelRequests, downloadUrls: depTreeResult.DownloadUrls, includeCachedPackages: ca.includeCachedPackages, + hfExplicitModel: tech == techutils.HuggingFaceML && ca.huggingFaceModel != "", } - rootNodes := map[string]struct{}{} - for _, tree := range depTreeResult.FullDepTrees { - rootNodes[tree.Id] = struct{}{} - } // Fetch status for each node from a flatten graph which, has no duplicate nodes. packagesStatusMap := sync.Map{} err = analyzer.fetchNodesStatus(depTreeResult.FlatTree, &packagesStatusMap, rootNodes) @@ -822,10 +1042,22 @@ func (ca *CurationAuditCommand) auditTree(tech techutils.Technology, results map sort.Slice(packagesStatus, func(i, j int) bool { return packagesStatus[i].ParentName < packagesStatus[j].ParentName }) - results[strings.TrimSuffix(fmt.Sprintf("%s:%s", projectName, projectVersion), ":")] = &CurationReport{ - packagesStatus: packagesStatus, - // We subtract 1 because the root node is not a package. - totalNumberOfPackages: len(depTreeResult.FlatTree.Nodes) - 1, + warnings := depTreeResult.Warnings + if len(analyzer.hfUnresolvedNodes) > 0 { + sort.Strings(analyzer.hfUnresolvedNodes) + warnings = append(warnings, fmt.Sprintf( + "Hugging Face: %d model reference(s) could not be resolved against the registry (HTTP 404) and were NOT audited:\n %s\nVerify the repo id/revision are correct and the repo is accessible from this Artifactory instance.", + len(analyzer.hfUnresolvedNodes), strings.Join(analyzer.hfUnresolvedNodes, "\n "))) + // Excluded from the total — unresolved nodes were never actually audited. + packageNodeCount -= len(analyzer.hfUnresolvedNodes) + } + key := strings.TrimSuffix(fmt.Sprintf("%s:%s", projectName, projectVersion), ":") + results[uniqueReportKey(results, key, tech)] = &CurationReport{ + packagesStatus: packagesStatus, + totalNumberOfPackages: packageNodeCount, + hfPartial: len(analyzer.hfUnresolvedNodes) > 0, + warnings: warnings, + huggingFaceReport: tech == techutils.HuggingFaceML, } return err } @@ -1044,6 +1276,23 @@ func (ca *CurationAuditCommand) SetRepo(tech techutils.Technology) error { return nil } + // Hugging Face derives its repo from HF_ENDPOINT, not from a 'jf -config' file. + // Pass in the already-resolved (--server-id-aware) server, rather than letting + // GetHuggingFaceRepositoryConfig reload the CLI default and potentially probe a + // different Artifactory instance than the one this command was invoked against. + if tech == techutils.HuggingFaceML { + serverDetails, err := ca.ServerDetails() + if err != nil { + return err + } + repoConfig, err := huggingface.GetHuggingFaceRepositoryConfig(serverDetails) + if err != nil { + return err + } + ca.setPackageManagerConfig(repoConfig) + return nil + } + // When --run-native is set for npm, read the Artifactory URL and repo name from the // project's .npmrc via native npm config — no jf npm-config/npm.yaml required. if ca.RunNative() && tech == techutils.Npm { @@ -1378,6 +1627,17 @@ func (nc *treeAnalyzer) fetchNodeStatus(node xrayUtils.GraphNode, p *sync.Map) e } return nil } + // HF 404: hard error for an explicit spot-check, warning for auto-discovery. + if resp != nil && resp.StatusCode == http.StatusNotFound && nc.tech == techutils.HuggingFaceML { + if nc.hfExplicitModel { + return errorutils.CheckErrorf("Hugging Face: %s:%s could not be resolved at %s (HTTP 404) — verify the repo id and revision are correct", name, version, packageUrl) + } + log.Debug(fmt.Sprintf("Hugging Face: %s:%s not resolvable at %s (HTTP 404) — recording as unaudited", name, version, packageUrl)) + nc.hfUnresolvedMu.Lock() + nc.hfUnresolvedNodes = append(nc.hfUnresolvedNodes, fmt.Sprintf("%s:%s", name, version)) + nc.hfUnresolvedMu.Unlock() + continue + } if err != nil { if resp != nil && resp.StatusCode >= 400 { return errorutils.CheckErrorf(errorTemplateHeadRequest, packageUrl, name, version, resp.StatusCode, err) @@ -1445,7 +1705,7 @@ func (ca *CurationAuditCommand) runCvsFallback(cvsErr *python.CvsBlockedError, t log.Warn(fmt.Sprintf("curation-blocked resolution fallback: could not determine working directory (%v) — reporting under fallback key", wdErr)) workPath = "unknown-project" } - results[filepath.Base(workPath)] = &CurationReport{ + results[uniqueReportKey(results, filepath.Base(workPath), tech)] = &CurationReport{ packagesStatus: packagesStatus, totalNumberOfPackages: len(packagesStatus), isPartial: true, @@ -1812,6 +2072,9 @@ func getUrlNameAndVersionByTech(tech techutils.Technology, node *xrayUtils.Graph case techutils.Docker: downloadUrls, name, version = getDockerNameAndVersion(node.Id, artiUrl, repo) return + case techutils.HuggingFaceML: + downloadUrls, name, version = getHuggingFaceNameAndVersion(node.Id, artiUrl, repo) + return } return } @@ -2057,6 +2320,43 @@ func getDockerNameAndVersion(id, artiUrl, repo string) (downloadUrls []string, n return } +// getHuggingFaceNameAndVersion extracts the model id and revision from a node id of the +// form "huggingfaceml://:" and builds the model-info probe URL. +// +// The probe targets the model metadata endpoint, which the curation service blocks +// (HEAD returns 403) for a malicious revision — independent of any specific file: +// +// {artiUrl}/api/huggingfaceml/{repo}/api/models/{repo_id}/revision/{revision} +func getHuggingFaceNameAndVersion(id, artiUrl, repo string) (downloadUrls []string, name, version string) { + if id == "" { + return + } + id = strings.TrimPrefix(id, huggingface.HuggingFacePackagePrefix) + + // Shared with ParseModelReference (flag parsing) — see SplitRepoIDAndRevision's + // doc comment for why this must not be guarded on the revision being slash-free + // (e.g. "refs/pr/3", a PR ref). The revision is path-escaped below since it may + // contain '/'. A leading colon (e.g. ":main") yields name == ""; the early-return + // below fires (defensive — ParseModelReference already rejects this at the + // flag-parsing stage). + name, version = huggingface.SplitRepoIDAndRevision(id) + if version == "" { + version = huggingface.DefaultRevision + } + + if name == "" { + return nil, "", "" + } + + if artiUrl != "" && repo != "" { + // version may contain '/' (e.g. "refs/pr/3"); path-escape it so it lands in the + // URL as a single path segment instead of introducing extra, unintended ones. + downloadUrls = []string{fmt.Sprintf("%s/api/huggingfaceml/%s/api/models/%s/revision/%s", + strings.TrimSuffix(artiUrl, "/"), repo, name, url.PathEscape(version))} + } + return +} + func GetCurationOutputFormat(formatFlagVal string) (format outFormat.OutputFormat, err error) { // Default print format is table. format = outFormat.Table diff --git a/commands/curation/curationaudit_test.go b/commands/curation/curationaudit_test.go index 96297df2c..45063d2d2 100644 --- a/commands/curation/curationaudit_test.go +++ b/commands/curation/curationaudit_test.go @@ -1385,6 +1385,279 @@ func Test_getDockerNameAndVersion(t *testing.T) { }) } } +func Test_getHuggingFaceNameAndVersion(t *testing.T) { + tests := []struct { + name string + id string + artiUrl string + repo string + wantDownloadUrls []string + wantName string + wantVersion string + }{ + { + name: "model with explicit sha revision", + id: "huggingfaceml://mcpotato/42-eicar-street:8fb61c4d511e9aaff0ea55396a124aa292830efc", + artiUrl: "https://test.jfrogdev.org/artifactory", + repo: "my-hugging-face-repo", + wantDownloadUrls: []string{"https://test.jfrogdev.org/artifactory/api/huggingfaceml/my-hugging-face-repo/api/models/mcpotato/42-eicar-street/revision/8fb61c4d511e9aaff0ea55396a124aa292830efc"}, + wantName: "mcpotato/42-eicar-street", + wantVersion: "8fb61c4d511e9aaff0ea55396a124aa292830efc", + }, + { + name: "model with branch revision", + id: "huggingfaceml://bert-base-uncased:main", + artiUrl: "https://test.jfrogdev.org/artifactory", + repo: "my-hugging-face-repo", + wantDownloadUrls: []string{"https://test.jfrogdev.org/artifactory/api/huggingfaceml/my-hugging-face-repo/api/models/bert-base-uncased/revision/main"}, + wantName: "bert-base-uncased", + wantVersion: "main", + }, + { + name: "model id with no revision defaults to main", + id: "huggingfaceml://org/model", + artiUrl: "https://test.jfrogdev.org/artifactory", + repo: "my-hugging-face-repo", + wantDownloadUrls: []string{"https://test.jfrogdev.org/artifactory/api/huggingfaceml/my-hugging-face-repo/api/models/org/model/revision/main"}, + wantName: "org/model", + wantVersion: "main", + }, + { + name: "empty artiUrl and repo produce no download URL", + id: "huggingfaceml://org/model:main", + artiUrl: "", + repo: "", + wantDownloadUrls: nil, + wantName: "org/model", + wantVersion: "main", + }, + { + name: "empty id returns empty results", + id: "", + artiUrl: "https://test.jfrogdev.org/artifactory", + repo: "my-hugging-face-repo", + wantDownloadUrls: nil, + wantName: "", + wantVersion: "", + }, + { + name: "trailing slash stripped from artiUrl", + id: "huggingfaceml://org/model:v1.0", + artiUrl: "https://test.jfrogdev.org/artifactory/", + repo: "my-hugging-face-repo", + wantDownloadUrls: []string{"https://test.jfrogdev.org/artifactory/api/huggingfaceml/my-hugging-face-repo/api/models/org/model/revision/v1.0"}, + wantName: "org/model", + wantVersion: "v1.0", + }, + { + name: "trailing colon defaults to main (not empty string)", + id: "huggingfaceml://org/model:", + artiUrl: "https://test.jfrogdev.org/artifactory", + repo: "my-hugging-face-repo", + wantDownloadUrls: []string{"https://test.jfrogdev.org/artifactory/api/huggingfaceml/my-hugging-face-repo/api/models/org/model/revision/main"}, + wantName: "org/model", + wantVersion: "main", + }, + { + name: "leading colon only (no repo id) returns empty", + id: "huggingfaceml://:main", + artiUrl: "https://test.jfrogdev.org/artifactory", + repo: "my-hugging-face-repo", + wantDownloadUrls: nil, + wantName: "", + wantVersion: "", + }, + { + // refs/pr/3 is a valid Hugging Face revision (a PR ref) and contains '/'. + // It must be split off as the revision (not glued onto name) and + // path-escaped in the URL so it lands as a single path segment. + name: "PR ref revision containing slashes is split and path-escaped", + id: "huggingfaceml://org/model:refs/pr/3", + artiUrl: "https://test.jfrogdev.org/artifactory", + repo: "my-hugging-face-repo", + wantDownloadUrls: []string{"https://test.jfrogdev.org/artifactory/api/huggingfaceml/my-hugging-face-repo/api/models/org/model/revision/refs%2Fpr%2F3"}, + wantName: "org/model", + wantVersion: "refs/pr/3", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotDownloadUrls, gotName, gotVersion := getHuggingFaceNameAndVersion(tt.id, tt.artiUrl, tt.repo) + assert.Equal(t, tt.wantDownloadUrls, gotDownloadUrls, "downloadUrls mismatch") + assert.Equal(t, tt.wantName, gotName, "name mismatch") + assert.Equal(t, tt.wantVersion, gotVersion, "version mismatch") + }) + } +} + +func Test_validateCurationAuditFlags_DockerAndHuggingFaceConflict(t *testing.T) { + ca := NewCurationAuditCommand(). + SetDockerImageName("my.registry/image:tag"). + SetHuggingFaceModel("org/model:main") + err := validateCurationAuditFlags(ca) + require.Error(t, err) + assert.Contains(t, err.Error(), "--docker-image and --hugging-face-model cannot be used together") +} + +func Test_validateCurationAuditFlags_DockerOnly(t *testing.T) { + ca := NewCurationAuditCommand().SetDockerImageName("my.registry/image:tag") + assert.NoError(t, validateCurationAuditFlags(ca)) +} + +func Test_hasPythonFiles_ExcludesDist(t *testing.T) { + dir := t.TempDir() + distDir := filepath.Join(dir, "dist") + require.NoError(t, os.MkdirAll(distDir, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(distDir, "only_here.py"), []byte("x = 1\n"), 0644)) + assert.False(t, hasPythonFiles(dir)) +} + +func Test_hasPythonFiles_FindsProjectSource(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "app.py"), []byte("x = 1\n"), 0644)) + assert.True(t, hasPythonFiles(dir)) +} + +func Test_isHuggingFaceReport(t *testing.T) { + hfPkg := &PackageStatus{PkgType: techutils.HuggingFaceML.String()} + tests := []struct { + name string + report *CurationReport + want bool + }{ + { + name: "empty report", + report: &CurationReport{}, + want: false, + }, + { + name: "warnings only without HF marker", + report: &CurationReport{warnings: []string{"unresolved"}}, + want: false, + }, + { + name: "HF warnings-only placeholder", + report: &CurationReport{warnings: []string{"unresolved"}, huggingFaceReport: true}, + want: true, + }, + { + name: "all HF packages", + report: &CurationReport{packagesStatus: []*PackageStatus{hfPkg}}, + want: true, + }, + { + name: "mixed package types", + report: &CurationReport{packagesStatus: []*PackageStatus{hfPkg, {PkgType: techutils.Pip.String()}}}, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, isHuggingFaceReport(tt.report)) + }) + } +} + +func Test_isWarningsOnlyReport(t *testing.T) { + assert.False(t, isWarningsOnlyReport(&CurationReport{})) + assert.True(t, isWarningsOnlyReport(&CurationReport{warnings: []string{"warn"}})) + assert.False(t, isWarningsOnlyReport(&CurationReport{ + packagesStatus: []*PackageStatus{{PkgType: techutils.HuggingFaceML.String()}}, + })) + // packagesStatus only ever holds blocked packages (see fetchNodeStatus), so an + // all-clean audit of a real package also has an empty packagesStatus — same + // shape as the true "nothing was audited" placeholder. totalNumberOfPackages + // must be the signal that disambiguates them, not packagesStatus emptiness. + assert.False(t, isWarningsOnlyReport(&CurationReport{ + totalNumberOfPackages: 1, + warnings: []string{"dynamic revision"}, + })) +} + +func Test_countPackageNodes(t *testing.T) { + rootNodes := func(ids ...string) map[string]struct{} { + m := make(map[string]struct{}, len(ids)) + for _, id := range ids { + m[id] = struct{}{} + } + return m + } + nodes := func(ids ...string) []*xrayUtils.GraphNode { + var n []*xrayUtils.GraphNode + for _, id := range ids { + n = append(n, &xrayUtils.GraphNode{Id: id}) + } + return n + } + + // Single root (e.g. npm/pip): FlatTree.Nodes includes the root's own self-entry + // alongside its dependencies — that self-entry must be excluded from the count. + assert.Equal(t, 2, countPackageNodes( + rootNodes("root"), nodes("root", "dep-a", "dep-b"))) + + // Multiple roots (e.g. multiple --working-dirs): every root self-entry present + // in the flat graph must be excluded, not just the first. + assert.Equal(t, 2, countPackageNodes( + rootNodes("root-a", "root-b"), nodes("root-a", "dep-a", "root-b", "dep-b"))) + + // Hugging Face: BuildDependencyTree never adds a root self-entry, so nothing + // should be subtracted — a single-dependency project must not undercount to 0. + assert.Equal(t, 1, countPackageNodes( + rootNodes("huggingface-project"), nodes("huggingfaceml://org/model:main"))) + + assert.Equal(t, 0, countPackageNodes(rootNodes("root"), nil)) +} + +func Test_convertResultsToSummary_SkipsWarningsOnly(t *testing.T) { + results := map[string]*CurationReport{ + "pip-project": {packagesStatus: []*PackageStatus{{PackageName: "requests"}}, totalNumberOfPackages: 1}, + hfUnresolvedReportKey: {warnings: []string{"unresolved HF ref"}, huggingFaceReport: true}, + } + summary := convertResultsToSummary(results) + require.Len(t, summary.Scans, 1) + assert.Equal(t, "pip-project", summary.Scans[0].Target) +} + +// Test_convertResultsToSummary_RecordsAllUnresolvedHFReport: an all-unresolved hfPartial +// report must still be recorded, not dropped like the true "not attempted" placeholder. +func Test_convertResultsToSummary_RecordsAllUnresolvedHFReport(t *testing.T) { + results := map[string]*CurationReport{ + "hf-project": {totalNumberOfPackages: 0, hfPartial: true, warnings: []string{"2 model reference(s)... HTTP 404"}}, + } + summary := convertResultsToSummary(results) + require.Len(t, summary.Scans, 1) + assert.Equal(t, "hf-project", summary.Scans[0].Target) + assert.Equal(t, 0, summary.Scans[0].CuratedPackages.PackageCount) + assert.True(t, summary.Scans[0].CuratedPackages.IsPartial) + assert.Equal(t, "hf_unresolved", summary.Scans[0].CuratedPackages.PartialReason) +} + +func Test_doCurateAudit_ExplicitHuggingFaceRequiresHFEndpoint(t *testing.T) { + cleanUpFlags := setCurationFlagsForTest(t) + defer cleanUpFlags() + + // Use a mock server so server resolution succeeds; the HF_ENDPOINT error then + // fires naturally from getHuggingFaceRepositoryConfig → repoFromHFEndpoint. + mockServer, serverConfig := hfMockServer(t, map[string]bool{}, map[string]bool{}, "hf-repo") + defer mockServer.Close() + + tempHomeDir, cleanUpHome := createHFTestHome(t, serverConfig) + defer cleanUpHome() + callbackHomeDir := clienttestutils.SetEnvWithCallbackAndAssert(t, coreutils.HomeDir, tempHomeDir) + defer callbackHomeDir() + + t.Setenv("HF_ENDPOINT", "") + ca := NewCurationAuditCommand() + ca.SetServerDetails(serverConfig) + ca.SetIsCurationCmd(true) + ca.SetInsecureTls(true) + ca.SetHuggingFaceModel("org/model:main") + results := map[string]*CurationReport{} + err := ca.doCurateAudit(results) + require.Error(t, err) + assert.Contains(t, err.Error(), "HF_ENDPOINT") +} + func Test_getNugetNameScopeAndVersion(t *testing.T) { tests := []struct { name string @@ -1498,8 +1771,9 @@ func Test_convertResultsToSummary(t *testing.T) { { Target: "project1", CuratedPackages: &formats.CuratedPackages{ - PackageCount: 1, - IsPartial: true, + PackageCount: 1, + IsPartial: true, + PartialReason: "cvs_fallback", Blocked: []formats.BlockedPackages{{ Policy: "p", Condition: "immature", @@ -1838,6 +2112,74 @@ func TestFetchNodesStatusConcurrentMapWrite(t *testing.T) { assert.Equal(t, numNodes, count, "expected all %d packages to be recorded as blocked", numNodes) } +// TestFetchNodeStatus_HFExplicit404ReturnsError verifies an unresolvable explicit +// --hugging-face-model (HTTP 404) is a hard error, not a silently-skipped node. +func TestFetchNodeStatus_HFExplicit404ReturnsError(t *testing.T) { + serverMock, _, rtManager := coreCommonTests.CreateRtRestsMockServer(t, func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodHead { + w.WriteHeader(http.StatusNotFound) + } + }) + defer serverMock.Close() + + rtAuth := rtManager.GetConfig().GetServiceDetails() + root := &xrayUtils.GraphNode{Id: "huggingfaceml://root"} + root.Nodes = append(root.Nodes, &xrayUtils.GraphNode{Id: "huggingfaceml://org/nonexistent-model:main"}) + + analyzer := treeAnalyzer{ + rtManager: rtManager, + extractPoliciesRegex: regexp.MustCompile(extractPoliciesRegexTemplate), + rtAuth: rtAuth, + httpClientDetails: rtAuth.CreateHttpClientDetails(), + url: rtAuth.GetUrl(), + repo: "hf-repo", + tech: techutils.HuggingFaceML, + hfExplicitModel: true, + } + + packagesStatusMap := sync.Map{} + rootNodes := map[string]struct{}{root.Id: {}} + err := analyzer.fetchNodesStatus(root, &packagesStatusMap, rootNodes) + require.Error(t, err) + assert.Contains(t, err.Error(), "org/nonexistent-model") + assert.Contains(t, err.Error(), "404") + assert.Empty(t, analyzer.hfUnresolvedNodes, "explicit mode must error, not silently record as unresolved") +} + +// TestFetchNodeStatus_HFAutoDiscovery404RecordsUnresolvedNotError verifies an +// unresolvable auto-discovered reference (HTTP 404) is recorded as a warning, +// not a hard error. +func TestFetchNodeStatus_HFAutoDiscovery404RecordsUnresolvedNotError(t *testing.T) { + serverMock, _, rtManager := coreCommonTests.CreateRtRestsMockServer(t, func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodHead { + w.WriteHeader(http.StatusNotFound) + } + }) + defer serverMock.Close() + + rtAuth := rtManager.GetConfig().GetServiceDetails() + root := &xrayUtils.GraphNode{Id: "huggingfaceml://root"} + root.Nodes = append(root.Nodes, &xrayUtils.GraphNode{Id: "huggingfaceml://org/nonexistent-model:main"}) + + analyzer := treeAnalyzer{ + rtManager: rtManager, + extractPoliciesRegex: regexp.MustCompile(extractPoliciesRegexTemplate), + rtAuth: rtAuth, + httpClientDetails: rtAuth.CreateHttpClientDetails(), + url: rtAuth.GetUrl(), + repo: "hf-repo", + tech: techutils.HuggingFaceML, + hfExplicitModel: false, + } + + packagesStatusMap := sync.Map{} + rootNodes := map[string]struct{}{root.Id: {}} + err := analyzer.fetchNodesStatus(root, &packagesStatusMap, rootNodes) + require.NoError(t, err) + require.Len(t, analyzer.hfUnresolvedNodes, 1) + assert.Equal(t, "org/nonexistent-model:main", analyzer.hfUnresolvedNodes[0]) +} + // ============================================================================= // Tests for Poetry support added to curationaudit.go. // Covers the new dispatcher case (Pip, Poetry -> getPythonNameVersion) and the @@ -2385,6 +2727,92 @@ func TestRunCvsFallbackGetWdFailurePreservesResults(t *testing.T) { assert.Contains(t, results, "unknown-project", "results key must be the fallback key when Getwd fails") } +// TestRunCvsFallback_KeepsSeparateTableFromExistingReport verifies pip's CVS-fallback report +// gets its own distinct key (a second table) instead of overwriting or fusing into an existing +// report already recorded under the same directory-basename key (e.g. by HF auto-discovery). +func TestRunCvsFallback_KeepsSeparateTableFromExistingReport(t *testing.T) { + const ( + repo = "test-pip-repo" + blockedPkg = "langchain-core" + blockedVer = "1.4.7" + whlRelativePath = "packages/ab/cd/langchain_core-1.4.7-py3-none-any.whl" + ) + blockJSON := `{"errors":[{"status":403,"message":"Package langchain-core:1.4.7 download was blocked by Curation service due to policy 'p'","policy":"p","condition":"immature","explanation":"too new","recommendation":"use older"}]}` + serverMock, serverDetails, _ := coreCommonTests.CreateRtRestsMockServer(t, func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/pypi/"+blockedPkg+"/"+blockedVer+"/json"): + w.WriteHeader(http.StatusOK) + _, _ = fmt.Fprintf(w, `{"urls":[{"packagetype":"bdist_wheel","url":"../../%s"}]}`, whlRelativePath) + case r.Method == http.MethodHead: + w.WriteHeader(http.StatusForbidden) + default: + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(blockJSON)) + } + }) + defer serverMock.Close() + + repoConfig := (&project.RepositoryConfig{}). + SetTargetRepo(repo). + SetServerDetails(serverDetails) + ca := &CurationAuditCommand{ + PackageManagerConfig: repoConfig, + extractPoliciesRegex: regexp.MustCompile(extractPoliciesRegexTemplate), + } + + orig := osGetwd + osGetwd = func() (string, error) { return "/home/user/ml-project", nil } + t.Cleanup(func() { osGetwd = orig }) + const key = "ml-project" + + // Pre-existing report under the same key, as HF auto-discovery would leave it. + results := map[string]*CurationReport{ + key: { + packagesStatus: []*PackageStatus{{PackageName: "org/malicious-model", PackageVersion: "main", PkgType: "huggingfaceml"}}, + warnings: []string{"Hugging Face: 1 model reference(s) could not be resolved"}, + huggingFaceReport: true, + hfPartial: true, + }, + } + + cvsErr := &python.CvsBlockedError{ + Packages: []python.PinnedRequirement{ + {Name: blockedPkg, Version: blockedVer, ParentName: blockedPkg, ParentVersion: blockedVer}, + }, + } + err := ca.runCvsFallback(cvsErr, techutils.Pip, results) + require.NoError(t, err) + + require.Len(t, results, 2, "pip's fallback must land in a new table, leaving the existing HF report untouched") + + hfReport := results[key] + require.NotNil(t, hfReport) + assert.True(t, hfReport.huggingFaceReport) + assert.True(t, hfReport.hfPartial) + assert.Len(t, hfReport.packagesStatus, 1, "the pre-existing HF report must be unmodified") + assert.Len(t, hfReport.warnings, 1) + + pipKey := uniqueReportKey(map[string]*CurationReport{key: hfReport}, key, techutils.Pip) + pipReport := results[pipKey] + require.NotNil(t, pipReport, "pip's report must be recorded under a disambiguated key, not merged into %q", key) + assert.Len(t, pipReport.packagesStatus, 1) + assert.True(t, pipReport.isPartial) + assert.False(t, pipReport.huggingFaceReport, "pip's own report must not carry the HF marker") +} + +// TestUniqueReportKey covers the disambiguation loop, including a 3-way collision where both +// the plain key and its first tech-suffixed candidate are already taken. +func TestUniqueReportKey(t *testing.T) { + results := map[string]*CurationReport{ + "proj": {}, + "proj (pip)": {}, + "proj (pip) #2": {}, + } + assert.Equal(t, "proj", uniqueReportKey(map[string]*CurationReport{}, "proj", techutils.Pip), "no collision — key returned as-is") + assert.Equal(t, "proj (pip)", uniqueReportKey(map[string]*CurationReport{"proj": {}}, "proj", techutils.Pip), "single collision — first suffixed candidate") + assert.Equal(t, "proj (pip) #3", uniqueReportKey(results, "proj", techutils.Pip), "triple collision — must skip to the next free numbered candidate") +} + // TestEffectiveParentVersion covers all branches of the effectiveParentVersion helper. func TestEffectiveParentVersion(t *testing.T) { cases := []struct { diff --git a/sca/bom/buildinfo/buildinfobom.go b/sca/bom/buildinfo/buildinfobom.go index 66fed8596..a3af4af0a 100644 --- a/sca/bom/buildinfo/buildinfobom.go +++ b/sca/bom/buildinfo/buildinfobom.go @@ -32,6 +32,7 @@ import ( "github.com/jfrog/jfrog-cli-security/sca/bom/buildinfo/technologies/docker" "github.com/jfrog/jfrog-cli-security/sca/bom/buildinfo/technologies/gem" _go "github.com/jfrog/jfrog-cli-security/sca/bom/buildinfo/technologies/go" + "github.com/jfrog/jfrog-cli-security/sca/bom/buildinfo/technologies/huggingface" "github.com/jfrog/jfrog-cli-security/sca/bom/buildinfo/technologies/java" "github.com/jfrog/jfrog-cli-security/sca/bom/buildinfo/technologies/npm" "github.com/jfrog/jfrog-cli-security/sca/bom/buildinfo/technologies/nuget" @@ -233,6 +234,8 @@ type DependencyTreeResult struct { FlatTree *xrayUtils.GraphNode FullDepTrees []*xrayUtils.GraphNode DownloadUrls map[string]string + // Warnings holds user-facing messages from tree-build (e.g. unresolved HF references). + Warnings []string } func GetTechDependencyTree(params technologies.BuildInfoBomGeneratorParams, artifactoryServerDetails *config.ServerDetails, tech techutils.Technology) (depTreeResult DependencyTreeResult, err error) { @@ -298,6 +301,8 @@ func GetTechDependencyTree(params technologies.BuildInfoBomGeneratorParams, arti depTreeResult.FullDepTrees, uniqueDepsIds, err = swift.BuildDependencyTree(params) case techutils.Docker: depTreeResult.FullDepTrees, uniqueDepsIds, err = docker.BuildDependencyTree(params) + case techutils.HuggingFaceML: + depTreeResult.FullDepTrees, uniqueDepsIds, depTreeResult.Warnings, err = huggingface.BuildDependencyTree(params) default: err = errorutils.CheckErrorf("%s is currently not supported", string(tech)) } diff --git a/sca/bom/buildinfo/technologies/common.go b/sca/bom/buildinfo/technologies/common.go index cdf6a0a4f..a7743b6be 100644 --- a/sca/bom/buildinfo/technologies/common.go +++ b/sca/bom/buildinfo/technologies/common.go @@ -75,6 +75,18 @@ type BuildInfoBomGeneratorParams struct { MaxTreeDepth string // Docker params DockerImageName string + // Hugging Face params + // HuggingFaceModel is the model/dataset reference to audit, e.g. "mcpotato/42-eicar-street:main". + // Set by the curation command's --hugging-face-model flag. + // When empty, BuildDependencyTree auto-discovers references from WorkingDirectory. + HuggingFaceModel string + // WorkingDirectory is the project root used for Hugging Face auto-discovery. + // Defaults to "." when empty. + WorkingDirectory string + // HFProjectName overrides the HF root-node name (falls back to the working + // directory basename when empty). Set by the curation command to a collision-free + // name when auditing multiple --working-dirs entries that share a basename. + HFProjectName string // NuGet params SolutionFilePath string } diff --git a/sca/bom/buildinfo/technologies/huggingface/discovery/notebook.go b/sca/bom/buildinfo/technologies/huggingface/discovery/notebook.go new file mode 100644 index 000000000..9a01d5189 --- /dev/null +++ b/sca/bom/buildinfo/technologies/huggingface/discovery/notebook.go @@ -0,0 +1,93 @@ +package discovery + +import ( + "encoding/json" + "fmt" + "os" + "strings" +) + +// ipynbNotebook is the minimal structure we need from a .ipynb JSON file. +type ipynbNotebook struct { + Cells []ipynbCell `json:"cells"` +} + +type ipynbCell struct { + CellType string `json:"cell_type"` + // Source is either a []string (list of lines) or a single string, depending + // on the nbformat version; json.RawMessage lets us handle both. + Source json.RawMessage `json:"source"` +} + +// ParseNotebook reads a .ipynb file and returns discovered/unresolved entries. +// Code cells are extracted and fed through the Python scanner. Cells that +// contain notebook magic commands (leading ! or %) are cleaned before parsing. +// root is an optional scan root, forwarded to ParsePythonSource for filesystem-backed +// classification of ambiguous local-output-path literals (see classifyRepoIDLiteral). +func ParseNotebook(path string, root ...string) (discovered []DiscoveredModel, unresolved []UnresolvedSite, err error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, nil, fmt.Errorf("reading notebook %s: %w", path, err) + } + return parseNotebookBytes(data, path, root...) +} + +// parseNotebookBytes is the testable core of ParseNotebook. +func parseNotebookBytes(data []byte, filename string, root ...string) (discovered []DiscoveredModel, unresolved []UnresolvedSite, err error) { + var nb ipynbNotebook + if err = json.Unmarshal(data, &nb); err != nil { + return nil, nil, fmt.Errorf("parsing notebook JSON %s: %w", filename, err) + } + + for cellIdx, cell := range nb.Cells { + if cell.CellType != "code" { + continue + } + src, parseErr := cellSource(cell.Source) + if parseErr != nil { + // Malformed cell — skip rather than abort. + continue + } + if strings.TrimSpace(src) == "" { + continue + } + src = stripMagics(src) + cellFile := fmt.Sprintf("%s#cell-%d", filename, cellIdx) + d, u := ParsePythonSource(src, cellFile, nil, root...) + discovered = append(discovered, d...) + unresolved = append(unresolved, u...) + } + return +} + +// cellSource decodes the source field which can be a JSON string or []string. +func cellSource(raw json.RawMessage) (string, error) { + // Try []string first (most common in nbformat 4). + var lines []string + if err := json.Unmarshal(raw, &lines); err == nil { + return strings.Join(lines, ""), nil + } + // Fall back to a single string. + var s string + if err := json.Unmarshal(raw, &s); err != nil { + return "", err + } + return s, nil +} + +// stripMagics removes lines that start with ! or % (IPython magic commands +// and shell escapes) so the Python parser doesn't choke on them. +// Lines starting with # (comments) are left intact. +func stripMagics(src string) string { + lines := strings.Split(src, "\n") + out := make([]string, 0, len(lines)) + for _, l := range lines { + trimmed := strings.TrimLeft(l, " \t") + if strings.HasPrefix(trimmed, "!") || strings.HasPrefix(trimmed, "%") { + out = append(out, "") // preserve line numbers + } else { + out = append(out, l) + } + } + return strings.Join(out, "\n") +} diff --git a/sca/bom/buildinfo/technologies/huggingface/discovery/notebook_test.go b/sca/bom/buildinfo/technologies/huggingface/discovery/notebook_test.go new file mode 100644 index 000000000..dd2d40db2 --- /dev/null +++ b/sca/bom/buildinfo/technologies/huggingface/discovery/notebook_test.go @@ -0,0 +1,106 @@ +package discovery + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseNotebook_CodeCell(t *testing.T) { + nb := `{ + "cells": [ + { + "cell_type": "markdown", + "source": ["# Heading"] + }, + { + "cell_type": "code", + "source": ["from transformers import AutoModel\n", "model = AutoModel.from_pretrained(\"org/model\", revision=\"v1\")\n"] + } + ] +}` + disc, unres, err := parseNotebookBytes([]byte(nb), "test.ipynb") + require.NoError(t, err) + require.Len(t, disc, 1) + assert.Equal(t, "org/model", disc[0].RepoID) + assert.Equal(t, "v1", disc[0].Revision) + assert.Contains(t, disc[0].Sources[0].File, "test.ipynb#cell-") + assert.Empty(t, unres) +} + +func TestParseNotebook_MagicsStripped(t *testing.T) { + nb := `{ + "cells": [ + { + "cell_type": "code", + "source": ["!pip install transformers\n", "%load_ext autoreload\n", "from_pretrained(\"org/model\")\n"] + } + ] +}` + disc, _, err := parseNotebookBytes([]byte(nb), "nb.ipynb") + require.NoError(t, err) + require.Len(t, disc, 1) + assert.Equal(t, "org/model", disc[0].RepoID) +} + +func TestParseNotebook_MarkdownCellSkipped(t *testing.T) { + nb := `{ + "cells": [ + { + "cell_type": "markdown", + "source": ["snapshot_download(repo_id=\"org/should-not-match\")"] + } + ] +}` + disc, _, err := parseNotebookBytes([]byte(nb), "nb.ipynb") + require.NoError(t, err) + assert.Empty(t, disc) +} + +func TestParseNotebook_SentenceTransformer(t *testing.T) { + nb := `{ + "cells": [ + { + "cell_type": "code", + "source": ["from sentence_transformers import SentenceTransformer\n", "model = SentenceTransformer(\"sentence-transformers/all-MiniLM-L6-v2\")\n"] + } + ] +}` + disc, unres, err := parseNotebookBytes([]byte(nb), "inference.ipynb") + require.NoError(t, err) + require.Len(t, disc, 1) + assert.Equal(t, "sentence-transformers/all-MiniLM-L6-v2", disc[0].RepoID) + assert.Equal(t, DefaultRevision, disc[0].Revision) + assert.Contains(t, disc[0].Sources[0].File, "inference.ipynb#cell-") + assert.Empty(t, unres) +} + +func TestParseNotebook_InvalidJSON(t *testing.T) { + _, _, err := parseNotebookBytes([]byte("not json"), "bad.ipynb") + require.Error(t, err) +} + +func TestStripMagics(t *testing.T) { + src := "!pip install foo\n%load_ext bar\nimport torch\n" + out := stripMagics(src) + lines := splitLines(out) + assert.Equal(t, "", lines[0]) + assert.Equal(t, "", lines[1]) + assert.Equal(t, "import torch", lines[2]) +} + +func splitLines(s string) []string { + var out []string + start := 0 + for i := 0; i < len(s); i++ { + if s[i] == '\n' { + out = append(out, s[start:i]) + start = i + 1 + } + } + if start < len(s) { + out = append(out, s[start:]) + } + return out +} diff --git a/sca/bom/buildinfo/technologies/huggingface/discovery/python.go b/sca/bom/buildinfo/technologies/huggingface/discovery/python.go new file mode 100644 index 000000000..3acadf56e --- /dev/null +++ b/sca/bom/buildinfo/technologies/huggingface/discovery/python.go @@ -0,0 +1,695 @@ +package discovery + +import ( + "os" + "path/filepath" + "regexp" + "sort" + "strings" +) + +// callPattern describes one of the HF call signatures we recognise. +type callPattern struct { + // namePattern is matched against the bare function/method name (suffix after the last dot). + namePattern *regexp.Regexp + // repoIDArgName is the keyword-arg name for repo_id (empty = first positional only). + repoIDArgName string + // keywordOnly suppresses the positional-argument fallback for repoIDArgName. + // Use for calls like pipeline() whose first positional arg is NOT the model id + // (it is the task string) — only the keyword form `model=` is a valid repo id. + keywordOnly bool + // revisionArgName is the keyword-arg name for the revision. + revisionArgName string + // repoTypeArgName is the keyword-arg name for repo_type (empty = fixed). + repoTypeArgName string + // defaultRepoType is used when repoTypeArgName is empty or not present in the call. + defaultRepoType RepoType +} + +// callName extracts the bare function/method name from a raw call token. +// e.g. "AutoModel.from_pretrained" → "from_pretrained", "snapshot_download" → "snapshot_download". +func callName(s string) string { + if idx := strings.LastIndex(s, "."); idx >= 0 { + return s[idx+1:] + } + return s +} + +var knownCalls = []callPattern{ + { + namePattern: regexp.MustCompile(`^from_pretrained$`), + repoIDArgName: "pretrained_model_name_or_path", + revisionArgName: "revision", + defaultRepoType: RepoTypeModel, + }, + { + namePattern: regexp.MustCompile(`^snapshot_download$`), + repoIDArgName: "repo_id", + revisionArgName: "revision", + repoTypeArgName: "repo_type", + defaultRepoType: RepoTypeModel, + }, + { + namePattern: regexp.MustCompile(`^hf_hub_download$`), + repoIDArgName: "repo_id", + revisionArgName: "revision", + repoTypeArgName: "repo_type", + defaultRepoType: RepoTypeModel, + }, + { + namePattern: regexp.MustCompile(`^load_dataset$`), + repoIDArgName: "path", + revisionArgName: "revision", + defaultRepoType: RepoTypeDataset, + }, + { + // sentence-transformers library: SentenceTransformer("model-id") + namePattern: regexp.MustCompile(`^SentenceTransformer$`), + repoIDArgName: "model_name_or_path", + revisionArgName: "revision", + defaultRepoType: RepoTypeModel, + }, + { + // transformers pipeline factory: pipeline("task", model="org/model") + // The first positional arg is the task string — model is always the keyword form. + // keywordOnly prevents positional[0] ("text-generation") being mistaken for a repo id. + namePattern: regexp.MustCompile(`^pipeline$`), + repoIDArgName: "model", + keywordOnly: true, + revisionArgName: "revision", + defaultRepoType: RepoTypeModel, + }, +} + +// argValue represents one resolved argument value. +type argValue struct { + literal string // non-empty when the value is a string literal + isDynamic bool // true when value is non-literal (Name not in const table, f-string, call, etc.) + isFString bool // sub-kind of dynamic: f-string (for a more specific reason string) +} + +// ParsePythonSource scans a Python source string and returns all discovered HF +// references. filename is used for Location.File in the results. +// constTable allows callers (e.g. a multi-file scanner) to pass in pre-populated +// module-level constants; pass nil and it will be built from src itself. +// root is an optional scan root; when provided, it lets an ambiguous single-slash +// literal (e.g. "output/gpt2-finetuned") be checked against the filesystem to +// decide whether it's confidently local. Omit it when no such root is available. +func ParsePythonSource(src, filename string, constTable map[string]string, root ...string) (discovered []DiscoveredModel, unresolved []UnresolvedSite) { + if constTable == nil { + constTable = buildConstTable(src) + } + scanRoot := "" + if len(root) > 0 { + scanRoot = root[0] + } + logicalLines := joinContinuationLines(src) + for _, ll := range logicalLines { + d, u := matchLogicalLine(ll.text, ll.startLine, filename, constTable, scanRoot) + discovered = append(discovered, d...) + unresolved = append(unresolved, u...) + } + return +} + +// ---- constant table ------------------------------------------------------- + +// simpleAssignRe matches: NAME = "literal" or NAME = 'literal' +// Anchored to avoid matching mid-expression assignments. +var simpleAssignRe = regexp.MustCompile(`(?m)^[ \t]*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*["']([^"']+)["'][ \t]*(?:#.*)?$`) + +// anyAssignRe matches any top-level "NAME = " or "NAME = " (e.g. +// "+=") assignment, regardless of RHS shape (literal, call, expression, ...). +// Used to catch reassignment via a non-literal expression or a compound +// assignment too, e.g. MODEL_ID = os.getenv(...) or MODEL_ID += "-v2" +// overriding a checked-in string literal fallback — simpleAssignRe alone would +// miss both and keep trusting the stale literal as a constant. No lookahead in +// RE2, so '==' is excluded by requiring the char after '=' to not be '=' or EOL. +var anyAssignRe = regexp.MustCompile(`(?m)^[ \t]*([A-Za-z_][A-Za-z0-9_]*)\s*(?:\*\*|//|>>|<<|[+\-*/%&|^@])?=(?:[^=]|$)`) + +// chainedAssignTargetRe catches the middle/right target(s) of a chained +// assignment, e.g. "MODEL_ID = FALLBACK_ID = \"...\"" — anyAssignRe only sees +// the line-leading NAME, so FALLBACK_ID's own reassignment here would +// otherwise go undetected. Matches "=" + NAME + "=" anywhere on the line; +// the same '==' guard as anyAssignRe applies to the trailing "=". +// +// Doesn't false-positive on chained comparisons ("a == b == c"): the '=' in +// "== b ==" is immediately preceded by another '=' with no space, and the +// character right after "b ==" is itself '=', which the trailing guard +// rejects. Nor on keyword args ("foo(x=1, y=2)"): there's no bare "= NAME =" +// pattern there — args are separated by ", ", not repeated "=". +var chainedAssignTargetRe = regexp.MustCompile(`(?m)=\s*([A-Za-z_][A-Za-z0-9_]*)\s*=(?:[^=]|$)`) + +// buildConstTable collects module-level single-assignment string constants in +// three passes over src: two to count all top-level assignments to each name +// (anyAssignRe for the line-leading target, chainedAssignTargetRe for any +// chained targets), one to gather literal ones (simpleAssignRe). A name +// assigned more than once anywhere in the module — even via a non-literal, +// compound, or chained expression — is removed (not trusted). +func buildConstTable(src string) map[string]string { + assignCount := map[string]int{} + for _, m := range anyAssignRe.FindAllStringSubmatch(src, -1) { + assignCount[m[1]]++ + } + for _, m := range chainedAssignTargetRe.FindAllStringSubmatch(src, -1) { + assignCount[m[1]]++ + } + + table := map[string]string{} + seen := map[string]bool{} + for _, m := range simpleAssignRe.FindAllStringSubmatch(src, -1) { + name, val := m[1], m[2] + if seen[name] || assignCount[name] > 1 { + delete(table, name) // reassigned (literal or not) → not constant + seen[name] = true + continue + } + seen[name] = true + table[name] = val + } + return table +} + +// ---- logical line joining -------------------------------------------------- + +type logicalLine struct { + text string + startLine int // 1-based line number of the first physical line +} + +// joinContinuationLines collapses backslash continuations and open-paren spans +// into single logical lines, preserving the starting line number of each. +func joinContinuationLines(src string) []logicalLine { + physical := strings.Split(src, "\n") + var result []logicalLine + var buf strings.Builder + startLine := 0 + depth := 0 + + flush := func() { + if buf.Len() > 0 { + result = append(result, logicalLine{text: buf.String(), startLine: startLine + 1}) + buf.Reset() + } + } + + for i, line := range physical { + trimmed := strings.TrimRight(line, " \t\r") + if buf.Len() == 0 { + startLine = i + } + // Count unquoted parens (rough but good enough for our call patterns) + depth += countParenDepthChange(trimmed) + + if strings.HasSuffix(trimmed, "\\") { + buf.WriteString(strings.TrimSuffix(trimmed, "\\")) + continue + } + buf.WriteString(trimmed) + if depth <= 0 { + depth = 0 + flush() + } else { + buf.WriteString(" ") + } + } + flush() + return result +} + +// countParenDepthChange returns the net change in paren depth ignoring quoted strings. +// An unquoted '#' starts a Python comment, so the rest of the line is prose (which may +// contain unbalanced parens or apostrophes like "tab's") and must be ignored — otherwise +// a comment can corrupt depth tracking and glue unrelated statements together. +func countParenDepthChange(s string) int { + depth := 0 + inSingle, inDouble := false, false + for i := 0; i < len(s); i++ { + c := s[i] + if c == '\\' && (inSingle || inDouble) { + i++ // skip escaped char + continue + } + switch { + case c == '#' && !inSingle && !inDouble: + return depth // rest of the line is a comment + case c == '\'' && !inDouble: + inSingle = !inSingle + case c == '"' && !inSingle: + inDouble = !inDouble + case c == '(' && !inSingle && !inDouble: + depth++ + case c == ')' && !inSingle && !inDouble: + depth-- + } + } + return depth +} + +// ---- call matching -------------------------------------------------------- + +// callNameRe matches a function/method name (with optional dotted prefix) immediately +// followed by '('. Used by findCallSpans to locate call sites; the argument string is +// then extracted with depth tracking so nested parens are handled correctly. +var callNameRe = regexp.MustCompile(`((?:[A-Za-z_][A-Za-z0-9_.]*\.)?[A-Za-z_][A-Za-z0-9_]*)\s*\(`) + +// callSpan is one call site found by findCallSpans. +type callSpan struct { + name string // bare or dotted function name + argsRaw string // raw argument string between the outermost parens +} + +// findCallSpans scans line for function calls and returns each call's name and its +// full argument string. Unlike a simple [^)]* regex, it uses depth tracking so calls +// with nested parentheses in their arguments are captured correctly. +func findCallSpans(line string) []callSpan { + var spans []callSpan + for _, loc := range callNameRe.FindAllStringSubmatchIndex(line, -1) { + // loc: [matchStart, matchEnd, group1Start, group1End] + name := line[loc[2]:loc[3]] + openParen := loc[1] - 1 // index of '(' in line + // Walk forward with depth tracking to find the matching ')'. + depth := 0 + inSingle, inDouble := false, false + argsStart := openParen + 1 + end := -1 + for i := openParen; i < len(line); i++ { + c := line[i] + if c == '\\' && (inSingle || inDouble) { + i++ + continue + } + switch { + case c == '\'' && !inDouble: + inSingle = !inSingle + case c == '"' && !inSingle: + inDouble = !inDouble + case c == '(' && !inSingle && !inDouble: + depth++ + case c == ')' && !inSingle && !inDouble: + depth-- + if depth == 0 { + end = i + } + } + if end >= 0 { + break + } + } + if end < 0 { + continue // unmatched paren — skip + } + spans = append(spans, callSpan{name: name, argsRaw: line[argsStart:end]}) + } + return spans +} + +func matchLogicalLine(line string, startLine int, filename string, constTable map[string]string, root string) (discovered []DiscoveredModel, unresolved []UnresolvedSite) { + // Strip leading whitespace and inline comments for cleaner matching. + trimmed := strings.TrimLeft(line, " \t") + if strings.HasPrefix(trimmed, "#") { + return + } + + for _, span := range findCallSpans(line) { + fullName := span.name + argsRaw := span.argsRaw + name := callName(fullName) + + for _, cp := range knownCalls { + if !cp.namePattern.MatchString(name) { + continue + } + loc := Location{File: filename, Line: startLine} + d, u := resolveCall(cp, argsRaw, loc, constTable, root) + discovered = append(discovered, d...) + unresolved = append(unresolved, u...) + } + } + return +} + +// ---- argument resolution -------------------------------------------------- + +// resolveCall parses the raw argument string for a matched call and produces +// DiscoveredModel or UnresolvedSite entries. root is the scan root, used to check +// the filesystem when a repo_id literal is ambiguous between a local output +// directory and a syntactically valid Hub id (see classifyRepoIDLiteral). +func resolveCall(cp callPattern, argsRaw string, loc Location, constTable map[string]string, root string) (discovered []DiscoveredModel, unresolved []UnresolvedSite) { + positional, keyword := parseArgs(argsRaw) + + // Resolve repo_id + var repoIDArg argValue + repoIDFound := false + if cp.repoIDArgName != "" { + if v, ok := keyword[cp.repoIDArgName]; ok { + repoIDArg = resolveArgValue(v, constTable) + repoIDFound = true + } else if !cp.keywordOnly && len(positional) > 0 { + // keywordOnly=true (e.g. pipeline) suppresses this fallback: positional[0] + // is the task string ("text-generation"), not a model id. + repoIDArg = resolveArgValue(positional[0], constTable) + repoIDFound = true + } + } else if len(positional) > 0 { + repoIDArg = resolveArgValue(positional[0], constTable) + repoIDFound = true + } + + if !repoIDFound { + // No repo_id argument found at all — silently skip. + // Example: pipeline("text-generation") with no model= kwarg uses a + // task-default model that cannot be determined statically; emitting an + // unresolved warning here would be noisy and unhelpful. + return + } + + if repoIDArg.literal == "" { + // Arg is present but non-literal (dynamic variable, f-string, call, etc.) + // — record as unresolved so the user knows coverage is incomplete. + reason := "non-literal repo_id" + if repoIDArg.isFString { + reason = "f-string repo_id" + } + snippet := buildSnippet(argsRaw) + unresolved = append(unresolved, UnresolvedSite{Location: loc, Snippet: snippet, Reason: reason}) + return + } + + // A local filesystem path is a valid literal but not a Hub repo id: the model + // was resolved outside source (e.g. downloaded via `jf hf download` in a prior + // build step). Report it as an advisory instead of probing curation with a + // bogus repo id, so the coverage gap is visible. + switch classifyRepoIDLiteral(repoIDArg.literal, root) { + case classLocalPath: + snippet := buildSnippet(argsRaw) + unresolved = append(unresolved, UnresolvedSite{Location: loc, Snippet: snippet, Reason: "local filesystem path"}) + return + case classAmbiguousPath: + // Structurally a valid Hub id (HF's namespace/repo_name grammar permits names + // like "output/model"), but it also matches a conventional local-output-dir + // name and no filesystem evidence could confirm or rule out a local directory. + // Don't guess either way: silently auditing it could probe a bogus repo id, + // and silently skipping it could hide a real, uncurated Hub reference. + snippet := buildSnippet(argsRaw) + unresolved = append(unresolved, UnresolvedSite{Location: loc, Snippet: snippet, Reason: "ambiguous local path or Hub repo id"}) + return + } + + // Resolve revision + var revision string + revDefaulted := false + revDynamic := false + if v, ok := keyword[cp.revisionArgName]; ok { + rv := resolveArgValue(v, constTable) + if rv.literal != "" { + revision = rv.literal + } else { + revDynamic = true + revision = DefaultRevision + } + } else { + revision = DefaultRevision + revDefaulted = true + } + + // Resolve repo_type + repoType := cp.defaultRepoType + if cp.repoTypeArgName != "" { + if v, ok := keyword[cp.repoTypeArgName]; ok { + rv := resolveArgValue(v, constTable) + switch rv.literal { + case "dataset": + repoType = RepoTypeDataset + case "model": + repoType = RepoTypeModel + default: + // Dynamic or unsupported repo_type (e.g. "space") — don't guess. + reason := "non-literal repo_type" + switch { + case rv.isFString: + reason = "f-string repo_type" + case rv.literal != "": + reason = "unsupported repo_type" + } + snippet := buildSnippet(argsRaw) + unresolved = append(unresolved, UnresolvedSite{Location: loc, Snippet: snippet, Reason: reason}) + return + } + } + } + + discovered = append(discovered, DiscoveredModel{ + RepoID: repoIDArg.literal, + Revision: revision, + RevisionDefaulted: revDefaulted, + RevisionDynamic: revDynamic, + RepoType: repoType, + Sources: []Location{loc}, + }) + return +} + +// parseArgs splits a raw argument string into positional and keyword slices. +// It handles simple cases well; deeply nested calls may produce imperfect +// splits, but those will fail the literal-string check and become UnresolvedSite. +func parseArgs(raw string) (positional []string, keyword map[string]string) { + keyword = map[string]string{} + if strings.TrimSpace(raw) == "" { + return + } + parts := splitArgs(raw) + for _, part := range parts { + part = strings.TrimSpace(part) + if part == "" { + continue + } + // keyword arg: name = value + if idx := strings.Index(part, "="); idx > 0 { + k := strings.TrimSpace(part[:idx]) + v := strings.TrimSpace(part[idx+1:]) + // Only treat as keyword if k is a valid identifier + if isIdentifier(k) { + keyword[k] = v + continue + } + } + positional = append(positional, part) + } + return +} + +type quoteState int + +const ( + quoteNone quoteState = iota + quoteSingle + quoteDouble + quoteTripleSingle + quoteTripleDouble +) + +// splitArgs splits a comma-separated argument list respecting quoted strings +// and nested parentheses. +func splitArgs(s string) []string { + var parts []string + depth := 0 + quote := quoteNone + start := 0 + for i := 0; i < len(s); i++ { + if quote == quoteTripleDouble { + if i+2 < len(s) && s[i:i+3] == `"""` { + quote = quoteNone + i += 2 + } + continue + } + if quote == quoteTripleSingle { + if i+2 < len(s) && s[i:i+3] == "'''" { + quote = quoteNone + i += 2 + } + continue + } + + c := s[i] + if c == '\\' && (quote == quoteSingle || quote == quoteDouble) { + i++ + continue + } + switch { + case quote == quoteNone && i+2 < len(s) && s[i:i+3] == `"""`: + quote = quoteTripleDouble + i += 2 + case quote == quoteNone && i+2 < len(s) && s[i:i+3] == "'''": + quote = quoteTripleSingle + i += 2 + case c == '\'' && quote == quoteNone: + quote = quoteSingle + case c == '"' && quote == quoteNone: + quote = quoteDouble + case c == '\'' && quote == quoteSingle: + quote = quoteNone + case c == '"' && quote == quoteDouble: + quote = quoteNone + case (c == '(' || c == '[' || c == '{') && quote == quoteNone: + depth++ + case (c == ')' || c == ']' || c == '}') && quote == quoteNone: + depth-- + case c == ',' && depth == 0 && quote == quoteNone: + parts = append(parts, s[start:i]) + start = i + 1 + } + } + parts = append(parts, s[start:]) + return parts +} + +// resolveArgValue classifies a raw argument token. +func resolveArgValue(raw string, constTable map[string]string) argValue { + s := strings.TrimSpace(raw) + + if lit, ok := unquotePythonString(s); ok { + return argValue{literal: lit} + } + + // f-string → dynamic + if strings.HasPrefix(s, `f"`) || strings.HasPrefix(s, `f'`) || + strings.HasPrefix(s, `F"`) || strings.HasPrefix(s, `F'`) { + return argValue{isDynamic: true, isFString: true} + } + + // Simple identifier → check constant table + if isIdentifier(s) { + if v, ok := constTable[s]; ok { + return argValue{literal: v} + } + return argValue{isDynamic: true} + } + + return argValue{isDynamic: true} +} + +// unquotePythonString extracts the value from a Python string literal token, +// including triple-quoted """...""" and ”'...”' forms. +func unquotePythonString(s string) (string, bool) { + if strings.HasPrefix(s, `"""`) && strings.HasSuffix(s, `"""`) && len(s) >= 6 { + return s[3 : len(s)-3], true + } + if strings.HasPrefix(s, "'''") && strings.HasSuffix(s, "'''") && len(s) >= 6 { + return s[3 : len(s)-3], true + } + if len(s) >= 2 && s[0] == '"' && s[len(s)-1] == '"' { + return s[1 : len(s)-1], true + } + if len(s) >= 2 && s[0] == '\'' && s[len(s)-1] == '\'' { + return s[1 : len(s)-1], true + } + return "", false +} + +// isIdentifier returns true if s is a valid Python/Go identifier token. +var identRe = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) + +func isIdentifier(s string) bool { + return identRe.MatchString(s) +} + +// winDriveRe matches a Windows drive-letter prefix such as "C:\" or "C:/". +var winDriveRe = regexp.MustCompile(`^[A-Za-z]:[\\/]`) + +// pathClass is the outcome of classifyRepoIDLiteral. +type pathClass int + +const ( + // classHubID means the literal is treated as a Hugging Face Hub repo id. + classHubID pathClass = iota + // classLocalPath means the literal is confidently a local filesystem path. + classLocalPath + // classAmbiguousPath means the literal is a structurally valid Hub id that also + // matches a conventional local-output-directory name, with no filesystem + // evidence to disambiguate. + classAmbiguousPath +) + +// localOutputPrefixes are first-segment names that look like Hub org/model IDs but +// are conventionally local output/checkpoint dirs in ML training scripts (e.g. +// "output/gpt2-finetuned"). A name match alone is a hint, not proof — see +// classifyRepoIDLiteral. +var localOutputPrefixes = []string{ + "output/", + "outputs/", + "runs/", + "checkpoints/", + "checkpoint/", + "saved_models/", + "saved_model/", + "artifacts/", + "results/", + "finetuned/", + "trained/", +} + +// classifyRepoIDLiteral classifies a resolved string literal as a Hub repo id, a +// confidently local filesystem path, or an ambiguous case needing user attention. +// Hub ids are a bare name or "namespace/name"; a leading path marker, backslash, +// drive letter, or more than one slash is unambiguously local. +// +// A single-slash value is structurally indistinguishable from a Hub "namespace/name" +// id. Filesystem evidence (when root is available) takes priority over the +// localOutputPrefixes heuristic; otherwise a prefix match is classAmbiguousPath. +func classifyRepoIDLiteral(s, root string) pathClass { + if s == "" { + return classHubID + } + switch { + case strings.HasPrefix(s, "/"), strings.HasPrefix(s, "./"), + strings.HasPrefix(s, "../"), strings.HasPrefix(s, "~"): + return classLocalPath + case strings.Contains(s, "\\"): + return classLocalPath + case winDriveRe.MatchString(s): + return classLocalPath + case strings.Count(s, "/") > 1: + return classLocalPath + } + if root != "" { + if info, err := os.Stat(filepath.Join(root, s)); err == nil && info.IsDir() { + return classLocalPath + } + } + for _, prefix := range localOutputPrefixes { + if strings.HasPrefix(s, prefix) { + return classAmbiguousPath + } + } + return classHubID +} + +// buildSnippet renders a call's arguments for the unresolved-reference warning, +// redacting anything that could contain a secret. Bare expressions (variable, +// attribute, call) are shown as-is to help identify the call site; anything +// containing a quote character may embed a string literal and is redacted. +func buildSnippet(argsRaw string) string { + positional, keyword := parseArgs(argsRaw) + redact := func(raw string) string { + if strings.ContainsAny(raw, `'"`) { + return "" + } + return strings.TrimSpace(raw) + } + + parts := make([]string, 0, len(positional)+len(keyword)) + for _, p := range positional { + parts = append(parts, redact(p)) + } + names := make([]string, 0, len(keyword)) + for k := range keyword { + names = append(names, k) + } + sort.Strings(names) + for _, k := range names { + parts = append(parts, k+"="+redact(keyword[k])) + } + return strings.Join(parts, ", ") +} diff --git a/sca/bom/buildinfo/technologies/huggingface/discovery/python_test.go b/sca/bom/buildinfo/technologies/huggingface/discovery/python_test.go new file mode 100644 index 000000000..39c4e7305 --- /dev/null +++ b/sca/bom/buildinfo/technologies/huggingface/discovery/python_test.go @@ -0,0 +1,521 @@ +package discovery + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParsePythonSource_Literals(t *testing.T) { + src := ` +from transformers import AutoModel +model = AutoModel.from_pretrained("mcpotato/42-eicar-street", revision="main") +` + disc, unres := ParsePythonSource(src, "test.py", nil) + require.Len(t, disc, 1) + assert.Equal(t, "mcpotato/42-eicar-street", disc[0].RepoID) + assert.Equal(t, "main", disc[0].Revision) + assert.False(t, disc[0].RevisionDefaulted) + assert.Equal(t, RepoTypeModel, disc[0].RepoType) + assert.Empty(t, unres) +} + +func TestParsePythonSource_RevisionDefaulted(t *testing.T) { + src := `snapshot_download(repo_id="bert-base-uncased")` + disc, unres := ParsePythonSource(src, "test.py", nil) + require.Len(t, disc, 1) + assert.Equal(t, "bert-base-uncased", disc[0].RepoID) + assert.Equal(t, DefaultRevision, disc[0].Revision) + assert.True(t, disc[0].RevisionDefaulted) + assert.Empty(t, unres) +} + +func TestParsePythonSource_Dataset(t *testing.T) { + src := `from datasets import load_dataset +ds = load_dataset("squad", revision="1.0.0")` + disc, unres := ParsePythonSource(src, "train.py", nil) + require.Len(t, disc, 1) + assert.Equal(t, "squad", disc[0].RepoID) + assert.Equal(t, "1.0.0", disc[0].Revision) + assert.Equal(t, RepoTypeDataset, disc[0].RepoType) + assert.Empty(t, unres) +} + +func TestParsePythonSource_LoadDatasetPathKwarg(t *testing.T) { + src := `load_dataset(path="squad", revision="1.0.0")` + disc, unres := ParsePythonSource(src, "train.py", nil) + require.Len(t, disc, 1) + assert.Equal(t, "squad", disc[0].RepoID) + assert.Equal(t, "1.0.0", disc[0].Revision) + assert.Equal(t, RepoTypeDataset, disc[0].RepoType) + assert.Empty(t, unres) +} + +func TestParsePythonSource_TripleQuotedString(t *testing.T) { + src := `load_dataset("""squad""", revision="1.0.0")` + disc, unres := ParsePythonSource(src, "train.py", nil) + require.Len(t, disc, 1) + assert.Equal(t, "squad", disc[0].RepoID) + assert.Equal(t, "1.0.0", disc[0].Revision) + assert.Empty(t, unres) +} + +func Test_unquotePythonString(t *testing.T) { + tests := []struct { + in string + want string + ok bool + }{ + {`"squad"`, "squad", true}, + {`"""squad"""`, "squad", true}, + {`'''squad'''`, "squad", true}, + {"args.model", "", false}, + } + for _, tt := range tests { + got, ok := unquotePythonString(tt.in) + assert.Equal(t, tt.ok, ok, "input %q", tt.in) + if ok { + assert.Equal(t, tt.want, got, "input %q", tt.in) + } + } +} + +func TestParsePythonSource_SnapshotDownloadWithRepoType(t *testing.T) { + src := `snapshot_download(repo_id="org/ds", revision="v2", repo_type="dataset")` + disc, unres := ParsePythonSource(src, "test.py", nil) + require.Len(t, disc, 1) + assert.Equal(t, RepoTypeDataset, disc[0].RepoType) + assert.Equal(t, "v2", disc[0].Revision) + assert.Empty(t, unres) +} + +func TestParsePythonSource_SnapshotDownloadWithExplicitModelRepoType(t *testing.T) { + src := `snapshot_download(repo_id="org/model", repo_type="model")` + disc, unres := ParsePythonSource(src, "test.py", nil) + require.Len(t, disc, 1) + assert.Equal(t, RepoTypeModel, disc[0].RepoType) + assert.Empty(t, unres) +} + +// TestParsePythonSource_DynamicRepoTypeIsUnresolved: a runtime repo_type value +// must not be silently emitted under the default (model) type. +func TestParsePythonSource_DynamicRepoTypeIsUnresolved(t *testing.T) { + discovered, unresolved := ParsePythonSource( + `snapshot_download(repo_id="org/data", repo_type=args.repo_type)`, + "app.py", nil) + assert.Empty(t, discovered) + require.Len(t, unresolved, 1) + assert.Equal(t, "non-literal repo_type", unresolved[0].Reason) +} + +// TestParsePythonSource_UnsupportedRepoTypeIsUnresolved: "space" is a real HF +// repo_type this scanner doesn't model, so it must be flagged as unresolved. +func TestParsePythonSource_UnsupportedRepoTypeIsUnresolved(t *testing.T) { + src := `snapshot_download(repo_id="org/my-space", repo_type="space")` + disc, unres := ParsePythonSource(src, "test.py", nil) + assert.Empty(t, disc) + require.Len(t, unres, 1) + assert.Equal(t, "unsupported repo_type", unres[0].Reason) +} + +// TestParsePythonSource_ApostropheInComment guards against a regression where an +// apostrophe inside a '#' comment (e.g. "tab's") was treated as an open string +// literal in countParenDepthChange, leaving paren depth stuck and gluing every +// following statement onto the comment line — which silently dropped real calls. +func TestParsePythonSource_ApostropheInComment(t *testing.T) { + src := `from huggingface_hub import snapshot_download + +# Whole-repo download (matches the "Resolve" tab's snapshot_download example). +LLAMA = snapshot_download(repo_id="meta-llama/Llama-2-7b-hf", revision="main") + +# A reference we EXPECT curation to block (it's malicious). +UNSAFE = snapshot_download(repo_id="mcpotato/42-eicar-street", revision="8fb61c4d511e9aaff0ea55396a124aa292830efc") +` + disc, unres := ParsePythonSource(src, "app.py", nil) + require.Len(t, disc, 2) + assert.Equal(t, "meta-llama/Llama-2-7b-hf", disc[0].RepoID) + assert.Equal(t, "main", disc[0].Revision) + assert.Equal(t, "mcpotato/42-eicar-street", disc[1].RepoID) + assert.Equal(t, "8fb61c4d511e9aaff0ea55396a124aa292830efc", disc[1].Revision) + assert.Empty(t, unres) +} + +func TestParsePythonSource_ConstantTable(t *testing.T) { + src := ` +MODEL_ID = "org/my-model" +from transformers import AutoTokenizer +tok = AutoTokenizer.from_pretrained(MODEL_ID, revision="abc123") +` + disc, unres := ParsePythonSource(src, "test.py", nil) + require.Len(t, disc, 1) + assert.Equal(t, "org/my-model", disc[0].RepoID) + assert.Equal(t, "abc123", disc[0].Revision) + assert.Empty(t, unres) +} + +func TestParsePythonSource_DynamicRepoID(t *testing.T) { + src := `from transformers import AutoModel +model = AutoModel.from_pretrained(args.model_name)` + disc, unres := ParsePythonSource(src, "trainer.py", nil) + assert.Empty(t, disc) + require.Len(t, unres, 1) + assert.Contains(t, unres[0].Reason, "non-literal") + assert.Equal(t, "trainer.py", unres[0].Location.File) +} + +// TestParsePythonSource_SnippetRedactsArgumentValues verifies that the warning +// snippet for an unresolved reference keeps bare variable references (useful for +// identifying the call site) but never contains a quoted literal, since it is +// printed to CI logs and a literal may embed a real secret. +func TestParsePythonSource_SnippetRedactsArgumentValues(t *testing.T) { + src := `from_pretrained(args.model_name, token="hf_abcdef123456")` + disc, unres := ParsePythonSource(src, "trainer.py", nil) + assert.Empty(t, disc) + require.Len(t, unres, 1) + assert.NotContains(t, unres[0].Snippet, "hf_abcdef123456") + assert.Equal(t, "args.model_name, token=", unres[0].Snippet) +} + +// TestParsePythonSource_SnippetRedactsNestedLiteral verifies that a literal nested +// inside a call expression (not just a direct keyword value) is still redacted. +func TestParsePythonSource_SnippetRedactsNestedLiteral(t *testing.T) { + src := `from_pretrained(args.model_name, token=get_token("hf_abcdef123456"))` + disc, unres := ParsePythonSource(src, "trainer.py", nil) + assert.Empty(t, disc) + require.Len(t, unres, 1) + assert.NotContains(t, unres[0].Snippet, "hf_abcdef123456") + assert.Equal(t, "args.model_name, token=", unres[0].Snippet) +} + +func TestParsePythonSource_FStringRepoID(t *testing.T) { + src := `from_pretrained(f"{ORG}/{name}")` + disc, unres := ParsePythonSource(src, "test.py", nil) + assert.Empty(t, disc) + require.Len(t, unres, 1) + assert.Equal(t, "f-string repo_id", unres[0].Reason) +} + +func TestParsePythonSource_LocalPathAdvisory(t *testing.T) { + cases := []struct { + name string + arg string + }{ + {"absolute", `"/opt/models/gpt2"`}, + {"relative-dot", `"./models/gpt2"`}, + {"relative-dotdot", `"../models/gpt2"`}, + {"home", `"~/models/gpt2"`}, + {"windows", `"C:\\models\\gpt2"`}, + {"multi-slash", `"models/foo/gpt2"`}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + src := "from transformers import AutoModel\nmodel = AutoModel.from_pretrained(" + tc.arg + ")" + disc, unres := ParsePythonSource(src, "app.py", nil) + assert.Empty(t, disc, "local path must not be audited as a repo id") + require.Len(t, unres, 1) + assert.Equal(t, "local filesystem path", unres[0].Reason) + }) + } +} + +func TestParsePythonSource_RepoIDNotMistakenForLocalPath(t *testing.T) { + src := `from transformers import AutoModel +model = AutoModel.from_pretrained("openai-community/gpt2")` + disc, unres := ParsePythonSource(src, "app.py", nil) + require.Len(t, disc, 1) + assert.Equal(t, "openai-community/gpt2", disc[0].RepoID) + assert.Empty(t, unres) +} + +func TestParsePythonSource_DynamicRevision(t *testing.T) { + src := `snapshot_download(repo_id="org/model", revision=args.rev)` + disc, unres := ParsePythonSource(src, "test.py", nil) + require.Len(t, disc, 1) + assert.Equal(t, "org/model", disc[0].RepoID) + assert.True(t, disc[0].RevisionDynamic) + assert.Equal(t, DefaultRevision, disc[0].Revision) // falls back to main + assert.Empty(t, unres) +} + +func TestParsePythonSource_HfHubDownload(t *testing.T) { + src := `hf_hub_download(repo_id="org/model", filename="model.bin", revision="v1")` + disc, unres := ParsePythonSource(src, "test.py", nil) + require.Len(t, disc, 1) + assert.Equal(t, "org/model", disc[0].RepoID) + assert.Equal(t, "v1", disc[0].Revision) + assert.Empty(t, unres) +} + +func TestParsePythonSource_MultipleModels(t *testing.T) { + src := ` +from transformers import AutoModel, AutoTokenizer +model = AutoModel.from_pretrained("org/model-a", revision="v1") +tok = AutoTokenizer.from_pretrained("org/model-b") +` + disc, unres := ParsePythonSource(src, "test.py", nil) + assert.Len(t, disc, 2) + assert.Empty(t, unres) +} + +func TestParsePythonSource_CommentsIgnored(t *testing.T) { + src := `# from_pretrained("should-not-match") +model = AutoModel.from_pretrained("org/real-model")` + disc, _ := ParsePythonSource(src, "test.py", nil) + require.Len(t, disc, 1) + assert.Equal(t, "org/real-model", disc[0].RepoID) +} + +func TestBuildConstTable_Reassignment(t *testing.T) { + src := ` +MODEL = "first-value" +MODEL = "second-value" +` + table := buildConstTable(src) + _, exists := table["MODEL"] + assert.False(t, exists, "reassigned name should be removed from constant table") +} + +func TestBuildConstTable_SingleAssignment(t *testing.T) { + src := `BASE = "org/model"` + table := buildConstTable(src) + assert.Equal(t, "org/model", table["BASE"]) +} + +// TestBuildConstTable_ReassignmentViaNonLiteralExpression verifies that a name +// reassigned via a non-literal expression (e.g. an env-var override) is also +// removed from the constant table, not just a second bare-literal reassignment. +func TestBuildConstTable_ReassignmentViaNonLiteralExpression(t *testing.T) { + src := ` +MODEL_ID = "sentence-transformers/all-MiniLM-L6-v2" +MODEL_ID = os.getenv("HF_MODEL_ID", MODEL_ID) +` + table := buildConstTable(src) + _, exists := table["MODEL_ID"] + assert.False(t, exists, "reassignment via a non-literal expression should invalidate the constant") +} + +// TestBuildConstTable_ReassignmentViaCompoundAssignment verifies that a name +// mutated via a compound assignment (e.g. "+=") is also removed from the +// constant table, not just a plain "NAME = ..." reassignment. +func TestBuildConstTable_ReassignmentViaCompoundAssignment(t *testing.T) { + src := ` +MODEL_ID = "sentence-transformers/all-MiniLM-L6-v2" +MODEL_ID += "-v2" +` + table := buildConstTable(src) + _, exists := table["MODEL_ID"] + assert.False(t, exists, "reassignment via a compound assignment should invalidate the constant") +} + +// TestBuildConstTable_ReassignmentViaChainedAssignment verifies that a name +// reassigned as the middle/right target of a chained assignment (e.g. +// "MODEL_ID = FALLBACK_ID = ..."), not just as the line-leading target, is +// also removed from the constant table. +func TestBuildConstTable_ReassignmentViaChainedAssignment(t *testing.T) { + src := ` +FALLBACK_ID = "sentence-transformers/all-MiniLM-L6-v2" +MODEL_ID = FALLBACK_ID = "org/other-model" +` + table := buildConstTable(src) + _, exists := table["FALLBACK_ID"] + assert.False(t, exists, "reassignment as a chained-assignment target should invalidate the constant") +} + +// TestBuildConstTable_ChainedComparisonNotMistakenForAssignment guards against +// a false positive: a chained comparison ("a == b == c") must not be mistaken +// for a chained assignment reassigning "b". +func TestBuildConstTable_ChainedComparisonNotMistakenForAssignment(t *testing.T) { + src := ` +MODEL_ID = "org/my-model" +if MODEL_ID == FALLBACK == "unused": + pass +` + table := buildConstTable(src) + val, exists := table["MODEL_ID"] + assert.True(t, exists, "chained comparison must not be mistaken for chained assignment") + assert.Equal(t, "org/my-model", val) +} + +// TestParsePythonSource_EnvOverrideInvalidatesLiteral is the reviewer's regression +// case: a common deployment pattern keeps a checked-in fallback model but overrides +// it via an environment variable. The runtime model must be reported as unresolved +// instead of auditing the stale fallback literal. +func TestParsePythonSource_EnvOverrideInvalidatesLiteral(t *testing.T) { + t.Parallel() + src := ` +import os +from transformers import AutoModel + +MODEL_ID = "sentence-transformers/all-MiniLM-L6-v2" +MODEL_ID = os.getenv("HF_MODEL_ID", MODEL_ID) +model = AutoModel.from_pretrained(MODEL_ID) +` + discovered, unresolved := ParsePythonSource(src, "serve.py", nil) + assert.Empty(t, discovered, "the runtime model is selected by HF_MODEL_ID") + require.Len(t, unresolved, 1) + assert.Equal(t, "non-literal repo_id", unresolved[0].Reason) +} + +func TestParsePythonSource_PipelineKeywordModel(t *testing.T) { + src := `from transformers import pipeline +classifier = pipeline("text-classification", model="typeform/distilbert-base-uncased-mnli") +` + disc, unres := ParsePythonSource(src, "test.py", nil) + require.Len(t, disc, 1) + assert.Equal(t, "typeform/distilbert-base-uncased-mnli", disc[0].RepoID) + assert.Equal(t, DefaultRevision, disc[0].Revision) + assert.True(t, disc[0].RevisionDefaulted) + assert.Equal(t, RepoTypeModel, disc[0].RepoType) + assert.Empty(t, unres) +} + +func TestParsePythonSource_PipelineWithRevision(t *testing.T) { + src := `classifier = pipeline("text-classification", model="org/model", revision="v1")` + disc, unres := ParsePythonSource(src, "test.py", nil) + require.Len(t, disc, 1) + assert.Equal(t, "org/model", disc[0].RepoID) + assert.Equal(t, "v1", disc[0].Revision) + assert.False(t, disc[0].RevisionDefaulted) + assert.Empty(t, unres) +} + +func TestParsePythonSource_PipelineNoModel(t *testing.T) { + // pipeline("task") with no model= kwarg — nothing to audit, nothing to warn. + src := `pipe = pipeline("text-generation")` + disc, unres := ParsePythonSource(src, "test.py", nil) + assert.Empty(t, disc) + assert.Empty(t, unres) +} + +func TestParsePythonSource_PipelineTaskStringNotMistakenForModelID(t *testing.T) { + // Regression for critical finding: pipeline("text-generation") must NOT record + // "text-generation" as a model id. The first positional arg is the task string, + // not a repo id; keywordOnly=true on the pipeline callPattern suppresses the + // positional fallback so no bogus probe is emitted. + taskOnlyCases := []string{ + `pipeline("text-generation")`, + `pipeline("text-classification")`, + `pipeline("question-answering")`, + `pipeline("summarization")`, + `pipeline("translation_en_to_fr")`, + } + for _, src := range taskOnlyCases { + t.Run(src, func(t *testing.T) { + disc, unres := ParsePythonSource(src, "test.py", nil) + assert.Empty(t, disc, "task string must not be recorded as a model id") + assert.Empty(t, unres, "absent model= kwarg must not produce an unresolved warning") + }) + } +} + +func TestParsePythonSource_PipelineDynamicModel(t *testing.T) { + // pipeline("task", model=args.model) — model= is present but dynamic → unresolved warning. + src := `pipe = pipeline("text-classification", model=args.model)` + disc, unres := ParsePythonSource(src, "test.py", nil) + assert.Empty(t, disc) + require.Len(t, unres, 1) + assert.Equal(t, "non-literal repo_id", unres[0].Reason) +} + +func TestParsePythonSource_AmbiguousLocalOutputPath(t *testing.T) { + // "output/gpt2-finetuned" has one slash and no leading marker — same shape as a + // syntactically valid Hub id — and starts with a known local-output prefix. With + // no scan root (no filesystem evidence either way), it must be reported as + // ambiguous rather than silently treated as local or as a Hub id. + cases := []string{ + `"output/gpt2-finetuned"`, + `"outputs/run-1"`, + `"runs/exp-42"`, + `"checkpoints/step-1000"`, + `"checkpoint/best"`, + `"saved_models/bert-ft"`, + `"artifacts/model"`, + `"results/final"`, + `"finetuned/llama-lora"`, + `"trained/my-model"`, + } + for _, arg := range cases { + t.Run(arg, func(t *testing.T) { + src := "model = AutoModel.from_pretrained(" + arg + ")" + disc, unres := ParsePythonSource(src, "test.py", nil) + assert.Empty(t, disc, "ambiguous local-output-shaped literal must not be audited as a Hub id") + require.Len(t, unres, 1) + assert.Equal(t, "ambiguous local path or Hub repo id", unres[0].Reason) + }) + } +} + +func TestParsePythonSource_LocalOutputPathConfirmedByFilesystem(t *testing.T) { + // When a scan root is available and "output/gpt2-finetuned" exists on disk as a + // real directory relative to it, filesystem evidence confirms it's local. + root := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(root, "output", "gpt2-finetuned"), 0o755)) + + src := `model = AutoModel.from_pretrained("output/gpt2-finetuned")` + disc, unres := ParsePythonSource(src, "test.py", nil, root) + assert.Empty(t, disc, "filesystem-confirmed local dir must not be audited as a Hub id") + require.Len(t, unres, 1) + assert.Equal(t, "local filesystem path", unres[0].Reason) +} + +func TestParsePythonSource_LocalOutputPathAmbiguousWhenDirMissing(t *testing.T) { + // Same name-shaped literal, but the directory does NOT exist under root — still + // ambiguous, since the absence doesn't prove it's a real Hub id either (the + // script may just not have run yet, e.g. this is a training script being audited + // before its first run). + root := t.TempDir() + + src := `model = AutoModel.from_pretrained("output/gpt2-finetuned")` + disc, unres := ParsePythonSource(src, "test.py", nil, root) + assert.Empty(t, disc) + require.Len(t, unres, 1) + assert.Equal(t, "ambiguous local path or Hub repo id", unres[0].Reason) +} + +// TestParsePythonSource_ExistingUnlistedLocalPath: filesystem disambiguation must +// run even when the literal doesn't match localOutputPrefixes (e.g. "models/gpt2"). +func TestParsePythonSource_ExistingUnlistedLocalPath(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(root, "models", "gpt2"), 0o755)) + + discovered, unresolved := ParsePythonSource( + `from_pretrained("models/gpt2")`, "app.py", nil, root) + assert.Empty(t, discovered) + require.Len(t, unresolved, 1) + assert.Equal(t, "local filesystem path", unresolved[0].Reason) +} + +func TestParsePythonSource_LocalOutputPrefixFalsePositiveGuard(t *testing.T) { + // Legitimate Hub ids that superficially resemble output paths must not be flagged. + legitimate := []string{ + `"microsoft/phi-3"`, + `"google-bert/bert-base-uncased"`, + `"meta-llama/Llama-3.1-8B"`, + `"openai-community/gpt2"`, + } + for _, arg := range legitimate { + t.Run(arg, func(t *testing.T) { + src := "model = AutoModel.from_pretrained(" + arg + ")" + disc, unres := ParsePythonSource(src, "test.py", nil) + require.Len(t, disc, 1, "legitimate Hub id must be audited") + assert.Empty(t, unres) + }) + } +} + +func TestParsePythonSource_NestedParenHandled(t *testing.T) { + // The outer call's first positional arg is a function call result (non-literal), + // so it goes to unresolved. The inner AutoConfig.from_pretrained("base-model") is + // itself a valid HF call and is discovered independently. + src := `model = AutoModel.from_pretrained(AutoConfig.from_pretrained("base-model"), revision="v1") +` + discovered, unresolved := ParsePythonSource(src, "test.py", nil) + // Inner call discovered correctly. + require.Len(t, discovered, 1, "inner from_pretrained(\"base-model\") should be discovered") + assert.Equal(t, "base-model", discovered[0].RepoID) + // Outer call goes to unresolved (non-literal first arg). + require.Len(t, unresolved, 1, "outer call with non-literal repo_id should be in UnresolvedSites") +} diff --git a/sca/bom/buildinfo/technologies/huggingface/discovery/result.go b/sca/bom/buildinfo/technologies/huggingface/discovery/result.go new file mode 100644 index 000000000..f1265e737 --- /dev/null +++ b/sca/bom/buildinfo/technologies/huggingface/discovery/result.go @@ -0,0 +1,75 @@ +package discovery + +// DefaultRevision is the fallback revision when none is pinned in source, +// mirroring the Hugging Face client's default behaviour. +const DefaultRevision = "main" + +// RepoType distinguishes HF model repos from dataset repos. Only models are +// audited; datasets are detected and reported but not curated (see RepoTypeDataset). +type RepoType string + +const ( + RepoTypeModel RepoType = "model" + RepoTypeDataset RepoType = "dataset" +) + +// Location identifies a specific line in a source file. +// For Jupyter notebooks the File is "notebook.ipynb#cell-" and Line is +// relative to the start of that cell. +type Location struct { + File string + Line int // 1-based +} + +// DiscoveredModel is a fully or partially resolved HF reference extracted from source. +type DiscoveredModel struct { + RepoID string + // Revision is the pinned branch/tag/sha, or DefaultRevision when absent in source. + Revision string + // RevisionDefaulted is true when no revision was present in the call — + // the audit targets whatever commit the branch currently points to. + RevisionDefaulted bool + // RevisionDynamic is true when a revision arg was present but non-literal. + // The model is still audited against DefaultRevision with a warning. + RevisionDynamic bool + RepoType RepoType + // Sources lists every call site that produced this reference (after dedup). + Sources []Location +} + +// UnresolvedSite records a call site whose repo_id could not be statically resolved. +// These are NOT audited; they are surfaced in the warning block so the user can +// pass them explicitly via --hugging-face-model. +type UnresolvedSite struct { + Location Location + // Snippet shows the call's arguments; quoted content is redacted as + // since it may embed a secret and this text goes to CI logs. + Snippet string + // Reason is one of "non-literal repo_id", "f-string repo_id", + // "dynamic repo_id", "local filesystem path", + // "ambiguous local path or Hub repo id", "non-literal repo_type", + // "f-string repo_type", or "unsupported repo_type". + Reason string +} + +// SkippedFile records a source file/entry that could not be read or parsed +// during the scan — any HF references it held were not discovered. +type SkippedFile struct { + Path string + Err string +} + +// ScanResult is the output of a full directory scan. +type ScanResult struct { + // Discovered holds deduplicated model (repo_type == model) tuples ready to + // hand to the curation walker. + Discovered []DiscoveredModel + // Datasets holds deduplicated dataset references. These are reported but NOT + // audited: Catalog does not score datasets, so curation cannot evaluate them. + Datasets []DiscoveredModel + // Unresolved holds call sites that could not be statically resolved. + Unresolved []UnresolvedSite + // Skipped holds files/entries that could not be read or parsed. Non-empty + // means the scan is partial. + Skipped []SkippedFile +} diff --git a/sca/bom/buildinfo/technologies/huggingface/discovery/scanner.go b/sca/bom/buildinfo/technologies/huggingface/discovery/scanner.go new file mode 100644 index 000000000..b5f27e6b3 --- /dev/null +++ b/sca/bom/buildinfo/technologies/huggingface/discovery/scanner.go @@ -0,0 +1,241 @@ +package discovery + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/jfrog/jfrog-client-go/utils/log" +) + +// defaultExcludeDirs are directory names skipped during the walk. +// These mirror the exclusion list used by other jf ca BOM builders. +var defaultExcludeDirs = map[string]struct{}{ + ".git": {}, + ".hg": {}, + "node_modules": {}, + "__pycache__": {}, + ".venv": {}, + "venv": {}, + "env": {}, + ".env": {}, + "site-packages": {}, + ".tox": {}, + "dist": {}, + "build": {}, + ".eggs": {}, + // Jupyter's autosave copies — scanning them would double-report or surface stale + // (already-edited-away) model references from the checkpointed version. + ".ipynb_checkpoints": {}, +} + +// IsExcludedWalkDir reports whether a directory basename should be skipped when +// scanning for Python/Hugging Face sources (venv artifacts, build outputs, etc.). +func IsExcludedWalkDir(name string) bool { + _, skip := defaultExcludeDirs[name] + return skip +} + +// maxScannableFileSize caps how large a single .py/.ipynb file can be before it's +// skipped unread. Generous relative to real source files (even notebooks with +// embedded base64 image outputs rarely approach this) — it exists to guard against +// a pathologically large or misnamed file being fully loaded into memory. +const maxScannableFileSize = 10 * 1024 * 1024 // 10 MiB + +// ScanDir walks root recursively and discovers all Hugging Face model/dataset +// references in *.py files and *.ipynb notebooks. +// It returns a deduplicated ScanResult with the warning block pre-formatted. +func ScanDir(root string) (*ScanResult, error) { + var allDiscovered []DiscoveredModel + var allUnresolved []UnresolvedSite + var skipped []SkippedFile + + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + log.Debug(fmt.Sprintf("huggingface scanner: skipping %s: %v", path, err)) + skipped = append(skipped, SkippedFile{Path: path, Err: err.Error()}) + return nil + } + if d.IsDir() { + if IsExcludedWalkDir(d.Name()) { + return filepath.SkipDir + } + return nil + } + ext := strings.ToLower(filepath.Ext(path)) + if ext != ".py" && ext != ".ipynb" { + log.Debug(fmt.Sprintf("huggingface scanner: skipping %s: unsupported extension", path)) + return nil + } + if info, infoErr := d.Info(); infoErr == nil && info.Size() > maxScannableFileSize { + ferr := fmt.Errorf("file size %d bytes exceeds the %d byte scan limit", info.Size(), maxScannableFileSize) + log.Debug(fmt.Sprintf("huggingface scanner: skipping %s: %v", path, ferr)) + skipped = append(skipped, SkippedFile{Path: path, Err: ferr.Error()}) + return nil + } + switch ext { + case ".py": + disc, unres, ferr := scanPyFile(path, root) + if ferr != nil { + log.Debug(fmt.Sprintf("huggingface scanner: skipping %s: %v", path, ferr)) + skipped = append(skipped, SkippedFile{Path: path, Err: ferr.Error()}) + return nil + } + allDiscovered = append(allDiscovered, disc...) + allUnresolved = append(allUnresolved, unres...) + case ".ipynb": + disc, unres, ferr := ParseNotebook(path, root) + if ferr != nil { + log.Debug(fmt.Sprintf("huggingface scanner: skipping notebook %s: %v", path, ferr)) + skipped = append(skipped, SkippedFile{Path: path, Err: ferr.Error()}) + return nil + } + // Relativise notebook path for display. + rel, relErr := filepath.Rel(root, path) + if relErr != nil { + rel = path + } + for i := range disc { + for j := range disc[i].Sources { + disc[i].Sources[j].File = rebaseNotebook(disc[i].Sources[j].File, path, rel) + } + } + for i := range unres { + unres[i].Location.File = rebaseNotebook(unres[i].Location.File, path, rel) + } + allDiscovered = append(allDiscovered, disc...) + allUnresolved = append(allUnresolved, unres...) + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("huggingface scanner: walking %s: %w", root, err) + } + + // Split models from datasets: Catalog scores only models, so datasets are + // reported (via FormatDatasetWarning) but never handed to the curation walker. + var models, datasets []DiscoveredModel + for _, m := range dedup(allDiscovered) { + if m.RepoType == RepoTypeDataset { + datasets = append(datasets, m) + } else { + models = append(models, m) + } + } + + for i := range skipped { + // Falls back to the absolute path on error (e.g. different volume on Windows) — + // still usable in the warning, just more verbose. + if rel, relErr := filepath.Rel(root, skipped[i].Path); relErr == nil { + skipped[i].Path = rel + } + } + + return &ScanResult{ + Discovered: models, + Datasets: datasets, + Unresolved: allUnresolved, + Skipped: skipped, + }, nil +} + +// FormatWarnings returns the consolidated warning block for unresolved call sites, +// or an empty string if there are none. +func FormatWarnings(unresolved []UnresolvedSite) string { + if len(unresolved) == 0 { + return "" + } + var sb strings.Builder + fmt.Fprintf(&sb, "%d Hugging Face reference(s) could not be statically resolved and were NOT audited:\n", len(unresolved)) + for _, u := range unresolved { + fmt.Fprintf(&sb, " %s:%d\t%s\t— %s\n", u.Location.File, u.Location.Line, u.Snippet, u.Reason) + } + sb.WriteString("These references could not be tied to a Hub repo id statically (runtime values or local filesystem paths), so they were not audited here.\n") + sb.WriteString("A local path is typically a model fetched in an earlier step (e.g. via `jf hf download`), where curation is enforced at download time.\n") + sb.WriteString("To audit them here, re-run with --hugging-face-model=: (comma-separate multiple models; pin the revision you ship).") + return sb.String() +} + +// FormatDatasetWarning returns an advisory for detected dataset references, which +// are reported but not audited (Catalog does not score datasets). Empty when none. +func FormatDatasetWarning(datasets []DiscoveredModel) string { + if len(datasets) == 0 { + return "" + } + var sb strings.Builder + fmt.Fprintf(&sb, "%d Hugging Face dataset reference(s) found but NOT audited — curation does not currently cover datasets (Catalog limitation). Only models are evaluated.\n", len(datasets)) + for _, d := range datasets { + loc := "" + if len(d.Sources) > 0 { + loc = fmt.Sprintf("%s:%d\t", d.Sources[0].File, d.Sources[0].Line) + } + fmt.Fprintf(&sb, " %s%s\n", loc, d.RepoID) + } + return strings.TrimRight(sb.String(), "\n") +} + +// FormatSkippedFilesWarning returns a user-visible warning listing source files +// that could not be read or parsed (scan is partial). Empty when none were skipped. +func FormatSkippedFilesWarning(skipped []SkippedFile) string { + if len(skipped) == 0 { + return "" + } + var sb strings.Builder + fmt.Fprintf(&sb, "%d source file(s) could not be scanned for Hugging Face references and were skipped — the scan results are PARTIAL:\n", len(skipped)) + for _, s := range skipped { + fmt.Fprintf(&sb, " %s — %s\n", s.Path, s.Err) + } + sb.WriteString("Any Hugging Face models/datasets referenced only in these files were not audited.") + return strings.TrimRight(sb.String(), "\n") +} + +// ---- internal helpers ----------------------------------------------------- + +func scanPyFile(path, root string) ([]DiscoveredModel, []UnresolvedSite, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, nil, err + } + rel, err := filepath.Rel(root, path) + if err != nil { + rel = path + } + disc, unres := ParsePythonSource(string(data), rel, nil, root) + return disc, unres, nil +} + +// rebaseNotebook replaces the absolute notebook path prefix with the relative one. +func rebaseNotebook(location, absPath, relPath string) string { + return strings.Replace(location, absPath, relPath, 1) +} + +// dedup collapses DiscoveredModel entries with the same (repo_type, repo_id, revision) +// into one, merging their Sources lists. +func dedup(models []DiscoveredModel) []DiscoveredModel { + type key struct { + repoType RepoType + repoID string + revision string + } + index := map[key]int{} + var result []DiscoveredModel + for _, m := range models { + k := key{m.RepoType, m.RepoID, m.Revision} + if idx, exists := index[k]; exists { + result[idx].Sources = append(result[idx].Sources, m.Sources...) + // Propagate flags: if any reference lacked a revision, flag it. + if m.RevisionDefaulted { + result[idx].RevisionDefaulted = true + } + if m.RevisionDynamic { + result[idx].RevisionDynamic = true + } + } else { + index[k] = len(result) + result = append(result, m) + } + } + return result +} diff --git a/sca/bom/buildinfo/technologies/huggingface/discovery/scanner_test.go b/sca/bom/buildinfo/technologies/huggingface/discovery/scanner_test.go new file mode 100644 index 000000000..c3fa6bf8d --- /dev/null +++ b/sca/bom/buildinfo/technologies/huggingface/discovery/scanner_test.go @@ -0,0 +1,210 @@ +package discovery + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestScanDir_BasicDiscovery(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "app.py", ` +from transformers import AutoModel +model = AutoModel.from_pretrained("org/model-a", revision="v1") +`) + result, err := ScanDir(dir) + require.NoError(t, err) + require.Len(t, result.Discovered, 1) + assert.Equal(t, "org/model-a", result.Discovered[0].RepoID) + assert.Empty(t, result.Unresolved) +} + +func TestScanDir_Dedup(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "a.py", `from_pretrained("org/model", revision="v1")`) + writeFile(t, dir, "b.py", `from_pretrained("org/model", revision="v1")`) + result, err := ScanDir(dir) + require.NoError(t, err) + assert.Len(t, result.Discovered, 1, "same model in two files should deduplicate") + assert.Len(t, result.Discovered[0].Sources, 2, "both source locations should be recorded") +} + +func TestScanDir_ExcludeVenv(t *testing.T) { + dir := t.TempDir() + venv := filepath.Join(dir, ".venv") + require.NoError(t, os.MkdirAll(venv, 0755)) + writeFile(t, venv, "excluded.py", `from_pretrained("org/should-not-appear")`) + writeFile(t, dir, "real.py", `from_pretrained("org/real-model")`) + result, err := ScanDir(dir) + require.NoError(t, err) + require.Len(t, result.Discovered, 1) + assert.Equal(t, "org/real-model", result.Discovered[0].RepoID) +} + +func TestScanDir_UnresolvedWarning(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "dynamic.py", `from_pretrained(args.model_name)`) + result, err := ScanDir(dir) + require.NoError(t, err) + assert.Empty(t, result.Discovered) + require.Len(t, result.Unresolved, 1) + + warn := FormatWarnings(result.Unresolved) + assert.Contains(t, warn, "could not be statically resolved") + assert.Contains(t, warn, "--hugging-face-model") +} + +func TestScanDir_MixedResolved(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "mixed.py", ` +from_pretrained("org/good-model", revision="v1") +from_pretrained(config.model_id) +snapshot_download(repo_id="org/dataset", repo_type="dataset") +`) + result, err := ScanDir(dir) + require.NoError(t, err) + // Only the model lands in Discovered; the dataset is separated out. + require.Len(t, result.Discovered, 1) + assert.Equal(t, "org/good-model", result.Discovered[0].RepoID) + require.Len(t, result.Datasets, 1) + assert.Equal(t, "org/dataset", result.Datasets[0].RepoID) + assert.Len(t, result.Unresolved, 1) +} + +func TestScanDir_DatasetsNotAudited(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "data.py", ` +from datasets import load_dataset +ds = load_dataset("squad", revision="1.0.0") +`) + result, err := ScanDir(dir) + require.NoError(t, err) + assert.Empty(t, result.Discovered, "datasets must not be handed to the curation walker") + require.Len(t, result.Datasets, 1) + assert.Equal(t, "squad", result.Datasets[0].RepoID) +} + +func TestFormatDatasetWarning_Empty(t *testing.T) { + assert.Equal(t, "", FormatDatasetWarning(nil)) + assert.Equal(t, "", FormatDatasetWarning([]DiscoveredModel{})) +} + +func TestFormatDatasetWarning_Content(t *testing.T) { + datasets := []DiscoveredModel{ + {RepoID: "squad", Revision: "1.0.0", RepoType: RepoTypeDataset, + Sources: []Location{{File: "data.py", Line: 3}}}, + } + out := FormatDatasetWarning(datasets) + assert.Contains(t, out, "1 Hugging Face dataset reference(s) found but NOT audited") + assert.Contains(t, out, "curation does not currently cover datasets (Catalog limitation)") + assert.Contains(t, out, "Only models are evaluated") + assert.Contains(t, out, "data.py:3") + assert.Contains(t, out, "squad") +} + +func TestScanDir_SkipsUnparsableFileAndWarns(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "real.py", `from_pretrained("org/real-model")`) + writeFile(t, dir, "bad.ipynb", `{not valid json`) + + result, err := ScanDir(dir) + require.NoError(t, err, "a single unparsable file must not fail the whole scan") + + // The good file is still fully scanned despite the bad one. + require.Len(t, result.Discovered, 1) + assert.Equal(t, "org/real-model", result.Discovered[0].RepoID) + + // The bad file is recorded as skipped rather than silently dropped. + require.Len(t, result.Skipped, 1) + assert.Equal(t, "bad.ipynb", result.Skipped[0].Path) + assert.NotEmpty(t, result.Skipped[0].Err) + + warn := FormatSkippedFilesWarning(result.Skipped) + assert.Contains(t, warn, "1 source file(s) could not be scanned") + assert.Contains(t, warn, "PARTIAL") + assert.Contains(t, warn, "bad.ipynb") +} + +func TestFormatSkippedFilesWarning_Empty(t *testing.T) { + assert.Equal(t, "", FormatSkippedFilesWarning(nil)) + assert.Equal(t, "", FormatSkippedFilesWarning([]SkippedFile{})) +} + +func TestScanDir_ExcludeDist(t *testing.T) { + dir := t.TempDir() + dist := filepath.Join(dir, "dist") + require.NoError(t, os.MkdirAll(dist, 0755)) + writeFile(t, dist, "excluded.py", `from_pretrained("org/should-not-appear")`) + writeFile(t, dir, "real.py", `from_pretrained("org/real-model")`) + result, err := ScanDir(dir) + require.NoError(t, err) + require.Len(t, result.Discovered, 1) + assert.Equal(t, "org/real-model", result.Discovered[0].RepoID) +} + +func TestScanDir_ExcludeIpynbCheckpoints(t *testing.T) { + dir := t.TempDir() + checkpoints := filepath.Join(dir, ".ipynb_checkpoints") + require.NoError(t, os.MkdirAll(checkpoints, 0755)) + writeFile(t, checkpoints, "stale-checkpoint.ipynb", `{"cells":[]}`) + writeFile(t, dir, "real.py", `from_pretrained("org/real-model")`) + result, err := ScanDir(dir) + require.NoError(t, err) + require.Len(t, result.Discovered, 1) + assert.Equal(t, "org/real-model", result.Discovered[0].RepoID) + assert.Empty(t, result.Skipped, "checkpoint dir should be skipped by the walk, not reported as unreadable") +} + +func TestScanDir_SkipsOversizedFileAndWarns(t *testing.T) { + dir := t.TempDir() + huge := make([]byte, maxScannableFileSize+1) + require.NoError(t, os.WriteFile(filepath.Join(dir, "huge.py"), huge, 0644)) + writeFile(t, dir, "real.py", `from_pretrained("org/real-model")`) + + result, err := ScanDir(dir) + require.NoError(t, err, "an oversized file must not fail the whole scan") + + require.Len(t, result.Discovered, 1) + assert.Equal(t, "org/real-model", result.Discovered[0].RepoID) + + require.Len(t, result.Skipped, 1) + assert.Equal(t, "huge.py", result.Skipped[0].Path) + assert.Contains(t, result.Skipped[0].Err, "exceeds") +} + +func TestScanDir_IgnoresUnsupportedExtensions(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "README.md", `from_pretrained("org/should-not-appear")`) + writeFile(t, dir, "requirements.txt", `transformers==4.0.0`) + writeFile(t, dir, "real.py", `from_pretrained("org/real-model")`) + + result, err := ScanDir(dir) + require.NoError(t, err) + require.Len(t, result.Discovered, 1) + assert.Equal(t, "org/real-model", result.Discovered[0].RepoID) + assert.Empty(t, result.Skipped, "unsupported extensions are ignored, not reported as skipped/unreadable") +} + +func TestFormatWarnings_Empty(t *testing.T) { + assert.Equal(t, "", FormatWarnings(nil)) + assert.Equal(t, "", FormatWarnings([]UnresolvedSite{})) +} + +func TestFormatWarnings_Content(t *testing.T) { + sites := []UnresolvedSite{ + {Location: Location{File: "trainer.py", Line: 42}, Snippet: "from_pretrained(args.m)", Reason: "non-literal repo_id"}, + } + out := FormatWarnings(sites) + assert.Contains(t, out, "could not be statically resolved") + assert.Contains(t, out, "trainer.py:42") + assert.Contains(t, out, "non-literal repo_id") + assert.Contains(t, out, "--hugging-face-model") +} + +func writeFile(t *testing.T, dir, name, content string) { + t.Helper() + require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(content), 0644)) +} diff --git a/sca/bom/buildinfo/technologies/huggingface/huggingface.go b/sca/bom/buildinfo/technologies/huggingface/huggingface.go new file mode 100644 index 000000000..b7344437d --- /dev/null +++ b/sca/bom/buildinfo/technologies/huggingface/huggingface.go @@ -0,0 +1,369 @@ +package huggingface + +import ( + "fmt" + "net/url" + "os" + "path/filepath" + "strings" + + "github.com/jfrog/jfrog-cli-core/v2/common/project" + "github.com/jfrog/jfrog-cli-core/v2/utils/config" + "github.com/jfrog/jfrog-cli-security/sca/bom/buildinfo/technologies" + "github.com/jfrog/jfrog-cli-security/sca/bom/buildinfo/technologies/huggingface/discovery" + "github.com/jfrog/jfrog-cli-security/utils/artifactory" + "github.com/jfrog/jfrog-client-go/utils/log" + xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils" +) + +// HuggingFacePackagePrefix is the node-id prefix used for Hugging Face model/dataset references. +const HuggingFacePackagePrefix = "huggingfaceml://" + +// DefaultRevision re-exports discovery.DefaultRevision (single source of truth). +const DefaultRevision = discovery.DefaultRevision + +// hfEndpointEnv is the env var the Hugging Face client uses to point at the Artifactory +// proxy, e.g. "https://my.jfrog.io/artifactory/api/huggingfaceml/my-hugging-face-repo". +const hfEndpointEnv = "HF_ENDPOINT" + +// hfEndpointRepoMarker precedes the Artifactory repository name in HF_ENDPOINT. +const hfEndpointRepoMarker = "api/huggingfaceml/" + +// ModelInfo holds the parsed components of a --hugging-face-model flag value +// ("[:]"). The Artifactory repository is derived from HF_ENDPOINT. +type ModelInfo struct { + RepoID string + Revision string +} + +// modelNodeID builds the graph node id for a model reference. Shared by both the +// explicit --hugging-face-model path and the auto-discovery path. +func modelNodeID(repoID, revision string) string { + return HuggingFacePackagePrefix + repoID + ":" + revision +} + +// SplitRepoIDAndRevision splits "[:]" on the last ':'. +// +// Revisions can contain '/' (e.g. "refs/pr/3"), so the split isn't slash-guarded, +// or such revisions get glued onto repoID and 404 on probe. +// +// No ':' in raw: revision is "" and repoID is raw unchanged. Trailing ':' (e.g. +// "org/model:") also yields revision == "" — callers apply their own default. +func SplitRepoIDAndRevision(raw string) (repoID, revision string) { + if idx := strings.LastIndex(raw, ":"); idx >= 0 { + return raw[:idx], raw[idx+1:] + } + return raw, "" +} + +// ParseModelReference parses "[:]" into a ModelInfo. +// Revision defaults to "main" when not specified. +func ParseModelReference(modelRef string) (*ModelInfo, error) { + modelRef = strings.TrimSpace(strings.TrimPrefix(modelRef, HuggingFacePackagePrefix)) + if modelRef == "" { + return nil, fmt.Errorf("hugging face model reference is empty") + } + + originalRef := modelRef + repoID, revision := SplitRepoIDAndRevision(modelRef) + if repoID == "" { + return nil, fmt.Errorf("invalid hugging face model reference %q: missing repo id; expected '[:]' (revision defaults to 'main'), comma-separated for multiple (e.g. 'mcpotato/42-eicar-street:main,bert-base-uncased')", originalRef) + } + if revision == "" { + revision = DefaultRevision + } + info := &ModelInfo{RepoID: repoID, Revision: revision} + + log.Debug(fmt.Sprintf("Parsed Hugging Face model - RepoID: %s, Revision: %s", info.RepoID, info.Revision)) + return info, nil +} + +// ParseModelReferences parses a comma-separated list of "[:]" entries. +func ParseModelReferences(modelRefs string) ([]*ModelInfo, error) { + var infos []*ModelInfo + for _, ref := range strings.Split(modelRefs, ",") { + ref = strings.TrimSpace(ref) + if ref == "" { + continue + } + info, err := ParseModelReference(ref) + if err != nil { + return nil, err + } + infos = append(infos, info) + } + if len(infos) == 0 { + return nil, fmt.Errorf("hugging face model reference is empty") + } + return infos, nil +} + +// repoFromHFEndpoint extracts the Artifactory repository name from the HF_ENDPOINT env var. +// HF_ENDPOINT looks like ".../artifactory/api/huggingfaceml/"; we return "". +func repoFromHFEndpoint() (string, error) { + endpoint := strings.TrimSpace(os.Getenv(hfEndpointEnv)) + if endpoint == "" { + return "", fmt.Errorf("%s is not set. Export it to your Artifactory Hugging Face repository, e.g. '%s=https:///artifactory/%s'", + hfEndpointEnv, hfEndpointEnv, hfEndpointRepoMarker) + } + idx := strings.Index(endpoint, hfEndpointRepoMarker) + if idx < 0 { + return "", fmt.Errorf("%s ('%s') does not contain '%s'; cannot determine the Artifactory repository", hfEndpointEnv, endpoint, hfEndpointRepoMarker) + } + // The repository is the first path segment after the marker (ignore any trailing path/query). + repo := strings.Trim(endpoint[idx+len(hfEndpointRepoMarker):], "/") + if before, _, found := strings.Cut(repo, "/"); found { + repo = before + } + if before, _, found := strings.Cut(repo, "?"); found { + repo = before + } + if repo == "" { + return "", fmt.Errorf("%s ('%s') has no repository segment after '%s'", hfEndpointEnv, endpoint, hfEndpointRepoMarker) + } + return repo, nil +} + +// BuildDependencyTree builds the HF dependency graph. +// If --hugging-face-model is set it audits only those models (spot-check, no source scan). +// Otherwise it auto-discovers model references in Python source and notebooks. +func BuildDependencyTree(params technologies.BuildInfoBomGeneratorParams) (trees []*xrayUtils.GraphNode, uniqueIDs []string, warnings []string, err error) { + workingDir := params.WorkingDirectory + if workingDir == "" { + workingDir = "." + } + + var children []*xrayUtils.GraphNode + seen := map[string]bool{} + add := func(nodeID string) { + if seen[nodeID] { + return + } + seen[nodeID] = true + children = append(children, &xrayUtils.GraphNode{Id: nodeID}) + uniqueIDs = append(uniqueIDs, nodeID) + } + + if params.HuggingFaceModel != "" { + models, perr := ParseModelReferences(params.HuggingFaceModel) + if perr != nil { + return nil, nil, nil, perr + } + for _, m := range models { + add(modelNodeID(m.RepoID, m.Revision)) + } + if len(children) == 0 { + return nil, nil, nil, nil + } + root := &xrayUtils.GraphNode{Id: rootNodeName(params), Nodes: children} + return []*xrayUtils.GraphNode{root}, uniqueIDs, nil, nil + } + + // 2) Auto-discovery mode: scan Python source / notebooks in the working dir. + log.Debug(fmt.Sprintf("Hugging Face: scanning %s for model references", workingDir)) + result, serr := discovery.ScanDir(workingDir) + if serr != nil { + return nil, nil, nil, fmt.Errorf("hugging face source scan failed: %w", serr) + } + if warn := discovery.FormatWarnings(result.Unresolved); warn != "" { + warnings = append(warnings, warn) + } + // Datasets are detected but not audited — Catalog does not score datasets. + if warn := discovery.FormatDatasetWarning(result.Datasets); warn != "" { + warnings = append(warnings, warn) + } + // Unreadable/unparsable files mean the scan is partial — surface this to the + // user rather than silently reporting complete coverage (debug log only). + if warn := discovery.FormatSkippedFilesWarning(result.Skipped); warn != "" { + warnings = append(warnings, warn) + } + for _, m := range result.Discovered { + nodeID := modelNodeID(m.RepoID, m.Revision) + if m.RevisionDefaulted { + log.Info(fmt.Sprintf("Hugging Face: %s has no pinned revision — auditing against current HEAD of '%s'", m.RepoID, m.Revision)) + } + if m.RevisionDynamic { + warnings = append(warnings, fmt.Sprintf("Hugging Face: %s has a dynamic revision in source — audited against '%s' (may not match the revision resolved at runtime)", m.RepoID, m.Revision)) + } + add(nodeID) + } + + if len(children) == 0 { + log.Debug("Hugging Face: no model references found (flag or source)") + return nil, nil, warnings, nil + } + + // Flat graph: one root node whose children are the unique model/dataset refs. + root := &xrayUtils.GraphNode{Id: rootNodeName(params), Nodes: children} + return []*xrayUtils.GraphNode{root}, uniqueIDs, warnings, nil +} + +// rootNodeName returns the synthetic root node ID for the HF dependency graph. +// Prefers params.HFProjectName (set via DisambiguateRootNodeNames for multiple +// --working-dirs); falls back to the working directory basename. +func rootNodeName(params technologies.BuildInfoBomGeneratorParams) string { + if params.HFProjectName != "" { + return params.HFProjectName + } + dir := params.WorkingDirectory + if dir == "" || dir == "." { + if cwd, err := os.Getwd(); err == nil { + dir = cwd + } + } + if name := filepath.Base(dir); name != "" && name != "." && name != "/" { + return name + } + return "huggingface-project" +} + +// DisambiguateRootNodeNames maps each working directory to a unique project name, +// even when directories share a basename (e.g. two "model" dirs under different +// parents). Basenames are used when unique; collisions fall back to "parent/base", +// then the full absolute path. +func DisambiguateRootNodeNames(workingDirs []string) map[string]string { + seen := map[string]bool{} + var abs []string + for _, wd := range workingDirs { + a := wd + if resolved, err := filepath.Abs(wd); err == nil { + a = resolved + } + if !seen[a] { + seen[a] = true + abs = append(abs, a) + } + } + + names := make(map[string]string, len(abs)) + // assign names paths whose candidate name is unique; returns the rest for the next tier. + assign := func(nameOf func(string) string) (remaining []string) { + counts := map[string]int{} + candidate := map[string]string{} + for _, p := range abs { + n := nameOf(p) + candidate[p] = n + counts[n]++ + } + for p, n := range candidate { + if counts[n] == 1 { + names[p] = n + } else { + remaining = append(remaining, p) + } + } + return + } + + remaining := assign(func(p string) string { + if name := filepath.Base(p); name != "" && name != "." && name != "/" { + return name + } + return "huggingface-project" + }) + if len(remaining) == 0 { + return names + } + + abs = remaining + remaining = assign(func(p string) string { + return filepath.Base(filepath.Dir(p)) + "/" + filepath.Base(p) + }) + if len(remaining) == 0 { + return names + } + + // Last resort: the full absolute path. + for _, p := range remaining { + names[p] = p + } + return names +} + +// GetHuggingFaceRepositoryConfig resolves the Artifactory repository from HF_ENDPOINT +// and verifies it exists on serverDetails (mirrors docker.GetDockerRepositoryConfig). +// +// serverDetails must be the server already resolved for this command (respects +// --server-id) — it is NOT re-resolved here. HF_ENDPOINT's host is validated against +// it so a stale HF_ENDPOINT can't silently probe the wrong Artifactory instance. +func GetHuggingFaceRepositoryConfig(serverDetails *config.ServerDetails) (*project.RepositoryConfig, error) { + if serverDetails == nil { + return nil, fmt.Errorf("no Artifactory server configured. Use 'jf c add' to configure a server") + } + repo, err := repoFromHFEndpoint() + if err != nil { + return nil, err + } + if err = validateHFEndpointHost(serverDetails); err != nil { + return nil, err + } + exists, err := artifactory.IsRepoExists(repo, serverDetails) + if err != nil { + return nil, fmt.Errorf("failed to check if repository '%s' exists on Artifactory '%s': %w", repo, serverDetails.Url, err) + } + if !exists { + return nil, fmt.Errorf("repository '%s' (from %s) was not found on Artifactory (%s), ensure the repository exists", repo, hfEndpointEnv, serverDetails.Url) + } + + repoConfig := &project.RepositoryConfig{} + repoConfig.SetServerDetails(serverDetails).SetTargetRepo(repo) + return repoConfig, nil +} + +// validateHFEndpointHost ensures HF_ENDPOINT points at the same Artifactory host as +// serverDetails. Only the host is compared — path/scheme differences are tolerated. +func validateHFEndpointHost(serverDetails *config.ServerDetails) error { + endpoint := strings.TrimSpace(os.Getenv(hfEndpointEnv)) + endpointHost, err := hostOf(endpoint) + if err != nil { + return fmt.Errorf("%s ('%s') is not a valid URL: %w", hfEndpointEnv, endpoint, err) + } + serverURL := serverDetails.GetArtifactoryUrl() + serverHost, err := hostOf(serverURL) + if err != nil { + return fmt.Errorf("failed to parse the configured Artifactory URL '%s': %w", serverURL, err) + } + if !strings.EqualFold(endpointHost, serverHost) { + return fmt.Errorf("%s ('%s') points at '%s', but the selected Artifactory server ('%s') is '%s'; "+ + "export %s for the same server you're auditing against (or pass the matching --server-id)", + hfEndpointEnv, endpoint, endpointHost, serverDetails.ServerId, serverHost, hfEndpointEnv) + } + return nil +} + +// hostOf parses rawURL and returns its host[:port], erroring if it has none. +// The port is omitted when it's the scheme's default (443/80), so equivalent +// URLs don't false-mismatch in validateHFEndpointHost. +// +// A URL with no scheme (e.g. "my.jfrog.io/artifactory/...") parses without +// error but with an empty Host — url.Parse treats it as a relative path — so +// that specific case gets a more actionable message pointing at the missing +// scheme, rather than a bare "no host in URL". +func hostOf(rawURL string) (string, error) { + parsed, err := url.Parse(rawURL) + if err != nil { + return "", err + } + if parsed.Host == "" { + if !strings.Contains(rawURL, "://") { + return "", fmt.Errorf("no host found — missing a scheme (e.g. 'https://')") + } + return "", fmt.Errorf("no host in URL") + } + if port := parsed.Port(); port != "" && isDefaultPort(parsed.Scheme, port) { + return parsed.Hostname(), nil + } + return parsed.Host, nil +} + +// isDefaultPort reports whether port is the well-known default for scheme. +func isDefaultPort(scheme, port string) bool { + switch strings.ToLower(scheme) { + case "http": + return port == "80" + case "https": + return port == "443" + default: + return false + } +} diff --git a/sca/bom/buildinfo/technologies/huggingface/huggingface_test.go b/sca/bom/buildinfo/technologies/huggingface/huggingface_test.go new file mode 100644 index 000000000..85c479797 --- /dev/null +++ b/sca/bom/buildinfo/technologies/huggingface/huggingface_test.go @@ -0,0 +1,434 @@ +package huggingface + +import ( + "os" + "path/filepath" + "testing" + + "github.com/jfrog/jfrog-cli-core/v2/utils/config" + "github.com/jfrog/jfrog-cli-security/sca/bom/buildinfo/technologies" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseModelReference(t *testing.T) { + tests := []struct { + name string + input string + wantRepoID string + wantRevision string + wantErr string + }{ + { + name: "model id with sha revision", + input: "mcpotato/42-eicar-street:8fb61c4d511e9aaff0ea55396a124aa292830efc", + wantRepoID: "mcpotato/42-eicar-street", + wantRevision: "8fb61c4d511e9aaff0ea55396a124aa292830efc", + }, + { + name: "model id with branch revision", + input: "mcpotato/42-eicar-street:main", + wantRepoID: "mcpotato/42-eicar-street", + wantRevision: "main", + }, + { + name: "no revision defaults to main", + input: "org/model", + wantRepoID: "org/model", + wantRevision: DefaultRevision, + }, + { + name: "single-segment model id with revision", + input: "bert-base-uncased:v1.0", + wantRepoID: "bert-base-uncased", + wantRevision: "v1.0", + }, + { + name: "huggingfaceml:// prefix stripped", + input: HuggingFacePackagePrefix + "org/model:v2", + wantRepoID: "org/model", + wantRevision: "v2", + }, + { + name: "trailing colon treated as no revision — defaults to main", + input: "org/model:", + wantRepoID: "org/model", + wantRevision: DefaultRevision, + }, + { + // refs/pr/ is a valid Hugging Face revision (a PR ref) and contains '/'. + // It must still be split off as the revision, not glued onto RepoID. + name: "PR ref revision containing slashes", + input: "org/model:refs/pr/3", + wantRepoID: "org/model", + wantRevision: "refs/pr/3", + }, + { + name: "refs/convert/parquet revision containing slashes", + input: "bert-base-uncased:refs/convert/parquet", + wantRepoID: "bert-base-uncased", + wantRevision: "refs/convert/parquet", + }, + { + name: "leading colon only revision — missing repo id", + input: ":main", + wantErr: "missing repo id", + }, + { + name: "bare colon — missing repo id", + input: ":", + wantErr: "missing repo id", + }, + { + name: "empty input", + input: "", + wantErr: "hugging face model reference is empty", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + info, err := ParseModelReference(tt.input) + if tt.wantErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, tt.wantRepoID, info.RepoID, "RepoID mismatch") + assert.Equal(t, tt.wantRevision, info.Revision, "Revision mismatch") + }) + } +} + +func TestParseModelReferences(t *testing.T) { + t.Run("comma-separated with whitespace and trailing comma", func(t *testing.T) { + infos, err := ParseModelReferences(" org/a:main , org/b:v2 ,") + require.NoError(t, err) + require.Len(t, infos, 2) + assert.Equal(t, "org/a", infos[0].RepoID) + assert.Equal(t, "main", infos[0].Revision) + assert.Equal(t, "org/b", infos[1].RepoID) + assert.Equal(t, "v2", infos[1].Revision) + }) + t.Run("single value", func(t *testing.T) { + infos, err := ParseModelReferences("org/only") + require.NoError(t, err) + require.Len(t, infos, 1) + assert.Equal(t, "org/only", infos[0].RepoID) + assert.Equal(t, DefaultRevision, infos[0].Revision) + }) + t.Run("all empty entries error", func(t *testing.T) { + _, err := ParseModelReferences(" , , ") + require.Error(t, err) + assert.Contains(t, err.Error(), "empty") + }) +} + +func TestRepoFromHFEndpoint(t *testing.T) { + tests := []struct { + name string + endpoint string + wantRepo string + wantErr string + }{ + { + name: "standard endpoint", + endpoint: "https://z0gytst.jfrogdev.org/artifactory/api/huggingfaceml/my-hugging-face-repo", + wantRepo: "my-hugging-face-repo", + }, + { + name: "endpoint with trailing slash", + endpoint: "https://my.jfrog.io/artifactory/api/huggingfaceml/hf-repo/", + wantRepo: "hf-repo", + }, + { + name: "endpoint with extra path after repo", + endpoint: "https://my.jfrog.io/artifactory/api/huggingfaceml/hf-repo/api/models", + wantRepo: "hf-repo", + }, + { + name: "not set", + endpoint: "", + wantErr: "HF_ENDPOINT is not set", + }, + { + name: "missing marker", + endpoint: "https://my.jfrog.io/artifactory/api/npm/npm-repo", + wantErr: "does not contain", + }, + { + name: "no repo segment after marker", + endpoint: "https://my.jfrog.io/artifactory/api/huggingfaceml/", + wantErr: "no repository segment", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv(hfEndpointEnv, tt.endpoint) + repo, err := repoFromHFEndpoint() + if tt.wantErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, tt.wantRepo, repo) + }) + } +} + +func TestBuildDependencyTree(t *testing.T) { + tests := []struct { + name string + modelRef string + wantLeafId string + wantUniqueDep string + }{ + { + name: "blocked malicious model with sha", + modelRef: "mcpotato/42-eicar-street:8fb61c4d511e9aaff0ea55396a124aa292830efc", + wantLeafId: HuggingFacePackagePrefix + "mcpotato/42-eicar-street:8fb61c4d511e9aaff0ea55396a124aa292830efc", + wantUniqueDep: HuggingFacePackagePrefix + "mcpotato/42-eicar-street:8fb61c4d511e9aaff0ea55396a124aa292830efc", + }, + { + name: "model with branch revision", + modelRef: "bert-base-uncased:main", + wantLeafId: HuggingFacePackagePrefix + "bert-base-uncased:main", + wantUniqueDep: HuggingFacePackagePrefix + "bert-base-uncased:main", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Empty working dir so only the flag contributes (no stray discovery). + workDir := t.TempDir() + params := technologies.BuildInfoBomGeneratorParams{HuggingFaceModel: tt.modelRef, WorkingDirectory: workDir} + trees, uniqueDeps, warnings, err := BuildDependencyTree(params) + require.NoError(t, err) + require.Len(t, trees, 1, "expected exactly one dependency tree") + // Root node name is the working directory basename (not the old hardcoded constant). + assert.Equal(t, filepath.Base(workDir), trees[0].Id, "root node id mismatch") + require.Len(t, trees[0].Nodes, 1, "expected exactly one child node") + assert.Equal(t, tt.wantLeafId, trees[0].Nodes[0].Id, "leaf node id mismatch") + require.Len(t, uniqueDeps, 1, "expected exactly one unique dep") + assert.Equal(t, tt.wantUniqueDep, uniqueDeps[0], "unique dep mismatch") + assert.Empty(t, warnings, "flag-only mode should not produce warnings") + }) + } +} + +// TestBuildDependencyTree_MultiValueAndAdditive verifies that the flag accepts a +// comma-separated list and that flag mode audits exactly the named models — no more, +// no less (source scan is skipped in flag mode). +func TestBuildDependencyTree_MultiValueAndAdditive(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile( + dir+"/app.py", []byte(`from_pretrained("org/discovered-model", revision="v1")`), 0644)) + + params := technologies.BuildInfoBomGeneratorParams{ + // Two explicit flag models; source file has a third model that must NOT appear. + HuggingFaceModel: "org/flag-model-a:main, org/flag-model-b:v2", + WorkingDirectory: dir, + } + trees, uniqueDeps, warnings, err := BuildDependencyTree(params) + require.NoError(t, err) + require.Len(t, trees, 1) + // Only the two flag models — discovered-model from source must be absent. + assert.Len(t, uniqueDeps, 2) + assert.Contains(t, uniqueDeps, HuggingFacePackagePrefix+"org/flag-model-a:main") + assert.Contains(t, uniqueDeps, HuggingFacePackagePrefix+"org/flag-model-b:v2") + assert.NotContains(t, uniqueDeps, HuggingFacePackagePrefix+"org/discovered-model:v1") + assert.Empty(t, warnings, "flag mode skips source scan so no warnings expected") +} + +// TestBuildDependencyTree_AutoDiscovery verifies that an empty HuggingFaceModel flag +// triggers source scanning on the working directory instead of returning an error. +func TestBuildDependencyTree_AutoDiscovery(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile( + dir+"/app.py", []byte(`from_pretrained("org/discovered-model", revision="v1")`), 0644)) + + params := technologies.BuildInfoBomGeneratorParams{ + HuggingFaceModel: "", + WorkingDirectory: dir, + } + trees, uniqueDeps, _, err := BuildDependencyTree(params) + require.NoError(t, err) + require.Len(t, trees, 1) + require.Len(t, uniqueDeps, 1) + assert.Contains(t, uniqueDeps[0], "org/discovered-model") +} + +// TestBuildDependencyTree_DatasetsReportedNotAudited verifies that dataset +// references are surfaced as an advisory warning and never handed to the curation +// walker (Catalog does not score datasets). +func TestBuildDependencyTree_DatasetsReportedNotAudited(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(dir+"/data.py", []byte( + "from datasets import load_dataset\n"+ + "ds = load_dataset(\"squad\", revision=\"1.0.0\")\n"), 0644)) + + params := technologies.BuildInfoBomGeneratorParams{WorkingDirectory: dir} + trees, uniqueDeps, warnings, err := BuildDependencyTree(params) + require.NoError(t, err) + assert.Empty(t, trees, "datasets must not be audited") + assert.Empty(t, uniqueDeps) + require.Len(t, warnings, 1) + assert.Contains(t, warnings[0], "dataset reference(s) found but NOT audited") + assert.Contains(t, warnings[0], "squad") +} + +// TestBuildDependencyTree_UnresolvedWarnings verifies that dynamic references are +// returned as warnings (for the caller to surface after the curation table) rather +// than being logged during the BOM-build phase. +func TestBuildDependencyTree_UnresolvedWarnings(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(dir+"/dyn.py", []byte( + "runtime = AutoModel.from_pretrained(args.model_name)\n"), 0644)) + + params := technologies.BuildInfoBomGeneratorParams{WorkingDirectory: dir} + trees, uniqueDeps, warnings, err := BuildDependencyTree(params) + require.NoError(t, err) + assert.Empty(t, trees, "no statically-resolvable models expected") + assert.Empty(t, uniqueDeps) + require.Len(t, warnings, 1, "expected one consolidated unresolved-references warning") + assert.Contains(t, warnings[0], "could not be statically resolved") + assert.Contains(t, warnings[0], "non-literal repo_id") +} + +// TestBuildDependencyTree_SkippedFileWarning verifies that a file the scanner +// could not parse produces a user-visible "PARTIAL" warning, rather than being +// silently swallowed at debug level, while the rest of the scan still proceeds. +func TestBuildDependencyTree_SkippedFileWarning(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile( + dir+"/good.py", []byte(`from_pretrained("org/discovered-model", revision="v1")`), 0644)) + require.NoError(t, os.WriteFile(dir+"/bad.ipynb", []byte(`{not valid json`), 0644)) + + params := technologies.BuildInfoBomGeneratorParams{WorkingDirectory: dir} + trees, uniqueDeps, warnings, err := BuildDependencyTree(params) + require.NoError(t, err) + require.Len(t, trees, 1, "the unparsable notebook must not block discovery of the good file") + require.Len(t, uniqueDeps, 1) + assert.Contains(t, uniqueDeps[0], "org/discovered-model") + + require.Len(t, warnings, 1) + assert.Contains(t, warnings[0], "could not be scanned") + assert.Contains(t, warnings[0], "PARTIAL") + assert.Contains(t, warnings[0], "bad.ipynb") +} + +// TestDisambiguateRootNodeNames verifies that working dirs sharing a basename get +// distinct names, while unrelated dirs keep the plain basename. +func TestDisambiguateRootNodeNames(t *testing.T) { + tmp := t.TempDir() + unique := filepath.Join(tmp, "unique-project") + servicesA := filepath.Join(tmp, "services", "a", "model") + servicesB := filepath.Join(tmp, "services", "b", "model") + // deepA/deepB also collide on basename "model" but, unlike servicesA/servicesB, + // share the same parent name too ("dup"), so tier 2 can't disambiguate them either. + deepA := filepath.Join(tmp, "p", "dup", "model") + deepB := filepath.Join(tmp, "q", "dup", "model") + + names := DisambiguateRootNodeNames([]string{unique, servicesA, servicesB, deepA, deepB, servicesA}) + + assert.Equal(t, "unique-project", names[unique]) + assert.Equal(t, "a/model", names[servicesA]) + assert.Equal(t, "b/model", names[servicesB]) + assert.Equal(t, deepA, names[deepA]) + assert.Equal(t, deepB, names[deepB]) + + seen := map[string]bool{} + for _, n := range names { + require.False(t, seen[n], "name %q assigned to more than one distinct directory", n) + seen[n] = true + } +} + +func TestHostOf(t *testing.T) { + tests := []struct { + name string + rawURL string + want string + wantErr bool + }{ + {name: "https with no explicit port", rawURL: "https://example.com/artifactory", want: "example.com"}, + {name: "https with explicit default port 443", rawURL: "https://example.com:443/artifactory", want: "example.com"}, + {name: "http with explicit default port 80", rawURL: "http://example.com:80/artifactory", want: "example.com"}, + {name: "https with non-default port kept", rawURL: "https://example.com:8443/artifactory", want: "example.com:8443"}, + {name: "http with non-default port kept", rawURL: "http://example.com:8081/artifactory", want: "example.com:8081"}, + {name: "no host", rawURL: "/just/a/path", wantErr: true}, + {name: "invalid URL", rawURL: "://bad-url", wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := hostOf(tt.rawURL) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +// TestHostOf_MissingSchemeGetsActionableError verifies that a schemeless +// HF_ENDPOINT/Artifactory URL (a common copy-paste mistake) produces a message +// pointing at the missing scheme, not a bare "no host in URL". +func TestHostOf_MissingSchemeGetsActionableError(t *testing.T) { + _, err := hostOf("my.jfrog.io/artifactory/api/huggingfaceml/hf-repo") + require.Error(t, err) + assert.Contains(t, err.Error(), "missing a scheme") +} + +func TestValidateHFEndpointHost(t *testing.T) { + tests := []struct { + name string + hfEndpoint string + artifactoryUrl string + wantErr bool + }{ + { + name: "same host, both without explicit port", + hfEndpoint: "https://my.jfrog.io/artifactory/api/huggingfaceml/hf-repo", + artifactoryUrl: "https://my.jfrog.io/artifactory", + wantErr: false, + }, + { + name: "same host, HF_ENDPOINT has explicit default https port, server does not", + hfEndpoint: "https://my.jfrog.io:443/artifactory/api/huggingfaceml/hf-repo", + artifactoryUrl: "https://my.jfrog.io/artifactory", + wantErr: false, + }, + { + name: "same host, server has explicit default https port, HF_ENDPOINT does not", + hfEndpoint: "https://my.jfrog.io/artifactory/api/huggingfaceml/hf-repo", + artifactoryUrl: "https://my.jfrog.io:443/artifactory", + wantErr: false, + }, + { + name: "different hosts", + hfEndpoint: "https://other.jfrog.io/artifactory/api/huggingfaceml/hf-repo", + artifactoryUrl: "https://my.jfrog.io/artifactory", + wantErr: true, + }, + { + name: "same host, genuinely different non-default ports", + hfEndpoint: "https://my.jfrog.io:8443/artifactory/api/huggingfaceml/hf-repo", + artifactoryUrl: "https://my.jfrog.io/artifactory", + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv(hfEndpointEnv, tt.hfEndpoint) + serverDetails := &config.ServerDetails{ArtifactoryUrl: tt.artifactoryUrl} + err := validateHFEndpointHost(serverDetails) + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + }) + } +} diff --git a/utils/formats/summary.go b/utils/formats/summary.go index 9dfbace04..c8d80dfba 100644 --- a/utils/formats/summary.go +++ b/utils/formats/summary.go @@ -63,6 +63,8 @@ type CuratedPackages struct { // IsPartial is true when the dependency tree could not be fully resolved (e.g. CVS pip fallback). // PackageCount reflects only recovered blocked packages, not the full tree size. IsPartial bool `json:"partial,omitempty"` + // PartialReason explains why IsPartial is true, e.g. "cvs_fallback" or "hf_unresolved". + PartialReason string `json:"partial_reason,omitempty"` } type BlockedPackages struct { diff --git a/utils/techutils/techutils.go b/utils/techutils/techutils.go index 66a553557..f20b3e89d 100644 --- a/utils/techutils/techutils.go +++ b/utils/techutils/techutils.go @@ -59,14 +59,15 @@ const ( Swift Technology = "swift" Gem Technology = "ruby" // Not Supported by build-info BOM generator - Docker Technology = "docker" - Oci Technology = "oci" - Rpm Technology = "rpm" - Debian Technology = "deb" - Composer Technology = "composer" - Alpine Technology = "alpine" - Cpp Technology = "cpp" - NoTech Technology = "" + Docker Technology = "docker" + HuggingFaceML Technology = "huggingfaceml" + Oci Technology = "oci" + Rpm Technology = "rpm" + Debian Technology = "deb" + Composer Technology = "composer" + Alpine Technology = "alpine" + Cpp Technology = "cpp" + NoTech Technology = "" ) // Alternative package types for some technologies @@ -86,6 +87,7 @@ var AllTechnologiesStrings = []string{ Nuget.String(), Dotnet.String(), Docker.String(), + HuggingFaceML.String(), Oci.String(), Conan.String(), Cocoapods.String(), @@ -299,6 +301,10 @@ var technologiesData = map[Technology]TechData{ formal: "Docker", projectType: project.Docker, }, + HuggingFaceML: { + formal: "Hugging Face", + xrayPackageType: "huggingfaceml", + }, Oci: {}, Rpm: {formal: "RPM"}, Debian: {formal: "Debian"},