fix(vulnfeeds): Migrate to latest OSV schema bindings (#4241)

Unblocks #4190

This PR updates `vulnfeeds` to use the latest version of
`github.com/ossf/osv-schema/bindings/go`.

To better handle ecosystem and database specific fields, a new function
`NewStructpbFromMap` is added to `utility` to convert `map[string]any`
to `structpb.Struct` so we don't need to worry about conversion every
time.

Author: cuixq

vulnfeeds/cmd/alpine/main.go

@@ -21,6 +21,7 @@ import (
 	"github.com/google/osv/vulnfeeds/utility/logger"
 	"github.com/google/osv/vulnfeeds/vulns"
 	"github.com/ossf/osv-schema/bindings/go/osvschema"
+	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
 const (
@@ -56,10 +57,10 @@ func main() {
 	vulnerabilities := make([]*osvschema.Vulnerability, 0, len(osvVulnerabilities))
 	for _, v := range osvVulnerabilities {
 		if len(v.Affected) == 0 {
-			logger.Warn(fmt.Sprintf("Skipping %s as no affected versions found.", v.ID), slog.String("id", v.ID))
+			logger.Warn(fmt.Sprintf("Skipping %s as no affected versions found.", v.Id), slog.String("id", v.Id))
 			continue
 		}
-		vulnerabilities = append(vulnerabilities, &v.Vulnerability)
+		vulnerabilities = append(vulnerabilities, v.Vulnerability)
 	}
 
 	ctx := context.Background()
@@ -171,15 +172,15 @@ func generateAlpineOSV(allAlpineSecDb map[string][]VersionAndPkg, allCVEs map[cv
 		}
 
 		v := &vulns.Vulnerability{
-			Vulnerability: osvschema.Vulnerability{
-				ID:        "ALPINE-" + cveID,
+			Vulnerability: &osvschema.Vulnerability{
+				Id:        "ALPINE-" + cveID,
 				Upstream:  []string{cveID},
-				Published: published,
+				Published: timestamppb.New(published),
 				Details:   details,
-				References: []osvschema.Reference{
+				References: []*osvschema.Reference{
 					{
-						Type: "ADVISORY",
-						URL:  "https://security.alpinelinux.org/vuln/" + cveID,
+						Type: osvschema.Reference_ADVISORY,
+						Url:  "https://security.alpinelinux.org/vuln/" + cveID,
 					},
 				},
 			},
@@ -198,7 +199,7 @@ func generateAlpineOSV(allAlpineSecDb map[string][]VersionAndPkg, allCVEs map[cv
 		}
 
 		if len(v.Affected) == 0 {
-			logger.Warn(fmt.Sprintf("Skipping %s as no affected versions found.", v.ID), slog.String("cveID", cveID))
+			logger.Warn(fmt.Sprintf("Skipping %s as no affected versions found.", v.Id), slog.String("cveID", cveID))
 			continue
 		}
 		if cve.CVE.Metrics != nil {

vulnfeeds/cmd/combine-to-osv/main.go

@@ -3,7 +3,6 @@ package main
 
 import (
 	"context"
-	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
@@ -21,6 +20,7 @@ import (
 	"github.com/google/osv/vulnfeeds/utility/logger"
 	"github.com/ossf/osv-schema/bindings/go/osvschema"
 	"google.golang.org/api/iterator"
+	"google.golang.org/protobuf/encoding/protojson"
 )
 
 const (
@@ -87,7 +87,7 @@ func main() {
 
 	vulnerabilities := make([]*osvschema.Vulnerability, 0, len(combinedData))
 	for _, v := range combinedData {
-		vulnerabilities = append(vulnerabilities, &v)
+		vulnerabilities = append(vulnerabilities, v)
 	}
 
 	upload.Upload(ctx, "OSV files", *uploadToGCS, *outputBucketName, *overridesBucketName, *numWorkers, *osvOutputPath, vulnerabilities, *syncDeletions)
@@ -137,8 +137,8 @@ func listBucketObjects(bucketName string, prefix string) ([]string, error) {
 // The function returns a map of CVE IDs to their corresponding Vulnerability objects.
 // Files that are not ".json" files, directories, or files ending in ".metrics.json" are skipped.
 // The function will log warnings for files that fail to open or decode, and will terminate if it fails to walk the directory.
-func loadOSV(osvPath string) map[cves.CVEID]osvschema.Vulnerability {
-	allVulns := make(map[cves.CVEID]osvschema.Vulnerability)
+func loadOSV(osvPath string) map[cves.CVEID]*osvschema.Vulnerability {
+	allVulns := make(map[cves.CVEID]*osvschema.Vulnerability)
 	logger.Info("Loading OSV records", slog.String("path", osvPath))
 	err := filepath.WalkDir(osvPath, func(path string, d fs.DirEntry, err error) error {
 		if err != nil {
@@ -148,20 +148,19 @@ func loadOSV(osvPath string) map[cves.CVEID]osvschema.Vulnerability {
 			return nil
 		}
 
-		file, err := os.Open(path)
+		file, err := os.ReadFile(path)
 		if err != nil {
 			logger.Warn("Failed to open OSV JSON file", slog.String("path", path), slog.Any("err", err))
 			return nil
 		}
 
 		var vuln osvschema.Vulnerability
-		decodeErr := json.NewDecoder(file).Decode(&vuln)
-		file.Close()
+		decodeErr := protojson.Unmarshal(file, &vuln)
 		if decodeErr != nil {
 			logger.Error("Failed to decode, skipping", slog.String("file", path), slog.Any("err", decodeErr))
 			return nil
 		}
-		allVulns[cves.CVEID(vuln.ID)] = vuln
+		allVulns[cves.CVEID(vuln.GetId())] = &vuln
 
 		return nil
 	})
@@ -174,12 +173,12 @@ func loadOSV(osvPath string) map[cves.CVEID]osvschema.Vulnerability {
 }
 
 // combineIntoOSV creates OSV entry by combining loaded CVEs from NVD and PackageInfo information from security advisories.
-func combineIntoOSV(cve5osv map[cves.CVEID]osvschema.Vulnerability, nvdosv map[cves.CVEID]osvschema.Vulnerability, mandatoryCVEIDs []string) map[cves.CVEID]osvschema.Vulnerability {
-	osvRecords := make(map[cves.CVEID]osvschema.Vulnerability)
+func combineIntoOSV(cve5osv map[cves.CVEID]*osvschema.Vulnerability, nvdosv map[cves.CVEID]*osvschema.Vulnerability, mandatoryCVEIDs []string) map[cves.CVEID]*osvschema.Vulnerability {
+	osvRecords := make(map[cves.CVEID]*osvschema.Vulnerability)
 
 	// Iterate through CVEs from security advisories (cve5) as the base
 	for cveID, cve5 := range cve5osv {
-		var baseOSV osvschema.Vulnerability
+		var baseOSV *osvschema.Vulnerability
 		nvd, ok := nvdosv[cveID]
 
 		if ok {
@@ -190,7 +189,7 @@ func combineIntoOSV(cve5osv map[cves.CVEID]osvschema.Vulnerability, nvdosv map[c
 			baseOSV = cve5
 		}
 
-		if len(baseOSV.Affected) == 0 {
+		if len(baseOSV.GetAffected()) == 0 {
 			// check if part exists.
 			if !slices.Contains(mandatoryCVEIDs, string(cveID)) {
 				continue
@@ -201,7 +200,7 @@ func combineIntoOSV(cve5osv map[cves.CVEID]osvschema.Vulnerability, nvdosv map[c
 
 	// Add any remaining CVEs from NVD that were not in the advisory data.
 	for cveID, nvd := range nvdosv {
-		if len(nvd.Affected) == 0 {
+		if len(nvd.GetAffected()) == 0 {
 			continue
 		}
 		osvRecords[cveID] = nvd
@@ -211,40 +210,40 @@ func combineIntoOSV(cve5osv map[cves.CVEID]osvschema.Vulnerability, nvdosv map[c
 }
 
 // combineTwoOSVRecords takes two osv records and combines them into one
-func combineTwoOSVRecords(cve5 osvschema.Vulnerability, nvd osvschema.Vulnerability) osvschema.Vulnerability {
+func combineTwoOSVRecords(cve5 *osvschema.Vulnerability, nvd *osvschema.Vulnerability) *osvschema.Vulnerability {
 	baseOSV := cve5
-	combinedAffected := pickAffectedInformation(cve5.Affected, nvd.Affected)
+	combinedAffected := pickAffectedInformation(cve5.GetAffected(), nvd.GetAffected())
 
 	baseOSV.Affected = combinedAffected
 	// Merge references, ensuring no duplicates.
 	refMap := make(map[string]bool)
-	for _, r := range baseOSV.References {
-		refMap[r.URL] = true
+	for _, r := range baseOSV.GetReferences() {
+		refMap[r.GetUrl()] = true
 	}
-	for _, r := range nvd.References {
-		if !refMap[r.URL] {
+	for _, r := range nvd.GetReferences() {
+		if !refMap[r.GetUrl()] {
 			baseOSV.References = append(baseOSV.References, r)
-			refMap[r.URL] = true
+			refMap[r.GetUrl()] = true
 		}
 	}
 
 	// Merge timestamps: latest modified, earliest published.
-	cve5Modified := baseOSV.Modified
-	if nvd.Modified.After(cve5Modified) {
-		baseOSV.Modified = nvd.Modified
+	cve5Modified := baseOSV.GetModified()
+	if nvd.GetModified().AsTime().After(cve5Modified.AsTime()) {
+		baseOSV.Modified = nvd.GetModified()
 	}
 
-	cve5Published := baseOSV.Published
-	if nvd.Published.Before(cve5Published) {
-		baseOSV.Published = nvd.Published
+	cve5Published := baseOSV.GetPublished()
+	if nvd.GetPublished().AsTime().Before(cve5Published.AsTime()) {
+		baseOSV.Published = nvd.GetPublished()
 	}
 
 	// Merge aliases, ensuring no duplicates.
 	aliasMap := make(map[string]bool)
-	for _, alias := range baseOSV.Aliases {
+	for _, alias := range baseOSV.GetAliases() {
 		aliasMap[alias] = true
 	}
-	for _, alias := range nvd.Aliases {
+	for _, alias := range nvd.GetAliases() {
 		if !aliasMap[alias] {
 			baseOSV.Aliases = append(baseOSV.Aliases, alias)
 			aliasMap[alias] = true
@@ -259,7 +258,7 @@ func combineTwoOSVRecords(cve5 osvschema.Vulnerability, nvd osvschema.Vulnerabil
 // If a match is found, it merges the version range information, preferring the entry
 // with more ranges. Unmatched nvdAffected packages are appended.
 // It returns a new slice and does not modify cve5Affected in place.
-func pickAffectedInformation(cve5Affected []osvschema.Affected, nvdAffected []osvschema.Affected) []osvschema.Affected {
+func pickAffectedInformation(cve5Affected []*osvschema.Affected, nvdAffected []*osvschema.Affected) []*osvschema.Affected {
 	if len(nvdAffected) == 0 {
 		return cve5Affected
 	}
@@ -268,31 +267,31 @@ func pickAffectedInformation(cve5Affected []osvschema.Affected, nvdAffected []os
 		return nvdAffected
 	}
 
-	nvdRepoMap := make(map[string][]osvschema.Range)
+	nvdRepoMap := make(map[string][]*osvschema.Range)
 	for _, affected := range nvdAffected {
-		for _, r := range affected.Ranges {
-			if r.Repo != "" {
-				repo := strings.ToLower(r.Repo)
+		for _, r := range affected.GetRanges() {
+			if r.GetRepo() != "" {
+				repo := strings.ToLower(r.GetRepo())
 				nvdRepoMap[repo] = append(nvdRepoMap[repo], r)
 			}
 		}
 	}
 
-	cve5RepoMap := make(map[string][]osvschema.Range)
+	cve5RepoMap := make(map[string][]*osvschema.Range)
 	for _, affected := range cve5Affected {
-		for _, r := range affected.Ranges {
-			if r.Repo != "" {
-				repo := strings.ToLower(r.Repo)
+		for _, r := range affected.GetRanges() {
+			if r.GetRepo() != "" {
+				repo := strings.ToLower(r.GetRepo())
 				cve5RepoMap[repo] = append(cve5RepoMap[repo], r)
 			}
 		}
 	}
 
-	newRepoAffectedMap := make(map[string]osvschema.Affected)
+	newRepoAffectedMap := make(map[string]*osvschema.Affected)
 
 	for repo, cveRanges := range cve5RepoMap {
 		if nvdRanges, ok := nvdRepoMap[repo]; ok {
-			var newAffectedRanges []osvschema.Range
+			var newAffectedRanges []*osvschema.Range
 
 			// Found a match. If NVD has more ranges, use its ranges.
 			if len(nvdRanges) > len(cveRanges) {
@@ -301,8 +300,8 @@ func pickAffectedInformation(cve5Affected []osvschema.Affected, nvdAffected []os
 			} else if len(nvdRanges) < len(cveRanges) {
 				newAffectedRanges = cveRanges
 			} else if len(cveRanges) == 1 && len(nvdRanges) == 1 {
-				c5Intro, c5Fixed := getRangeBoundaryVersions(cveRanges[0].Events)
-				nvdIntro, nvdFixed := getRangeBoundaryVersions(nvdRanges[0].Events)
+				c5Intro, c5Fixed := getRangeBoundaryVersions(cveRanges[0].GetEvents())
+				nvdIntro, nvdFixed := getRangeBoundaryVersions(nvdRanges[0].GetEvents())
 
 				// Prefer cve5 data, but use nvd data if cve5 data is missing.
 				if c5Intro == "" {
@@ -315,30 +314,30 @@ func pickAffectedInformation(cve5Affected []osvschema.Affected, nvdAffected []os
 				if c5Intro != "" || c5Fixed != "" {
 					newRange := cves.BuildVersionRange(c5Intro, "", c5Fixed)
 					newRange.Repo = repo
-					newRange.Type = osvschema.RangeGit // Preserve the repo
+					newRange.Type = osvschema.Range_GIT // Preserve the repo
 					newAffectedRanges = append(newAffectedRanges, newRange)
 				}
 			}
 			// Remove from map so we know which NVD packages are left.
 			delete(nvdRepoMap, repo)
-			newRepoAffectedMap[repo] = osvschema.Affected{
+			newRepoAffectedMap[repo] = &osvschema.Affected{
 				Ranges: newAffectedRanges,
 			}
 		} else {
-			newRepoAffectedMap[repo] = osvschema.Affected{
+			newRepoAffectedMap[repo] = &osvschema.Affected{
 				Ranges: cveRanges,
 			}
 		}
 	}
 
 	// Add remaining NVD packages that were not in cve5.
 	for repo, nvdRange := range nvdRepoMap {
-		newRepoAffectedMap[repo] = osvschema.Affected{
+		newRepoAffectedMap[repo] = &osvschema.Affected{
 			Ranges: nvdRange,
 		}
 	}
 
-	var combinedAffected []osvschema.Affected //nolint:prealloc
+	var combinedAffected []*osvschema.Affected //nolint:prealloc
 	for _, aff := range newRepoAffectedMap {
 		combinedAffected = append(combinedAffected, aff)
 	}
@@ -348,13 +347,13 @@ func pickAffectedInformation(cve5Affected []osvschema.Affected, nvdAffected []os
 
 // getRangeBoundaryVersions extracts the introduced and fixed versions from a slice of OSV events.
 // It iterates through the events and returns the last non-empty "introduced" and "fixed" versions found.
-func getRangeBoundaryVersions(events []osvschema.Event) (introduced, fixed string) {
+func getRangeBoundaryVersions(events []*osvschema.Event) (introduced, fixed string) {
 	for _, e := range events {
-		if e.Introduced != "0" && e.Introduced != "" {
-			introduced = e.Introduced
+		if e.GetIntroduced() != "0" && e.GetIntroduced() != "" {
+			introduced = e.GetIntroduced()
 		}
-		if e.Fixed != "" {
-			fixed = e.Fixed
+		if e.GetFixed() != "" {
+			fixed = e.GetFixed()
 		}
 	}