all: switch to the new shared vuln schema

Change-Id: Ibbbf153cc8078884bf9ac5a3a8b01a75894abb17
Reviewed-on: https://team-review.git.corp.google.com/c/golang/vulndb/+/1055915
Reviewed-by: Roland Shoemaker <bracewell@google.com>

Author: rolandshoemaker

client/client.go

@@ -141,7 +141,7 @@ func (hs *httpSource) Get(packages []string) ([]*osv.Entry, error) {
 			} else if len(cached) != 0 {
 				var stale bool
 				for _, c := range cached {
-					if c.LastModified.Before(lastModified) {
+					if c.Modified.Before(lastModified) {
 						stale = true
 						break
 					}

client/client_test.go

@@ -54,9 +54,9 @@ func cachedTestVuln2(dbName string) func() Cache {
 	return func() Cache {
 		c := &fsCache{}
 		e := &osv.Entry{
-			ID:           "ID2",
-			Summary:      "cached",
-			LastModified: time.Now(),
+			ID:       "ID2",
+			Details:  "cached",
+			Modified: time.Now(),
 		}
 		c.WriteEntries(dbName, "golang.org/example/two", []*osv.Entry{e})
 		return c
@@ -138,8 +138,8 @@ func TestClient(t *testing.T) {
 		}
 
 		for _, v := range vulns {
-			if s, ok := test.summaries[v.ID]; !ok || v.Summary != s {
-				t.Errorf("want '%s' summary for vuln with id %v in %s; got '%s'", s, v.ID, test.name, v.Summary)
+			if s, ok := test.summaries[v.ID]; !ok || v.Details != s {
+				t.Errorf("want '%s' summary for vuln with id %v in %s; got '%s'", s, v.ID, test.name, v.Details)
 			}
 		}
 	}

cmd/gendb/main.go

@@ -91,8 +91,8 @@ func main() {
 			fail(fmt.Sprintf("failed to write %q: %s", outPath+".json", err))
 		}
 		for _, v := range vulns {
-			if v.LastModified.After(index[path]) {
-				index[path] = v.LastModified
+			if v.Modified.After(index[path]) {
+				index[path] = v.Modified
 			}
 		}
 	}

osv/json.go

@@ -18,28 +18,10 @@ import (
 // vulndb implementatiion detail.
 type DBIndex map[string]time.Time
 
-type Severity int
+type AffectsRangeType int
 
 const (
-	SevNone Severity = iota
-	SevLow
-	SevMedium
-	SevHigh
-	SevCritical
-)
-
-var strToSev = map[string]Severity{
-	// "": SevNone,
-	"low":      SevLow,
-	"medium":   SevMedium,
-	"high":     SevHigh,
-	"critical": SevCritical,
-}
-
-type Type int
-
-const (
-	TypeUnspecified Type = iota
+	TypeUnspecified AffectsRangeType = iota
 	TypeGit
 	TypeSemver
 )
@@ -54,7 +36,7 @@ type Package struct {
 }
 
 type AffectsRange struct {
-	Type       Type
+	Type       AffectsRangeType
 	Introduced string
 	Fixed      string
 }
@@ -113,59 +95,66 @@ type GoSpecific struct {
 	URL     string
 }
 
+type Reference struct {
+	Type string
+	URL  string
+}
+
 // Entry represents a OSV style JSON vulnerability database
 // entry
 type Entry struct {
-	ID                string
-	Package           Package
-	Summary           string
-	Details           string
-	Severity          Severity
-	Affects           Affects
-	ReferenceURLs     []string   `json:"reference_urls,omitempty"`
-	Aliases           []string   `json:",omitempty"`
-	EcosystemSpecific GoSpecific `json:"ecosystem_specific,omitempty"`
-	LastModified      time.Time  `json:"last_modified"`
+	ID         string
+	Published  time.Time
+	Modified   time.Time
+	Withdrawn  *time.Time
+	Aliases    []string `json:",omitempty"`
+	Package    Package
+	Details    string
+	Affects    Affects
+	References []Reference `json:",omitempty"`
+	Extra      struct {
+		Go GoSpecific
+	}
 }
 
 func Generate(id string, url string, r report.Report) []Entry {
 	importPath := r.Module
 	if r.Package != "" {
 		importPath = r.Package
 	}
+	lastModified := r.Published
+	if r.LastModified != nil {
+		lastModified = *r.LastModified
+	}
 	entry := Entry{
-		ID: id,
+		ID:        id,
+		Published: r.Published,
+		Modified:  lastModified,
+		Withdrawn: r.Withdrawn,
 		Package: Package{
 			Name:      importPath,
 			Ecosystem: GoEcosystem,
 		},
-		Summary:      "", // TODO: think if we want to populate this in reports
-		Details:      r.Description,
-		Affects:      generateAffects(r.Versions),
-		LastModified: time.Now(),
-		EcosystemSpecific: GoSpecific{
-			Symbols: r.Symbols,
-			GOOS:    r.OS,
-			GOARCH:  r.Arch,
-			URL:     url,
+		Details: r.Description,
+		Affects: generateAffects(r.Versions),
+		Extra: struct{ Go GoSpecific }{
+			Go: GoSpecific{
+				Symbols: r.Symbols,
+				GOOS:    r.OS,
+				GOARCH:  r.Arch,
+				URL:     url,
+			},
 		},
 	}
 
-	if r.Severity != "" {
-		entry.Severity = strToSev[r.Severity]
-	} else {
-		// Default to medium or none?
-		entry.Severity = SevMedium
-	}
-
 	if r.Links.PR != "" {
-		entry.ReferenceURLs = append(entry.ReferenceURLs, r.Links.PR)
+		entry.References = append(entry.References, Reference{Type: "code review", URL: r.Links.PR})
 	}
 	if r.Links.Commit != "" {
-		entry.ReferenceURLs = append(entry.ReferenceURLs, r.Links.Commit)
+		entry.References = append(entry.References, Reference{Type: "fix", URL: r.Links.Commit})
 	}
-	if r.Links.Context != nil {
-		entry.ReferenceURLs = append(entry.ReferenceURLs, r.Links.Context...)
+	for _, link := range r.Links.Context {
+		entry.References = append(entry.References, Reference{Type: "misc", URL: link})
 	}
 
 	if r.CVE != "" {
@@ -174,15 +163,15 @@ func Generate(id string, url string, r report.Report) []Entry {
 
 	entries := []Entry{entry}
 
-	// It would be better if this was just a recursive thing probably
+	// It would be better if this was just a recursive thing maybe?
 	for _, additional := range r.AdditionalPackages {
 		entryCopy := entry
 		additionalImportPath := additional.Module
 		if additional.Package != "" {
 			additionalImportPath = additional.Package
 		}
 		entryCopy.Package.Name = additionalImportPath
-		entryCopy.EcosystemSpecific.Symbols = additional.Symbols
+		entryCopy.Extra.Go.Symbols = additional.Symbols
 		entryCopy.Affects = generateAffects(additional.Versions)
 
 		entries = append(entries, entryCopy)

osv/json_test.go

@@ -38,7 +38,6 @@ func TestGenerate(t *testing.T) {
 			{Introduced: "v2.5.0"},
 		},
 		Description: "It's a real bad one, I'll tell you that",
-		Severity:    "medium",
 		CVE:         "CVE-0000-0000",
 		Credit:      "ignored",
 		Symbols:     []string{"A", "B.b"},
@@ -62,8 +61,7 @@ func TestGenerate(t *testing.T) {
 				Name:      "example.com/vulnerable/v2",
 				Ecosystem: "go",
 			},
-			Details:  "It's a real bad one, I'll tell you that",
-			Severity: 2,
+			Details: "It's a real bad one, I'll tell you that",
 			Affects: Affects{
 				Ranges: []AffectsRange{
 					{
@@ -81,18 +79,20 @@ func TestGenerate(t *testing.T) {
 					},
 				},
 			},
-			ReferenceURLs: []string{
-				"pr",
-				"commit",
-				"issue-a",
-				"issue-b",
+			References: []Reference{
+				Reference{Type: "code review", URL: "pr"},
+				Reference{Type: "fix", URL: "commit"},
+				Reference{Type: "misc", URL: "issue-a"},
+				Reference{Type: "misc", URL: "issue-b"},
 			},
 			Aliases: []string{"CVE-0000-0000"},
-			EcosystemSpecific: GoSpecific{
-				Symbols: []string{"A", "B.b"},
-				GOOS:    []string{"windows"},
-				GOARCH:  []string{"arm64"},
-				URL:     "https://vulns.golang.org/GO-1991-0001.html",
+			Extra: struct{ Go GoSpecific }{
+				Go: GoSpecific{
+					Symbols: []string{"A", "B.b"},
+					GOOS:    []string{"windows"},
+					GOARCH:  []string{"arm64"},
+					URL:     "https://vulns.golang.org/GO-1991-0001.html",
+				},
 			},
 		},
 		{
@@ -102,8 +102,7 @@ func TestGenerate(t *testing.T) {
 				Name:      "vanity.host/vulnerable/package",
 				Ecosystem: "go",
 			},
-			Details:  "It's a real bad one, I'll tell you that",
-			Severity: 2,
+			Details: "It's a real bad one, I'll tell you that",
 			Affects: Affects{
 				Ranges: []AffectsRange{
 					{
@@ -121,18 +120,20 @@ func TestGenerate(t *testing.T) {
 					},
 				},
 			},
-			ReferenceURLs: []string{
-				"pr",
-				"commit",
-				"issue-a",
-				"issue-b",
+			References: []Reference{
+				Reference{Type: "code review", URL: "pr"},
+				Reference{Type: "fix", URL: "commit"},
+				Reference{Type: "misc", URL: "issue-a"},
+				Reference{Type: "misc", URL: "issue-b"},
 			},
 			Aliases: []string{"CVE-0000-0000"},
-			EcosystemSpecific: GoSpecific{
-				Symbols: []string{"b", "A.b"},
-				GOOS:    []string{"windows"},
-				GOARCH:  []string{"arm64"},
-				URL:     "https://vulns.golang.org/GO-1991-0001.html",
+			Extra: struct{ Go GoSpecific }{
+				Go: GoSpecific{
+					Symbols: []string{"b", "A.b"},
+					GOOS:    []string{"windows"},
+					GOARCH:  []string{"arm64"},
+					URL:     "https://vulns.golang.org/GO-1991-0001.html",
+				},
 			},
 		},
 	}

report/lint.go

@@ -126,6 +126,11 @@ func checkModVersions(path string, vr []VersionRange) error {
 
 var cveRegex = regexp.MustCompile(`^CVE-\d{4}-\d{4,}$`)
 
+// Lint checks the content of a Report.
+// TODO: instead of returning a single error we may want to return a slice, so that
+// we aren't fixing one thing at a time. Similarly it might make sense to include
+// warnings or informational things alongside errors, especially during for use
+// during the triage process.
 func (vuln *Report) Lint() error {
 	var importPath string
 	if !vuln.Stdlib {
@@ -184,17 +189,12 @@ func (vuln *Report) Lint() error {
 		return errors.New("missing description")
 	}
 
-	sevs := map[string]bool{
-		"low":      true,
-		"medium":   true,
-		"high":     true,
-		"critical": true,
+	if vuln.Published.IsZero() {
+		return errors.New("missing published")
 	}
-	// Could also just default to medium if not provided?
-	// Need to document what the default case is and what factors lower
-	// or raise the sev
-	if vuln.Severity != "" && !sevs[vuln.Severity] {
-		return fmt.Errorf("unknown severity %q", vuln.Severity)
+
+	if vuln.LastModified != nil && vuln.LastModified.Before(vuln.Published) {
+		return errors.New("last_modified is before published")
 	}
 
 	if vuln.CVE != "" && vuln.CVEMetadata != nil && vuln.CVEMetadata.ID != "" {
@@ -203,7 +203,16 @@ func (vuln *Report) Lint() error {
 	}
 
 	if vuln.CVE != "" && !cveRegex.MatchString(vuln.CVE) {
-		return fmt.Errorf("malformed CVE number: %s", vuln.CVE)
+		return fmt.Errorf("malformed cve: %s", vuln.CVE)
+	}
+
+	if vuln.CVEMetadata != nil {
+		if vuln.CVEMetadata.ID == "" {
+			return errors.New("cve_metadata.id is required")
+		}
+		if !cveRegex.MatchString(vuln.CVEMetadata.ID) {
+			return fmt.Errorf("malformed cve_metadata.id: %s", vuln.CVEMetadata.ID)
+		}
 	}
 
 	return nil

report/report.go

@@ -35,8 +35,8 @@ type Report struct {
 	Versions     []VersionRange
 	Description  string
 	Published    time.Time
-	LastModified time.Time `toml:"last_modified"`
-	Severity     string
+	LastModified *time.Time `toml:"last_modified"`
+	Withdrawn    *time.Time
 	CVE          string
 	Credit       string
 	Symbols      []string

reports/GO-2020-0001.toml

@@ -12,6 +12,8 @@ credit = "@thinkerou <thinkerou@gmail.com>"
 # Test symbol inclusion by making a gin handler without Default or Logger.
 symbols = ["defaultLogFormatter"]
 
+published = "2021-04-14T12:00:00Z"
+
 [[versions]]
 # v1.5.1 doesn't exist? not sure how `go mod` is picking this pseudoversion
 fixed = "v1.6.0"
@@ -21,7 +23,7 @@ pr = "https://github.com/gin-gonic/gin/pull/2237"
 commit = "https://github.com/gin-gonic/gin/commit/a71af9c144f9579f6dbe945341c1df37aaf09c0d"
 
 [cve_metadata]
-id = "CVE-XXXX-0001"
+id = "CVE-9999-0001"
 description = """
 Unsanitized input in the default logger in github.com/gin-gonic/gin before v1.6.0
 allows remote attackers to inject arbitary log lines.

reports/GO-2020-0002.toml

@@ -10,6 +10,8 @@ leading to crashes and potentially code execution through a use-after-free.
 
 credit = "Ulrich Obergfell <uobergfe@redhat.com>"
 
+published = "2021-04-14T12:00:00Z"
+
 [[versions]]
 fixed = "v0.1.1"
 

reports/GO-2020-0003.toml

@@ -9,6 +9,8 @@ crash by manipulating the request query.
 
 credit = "@SYM01"
 
+published = "2021-04-14T12:00:00Z"
+
 [[versions]]
 fixed = "v1.0.0"
 
@@ -18,7 +20,7 @@ pr = "https://github.com/revel/revel/pull/1427"
 context = ["https://github.com/revel/revel/issues/1424"]
 
 [cve_metadata]
-id = "CVE-XXXX-0002"
+id = "CVE-9999-0002"
 description = """
 Unsanitized input in the query parser in github.com/revel/revel before v1.0.0
 allows remote attackers to cause resource exhaustion via memory allocation.