all: use consistent index type across packages

All reports still need to have their published fields set.

Change-Id: I64feda32742bb5f85e310211f8da270e4346ad6b
Reviewed-on: https://team-review.git.corp.google.com/c/golang/vulndb/+/1036000
Reviewed-by: Roland Shoemaker <bracewell@google.com>

Author: rolandshoemaker

client/cache.go

@@ -25,7 +25,7 @@ import (
 // $GOPATH/pkg/mod/cache/download/vulndb/{db hostname}/indexes/index.json
 //   {
 //       Retrieved time.Time
-//       Index map[string]time.Time
+//       Index osv.DBIndex
 //   }
 //
 // Each package also has a JSON file which contains the array of vulnerability
@@ -35,8 +35,8 @@ import (
 //   []*osv.Entry
 
 type Cache interface {
-	ReadIndex(string) (map[string]time.Time, time.Time, error)
-	WriteIndex(string, map[string]time.Time, time.Time) error
+	ReadIndex(string) (osv.DBIndex, time.Time, error)
+	WriteIndex(string, osv.DBIndex, time.Time) error
 	ReadEntries(string, string) ([]*osv.Entry, error)
 	WriteEntries(string, string, []*osv.Entry) error
 }
@@ -54,10 +54,10 @@ var cacheRoot = filepath.Join(build.Default.GOPATH, "/pkg/mod/cache/download/vul
 
 type cachedIndex struct {
 	Retrieved time.Time
-	Index     map[string]time.Time
+	Index     osv.DBIndex
 }
 
-func (c *fsCache) ReadIndex(dbName string) (map[string]time.Time, time.Time, error) {
+func (c *fsCache) ReadIndex(dbName string) (osv.DBIndex, time.Time, error) {
 	b, err := os.ReadFile(filepath.Join(cacheRoot, dbName, "index.json"))
 	if err != nil {
 		if os.IsNotExist(err) {
@@ -72,7 +72,7 @@ func (c *fsCache) ReadIndex(dbName string) (map[string]time.Time, time.Time, err
 	return index.Index, index.Retrieved, nil
 }
 
-func (c *fsCache) WriteIndex(dbName string, index map[string]time.Time, retrieved time.Time) error {
+func (c *fsCache) WriteIndex(dbName string, index osv.DBIndex, retrieved time.Time) error {
 	path := filepath.Join(cacheRoot, dbName)
 	if err := os.MkdirAll(path, 0777); err != nil {
 		return err

client/cache_test.go

@@ -38,7 +38,7 @@ func TestCache(t *testing.T) {
 	}
 
 	now := time.Now()
-	expectedIdx := map[string]time.Time{
+	expectedIdx := osv.DBIndex{
 		"a.vuln.example.com": time.Time{}.Add(time.Hour),
 		"b.vuln.example.com": time.Time{}.Add(time.Hour * 2),
 		"c.vuln.example.com": time.Time{}.Add(time.Hour * 3),

client/client.go

@@ -19,7 +19,7 @@ type dbIndex struct{}
 
 type source interface {
 	Get([]string) ([]*osv.Entry, error)
-	Index() (map[string]time.Time, error)
+	Index() (osv.DBIndex, error)
 }
 
 type localSource struct {
@@ -44,8 +44,8 @@ func (ls *localSource) Get(packages []string) ([]*osv.Entry, error) {
 	return entries, nil
 }
 
-func (ls *localSource) Index() (map[string]time.Time, error) {
-	var index map[string]time.Time
+func (ls *localSource) Index() (osv.DBIndex, error) {
+	var index osv.DBIndex
 	b, err := os.ReadFile(filepath.Join(ls.dir, "index.json"))
 	if err != nil {
 		return nil, err
@@ -63,8 +63,8 @@ type httpSource struct {
 	dbName string
 }
 
-func (hs *httpSource) Index() (map[string]time.Time, error) {
-	var cachedIndex map[string]time.Time
+func (hs *httpSource) Index() (osv.DBIndex, error) {
+	var cachedIndex osv.DBIndex
 	var cachedIndexRetrieved *time.Time
 
 	if hs.cache != nil {
@@ -104,7 +104,7 @@ func (hs *httpSource) Index() (map[string]time.Time, error) {
 	if err != nil {
 		return nil, err
 	}
-	var index map[string]time.Time
+	var index osv.DBIndex
 	if err = json.Unmarshal(b, &index); err != nil {
 		return nil, err
 	}

cmd/gendb/main.go

@@ -9,18 +9,12 @@ import (
 	"path/filepath"
 	"reflect"
 	"strings"
-	"time"
 
 	"github.com/BurntSushi/toml"
 	"golang.org/x/vulndb/osv"
 	"golang.org/x/vulndb/report"
 )
 
-type IndexEntry struct {
-	LastModified   time.Time
-	LastNewFinding time.Time
-}
-
 func fail(why string) {
 	fmt.Fprintln(os.Stderr, why)
 	os.Exit(1)
@@ -77,46 +71,26 @@ func main() {
 		}
 	}
 
-	index := map[string]*IndexEntry{}
-	if content, err := ioutil.ReadFile(filepath.Join(*jsonDir, "index.json")); err == nil {
-		err = json.Unmarshal(content, &index)
-		if err != nil {
-			fail(fmt.Sprintf("failed to parse index: %s", err))
-		}
-	} else if err != nil && !os.IsNotExist(err) {
-		fail(fmt.Sprintf("failed to read index %q: %s", filepath.Join(*jsonDir, "index.json"), err))
-	}
-
-	// TODO(bracewell): I'm pretty sure the freshness stuff is basically
-	// completely broken at the moment.
-	now := time.Now()
-	for path, v := range jsonVulns {
+	index := make(osv.DBIndex, len(jsonVulns))
+	for path, vulns := range jsonVulns {
 		outPath := filepath.Join(*jsonDir, path)
-		content, err := json.Marshal(v)
+		content, err := json.Marshal(vulns)
 		if err != nil {
 			fail(fmt.Sprintf("failed to marshal json: %s", err))
 		}
-		// fmt.Println("making", filepath.Dir(outPath))
 		err = os.MkdirAll(filepath.Dir(outPath), 0700)
 		if err != nil {
 			fail(fmt.Sprintf("failed to create directory %q: %s", filepath.Dir(outPath), err))
 		}
-		// if there is already an index entry, only update the file
-		// if the set of vulns differ from what is already on disk
-		if _, ok := index[path]; ok && matchesCurrent(outPath, v) {
-			// fmt.Println("skipping", outPath)
-			continue
-		}
-		// fmt.Println("writing", outPath, string(content))
 		err = ioutil.WriteFile(outPath+".json", content, 0644)
 		if err != nil {
 			fail(fmt.Sprintf("failed to write %q: %s", outPath+".json", err))
 		}
-		if index[path] == nil {
-			index[path] = &IndexEntry{}
+		for _, v := range vulns {
+			if v.LastModified.After(index[path]) {
+				index[path] = v.LastModified
+			}
 		}
-		index[path].LastModified = now
-		// also need to set the LastNewFinding, somewhat more complicated...
 	}
 
 	indexJSON, err := json.Marshal(index)

osv/json.go

@@ -7,6 +7,13 @@ import (
 	"golang.org/x/vulndb/report"
 )
 
+// DBIndex contains a mapping of vulnerable packages to the
+// last time a new vulnerability was added to the database.
+// TODO: this is probably not the correct place to put this
+// type, since it's not really an OSV/CVF thing, but rather
+// vulndb implementatiion detail.
+type DBIndex map[string]time.Time
+
 type Severity int
 
 const (
@@ -166,7 +173,11 @@ func Generate(id string, url string, r report.Report) []Entry {
 	// It would be better if this was just a recursive thing probably
 	for _, additional := range r.AdditionalPackages {
 		entryCopy := entry
-		entryCopy.Package.Name = additional.Package
+		additionalImportPath := additional.Module
+		if additional.Package != "" {
+			additionalImportPath = additional.Package
+		}
+		entryCopy.Package.Name = additionalImportPath
 		entryCopy.EcosystemSpecific.Symbols = additional.Symbols
 		entryCopy.Affects = generateAffects(additional.Versions)
 

report/report.go

@@ -1,5 +1,7 @@
 package report
 
+import "time"
+
 type VersionRange struct {
 	Introduced string
 	Fixed      string
@@ -26,15 +28,17 @@ type Report struct {
 		Symbols  []string
 		Versions []VersionRange
 	} `toml:"additional_packages"`
-	Versions    []VersionRange
-	Description string
-	Severity    string
-	CVE         string
-	Credit      string
-	Symbols     []string
-	OS          []string
-	Arch        []string
-	Links       struct {
+	Versions     []VersionRange
+	Description  string
+	Published    time.Time
+	LastModified time.Time `toml:"last_modified"`
+	Severity     string
+	CVE          string
+	Credit       string
+	Symbols      []string
+	OS           []string
+	Arch         []string
+	Links        struct {
 		PR      string
 		Commit  string
 		Context []string