all: initial commit

Author: rolandshoemaker

README.md

@@ -0,0 +1,12 @@
+This repository contains a handful of prototypes for the Go vulnerability database,
+as well as a initial set of vulnerability reports. Some of these packages can probably
+be coalesced, but for now are easier to work on in a more segmented fashion.
+
+* `reports` contains TOML security reports, the format is described in `format.md`
+* `report` provides a package for parsing and linting TOML reports
+* `osv` provides a package for generating OSV-style JSON vulnerability entries from a `report.Report`
+* `client` contains a client for accesing HTTP/fs based vulnerability databases, as well as a minimal caching implementation
+* `cmd/gendb` provides a tool for converting TOML reports into JSON database
+* `cmd/genhtml` provides a tool for converting TOML reports into a HTML website
+* `cmd/linter` provides a tool for linting individual reports
+* `cmd/report2cve` provides a tool for converting TOML reports into JSON CVEs

client/cache.go

@@ -0,0 +1,115 @@
+package client
+
+import (
+	"encoding/json"
+	"go/build"
+	"os"
+	"path/filepath"
+	"time"
+
+	"golang.org/x/vulndb/osv"
+)
+
+// NOTE: this cache implementation should be internal to the go tooling
+// (i.e. cmd/go/internal/something) so that the vulndb cache is owned
+// by the go command. Also it is currently NOT CONCURRENCY SAFE since
+// it does not implement file locking. When ported to the stdlib it
+// should use cmd/go/internal/lockedfile.
+
+// The cahce uses a single JSON index file for each vulnerability database
+// which contains the map from packages to the time the last
+// vulnerability for that package was added/modified and the time that
+// the index was retrieved from the vulnerability database. The JSON
+// format is as follows:
+//
+// $GOPATH/pkg/mod/cache/download/vulndb/{db hostname}/indexes/index.json
+//   {
+//       Retrieved time.Time
+//       Index map[string]time.Time
+//   }
+//
+// Each package also has a JSON file which contains the array of vulnerability
+// entries for the package. The JSON format is as follows:
+//
+// $GOPATH/pkg/mod/cache/download/vulndb/{db hostname}/{import path}/vulns.json
+//   []*osv.Entry
+
+type Cache interface {
+	ReadIndex(string) (map[string]time.Time, time.Time, error)
+	WriteIndex(string, map[string]time.Time, time.Time) error
+	ReadEntries(string, string) ([]*osv.Entry, error)
+	WriteEntries(string, string, []*osv.Entry) error
+}
+
+type fsCache struct{}
+
+// should be cfg.GOMODCACHE when doing this inside the cmd/go/internal
+var cacheRoot = filepath.Join(build.Default.GOPATH, "/pkg/mod/cache/downlaod/vulndb")
+
+type cachedIndex struct {
+	Retrieved time.Time
+	Index     map[string]time.Time
+}
+
+func (c *fsCache) ReadIndex(dbName string) (map[string]time.Time, time.Time, error) {
+	b, err := os.ReadFile(filepath.Join(cacheRoot, dbName, "index.json"))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, time.Time{}, nil
+		}
+		return nil, time.Time{}, err
+	}
+	var index cachedIndex
+	if err := json.Unmarshal(b, &index); err != nil {
+		return nil, time.Time{}, err
+	}
+	return index.Index, index.Retrieved, nil
+}
+
+func (c *fsCache) WriteIndex(dbName string, index map[string]time.Time, retrieved time.Time) error {
+	path := filepath.Join(cacheRoot, dbName)
+	if err := os.MkdirAll(path, 0777); err != nil {
+		return err
+	}
+	j, err := json.Marshal(cachedIndex{
+		Index:     index,
+		Retrieved: retrieved,
+	})
+	if err != nil {
+		return err
+	}
+	if err := os.WriteFile(filepath.Join(path, "index.json"), j, 0666); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (c *fsCache) ReadEntries(dbName string, p string) ([]*osv.Entry, error) {
+	b, err := os.ReadFile(filepath.Join(cacheRoot, dbName, p, "vulns.json"))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+	var entries []*osv.Entry
+	if err := json.Unmarshal(b, &entries); err != nil {
+		return nil, err
+	}
+	return entries, nil
+}
+
+func (c *fsCache) WriteEntries(dbName string, p string, entries []*osv.Entry) error {
+	path := filepath.Join(cacheRoot, dbName, p)
+	if err := os.MkdirAll(path, 0777); err != nil {
+		return err
+	}
+	j, err := json.Marshal(entries)
+	if err != nil {
+		return err
+	}
+	if err := os.WriteFile(filepath.Join(path, "vulns.json"), j, 0666); err != nil {
+		return err
+	}
+	return nil
+}

client/cache_test.go

@@ -0,0 +1,81 @@
+package client
+
+import (
+	"os"
+	"path/filepath"
+	"reflect"
+	"testing"
+	"time"
+
+	"golang.org/x/vulndb/osv"
+)
+
+func TestCache(t *testing.T) {
+	originalRoot := cacheRoot
+	defer func() { cacheRoot = originalRoot }()
+
+	tmp, err := os.MkdirTemp("", "vulndb-cache")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmp)
+	cacheRoot = tmp
+
+	cache := &fsCache{}
+	dbName := "vulndb.golang.org"
+
+	_, _, err = cache.ReadIndex(dbName)
+	if err != nil {
+		t.Fatalf("ReadIndex failed for non-existent database: %v", err)
+	}
+
+	if err = os.Mkdir(filepath.Join(tmp, dbName), 0777); err != nil {
+		t.Fatalf("os.Mkdir failed: %v", err)
+	}
+	_, _, err = cache.ReadIndex(dbName)
+	if err != nil {
+		t.Fatalf("ReadIndex failed for database without cached index: %v", err)
+	}
+
+	now := time.Now()
+	expectedIdx := map[string]time.Time{
+		"a.vuln.example.com": time.Time{}.Add(time.Hour),
+		"b.vuln.example.com": time.Time{}.Add(time.Hour * 2),
+		"c.vuln.example.com": time.Time{}.Add(time.Hour * 3),
+	}
+	if err = cache.WriteIndex(dbName, expectedIdx, now); err != nil {
+		t.Fatalf("WriteIndex failed to write index: %v", err)
+	}
+
+	idx, retrieved, err := cache.ReadIndex(dbName)
+	if err != nil {
+		t.Fatalf("ReadIndex failed for database with cached index: %v", err)
+	}
+	if !reflect.DeepEqual(idx, expectedIdx) {
+		t.Errorf("ReadIndex returned unexpected index, got:\n%s\nwant:\n%s", idx, expectedIdx)
+	}
+	if !retrieved.Equal(now) {
+		t.Errorf("ReadIndex returned unexpected retrieved: got %s, want %s", retrieved, now)
+	}
+
+	if _, err = cache.ReadEntries(dbName, "vuln.example.com"); err != nil {
+		t.Fatalf("ReadEntires failed for non-existent package: %v", err)
+	}
+
+	expectedEntries := []*osv.Entry{
+		&osv.Entry{ID: "001"},
+		&osv.Entry{ID: "002"},
+		&osv.Entry{ID: "003"},
+	}
+	if err := cache.WriteEntries(dbName, "vuln.example.com", expectedEntries); err != nil {
+		t.Fatalf("WriteEntries failed: %v", err)
+	}
+
+	entries, err := cache.ReadEntries(dbName, "vuln.example.com")
+	if err != nil {
+		t.Fatalf("ReadEntries failed for cached package: %v", err)
+	}
+	if !reflect.DeepEqual(entries, expectedEntries) {
+		t.Errorf("ReadEntries returned unexpected entries, got:\n%v\nwant:\n%v", entries, expectedEntries)
+	}
+}