[pkgsite] internal/vuln: add support for v1 client

0 views
Skip to first unread message

Tatiana Bradley (Gerrit)

unread,
Mar 28, 2023, 4:36:54 PM3/28/23
to goph...@pubsubhelper.golang.org, golang-...@googlegroups.com, kokoro, Julie Qiu, golang-co...@googlegroups.com

Tatiana Bradley submitted this change.

View Change

Approvals: Tatiana Bradley: Looks good to me, approved; Run TryBots kokoro: TryBots succeeded Julie Qiu: Looks good to me, approved
internal/vuln: add support for v1 vulndb client

Add a new client struct, clientV1, that can read from Go vulnerability
databases in the new v1 format. clientV1 implements the internal "client"
interface, and will eventually be renamed to simply Client, and
completely replace the existing Client type.

The clientV1 struct contains a "source" interface. The source interface
is used to read raw JSON data from a given endpoint. The implemented sources
are an HTTP source, used to read from the actual database, and directory
and in-memory sources used for testing.

This struct and its methods are implemented and tested in this change,
but not yet used outside of testing.

For golang/go#58928

Change-Id: Icd4491aeb98a7f7e3bf10301c71ec620cf5cdea8
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/476555
TryBot-Result: kokoro <noreply...@google.com>
Run-TryBot: Tatiana Bradley <tatiana...@google.com>
Reviewed-by: Julie Qiu <juli...@google.com>
Reviewed-by: Tatiana Bradley <tatiana...@google.com>
---
M internal/vuln/client.go
M internal/vuln/client_test.go
A internal/vuln/client_v1.go
A internal/vuln/schema.go
A internal/vuln/source.go
A internal/vuln/source_test.go
M internal/vuln/test_client.go
A internal/vuln/testdata/db.txtar
A internal/vuln/testdata/db2.txtar
A internal/vuln/url.go
A internal/vuln/url_test.go
M internal/vuln/vulns_test.go
12 files changed, 1,114 insertions(+), 37 deletions(-)

diff --git a/internal/vuln/client.go b/internal/vuln/client.go
index 02cccb1..58373cb 100644
--- a/internal/vuln/client.go
+++ b/internal/vuln/client.go
@@ -6,6 +6,7 @@

import (
"context"
+ "fmt"

vulnc "golang.org/x/vuln/client"
"golang.org/x/vuln/osv"
@@ -14,19 +15,22 @@
// Client reads Go vulnerability databases.
type Client struct {
legacy *legacyClient
+ // v1 client, currently for testing only.
+ // Always nil if created via NewClient.
+ v1 *client
}

// NewClient returns a client that can read from the vulnerability
// database in src (a URL representing either a http or file source).
func NewClient(src string) (*Client, error) {
- legacyCli, err := vulnc.NewClient([]string{src}, vulnc.Options{
+ legacy, err := vulnc.NewClient([]string{src}, vulnc.Options{
HTTPCache: newCache(),
})
if err != nil {
return nil, err
}

- return &Client{legacy: &legacyClient{legacyCli}}, nil
+ return &Client{legacy: &legacyClient{legacy}}, nil
}

type PackageRequest struct {
@@ -46,29 +50,53 @@
}

func (c *Client) ByPackage(ctx context.Context, req *PackageRequest) (_ []*osv.Entry, err error) {
- return c.cli(ctx).ByPackage(ctx, req)
+ cli, err := c.cli(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return cli.ByPackage(ctx, req)
}

func (c *Client) ByID(ctx context.Context, id string) (*osv.Entry, error) {
- return c.cli(ctx).ByID(ctx, id)
+ cli, err := c.cli(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return cli.ByID(ctx, id)
}

func (c *Client) ByAlias(ctx context.Context, alias string) ([]*osv.Entry, error) {
- return c.cli(ctx).ByAlias(ctx, alias)
+ cli, err := c.cli(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return cli.ByAlias(ctx, alias)
}

func (c *Client) IDs(ctx context.Context) ([]string, error) {
- return c.cli(ctx).IDs(ctx)
+ cli, err := c.cli(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return cli.IDs(ctx)
}

-func (c *Client) cli(ctx context.Context) client {
- return c.legacy
+// cli returns the underlying client, favoring the legacy client
+// if both are present.
+func (c *Client) cli(ctx context.Context) (cli, error) {
+ if c.legacy != nil {
+ return c.legacy, nil
+ }
+ if c.v1 != nil {
+ return c.v1, nil
+ }
+ return nil, fmt.Errorf("vuln.Client: no underlying client defined")
}

-// client is an interface used temporarily to allow us to support
+// cli is an interface used temporarily to allow us to support
// both the legacy and v1 databases. It will be removed once we have
-// confidence that the v1 client is working.
-type client interface {
+// confidence that the v1 cli is working.
+type cli interface {
ByPackage(ctx context.Context, req *PackageRequest) (_ []*osv.Entry, err error)
ByID(ctx context.Context, id string) (*osv.Entry, error)
ByAlias(ctx context.Context, alias string) ([]*osv.Entry, error)
diff --git a/internal/vuln/client_test.go b/internal/vuln/client_test.go
index a9de101..f521713 100644
--- a/internal/vuln/client_test.go
+++ b/internal/vuln/client_test.go
@@ -14,6 +14,10 @@
"golang.org/x/vuln/osv"
)

+const (
+ dbTxtar = "testdata/db.txtar"
+)
+
var (
jan1999 = time.Date(1999, 1, 1, 0, 0, 0, 0, time.UTC)
jan2000 = time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
@@ -107,7 +111,7 @@
)

func TestByPackage(t *testing.T) {
- runClientTest(t, func(t *testing.T, c client) {
+ runClientTest(t, func(t *testing.T, c cli) {
tests := []struct {
name string
req *PackageRequest
@@ -201,6 +205,22 @@
},
want: nil,
},
+ {
+ name: "go prefix, no patch version - in range",
+ req: &PackageRequest{
+ Module: "stdlib",
+ Version: "go1.2",
+ },
+ want: []*osv.Entry{&testOSV1},
+ },
+ {
+ name: "go prefix, no patch version - out of range",
+ req: &PackageRequest{
+ Module: "stdlib",
+ Version: "go1.3",
+ },
+ want: nil,
+ },
}

for _, test := range tests {
@@ -227,7 +247,7 @@
}

func TestByAlias(t *testing.T) {
- runClientTest(t, func(t *testing.T, c client) {
+ runClientTest(t, func(t *testing.T, c cli) {
tests := []struct {
name string
alias string
@@ -266,7 +286,7 @@
}

func TestByID(t *testing.T) {
- runClientTest(t, func(t *testing.T, c client) {
+ runClientTest(t, func(t *testing.T, c cli) {
tests := []struct {
id string
want *osv.Entry
@@ -301,7 +321,7 @@
}

func TestIDs(t *testing.T) {
- runClientTest(t, func(t *testing.T, c client) {
+ runClientTest(t, func(t *testing.T, c cli) {
ctx := context.Background()

got, err := c.IDs(ctx)
@@ -316,12 +336,62 @@
})
}

-// Run the test legacy client.
-// TODO(tatianabradley): Run test for v1 client once implemented.
-func runClientTest(t *testing.T, test func(*testing.T, client)) {
+// Test that Client can pick the right underlying client.
+func TestCli(t *testing.T) {
+ ctx := context.Background()
+
+ v1, err := newTestV1Client(dbTxtar)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ legacy := newTestLegacyClient([]*osv.Entry{&testOSV1, &testOSV2, &testOSV3})
+
+ t.Run("legacy preferred", func(t *testing.T) {
+ c := Client{legacy: legacy, v1: v1}
+ cli, err := c.cli(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, ok := cli.(*legacyClient); !ok {
+ t.Errorf("Client.cli() = %s, want type *legacyClient", cli)
+ }
+ })
+
+ t.Run("v1 if no legacy", func(t *testing.T) {
+ c := Client{v1: v1}
+ cli, err := c.cli(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, ok := cli.(*client); !ok {
+ t.Errorf("Client.cli() = %s, want type *clientV1", cli)
+ }
+ })
+
+ t.Run("error if both nil", func(t *testing.T) {
+ c := Client{}
+ cli, err := c.cli(ctx)
+ if err == nil {
+ t.Errorf("Client.cli() = %s, want error", cli)
+ }
+ })
+}
+
+// Run the test for both the v1 and legacy clients.
+func runClientTest(t *testing.T, test func(*testing.T, cli)) {
+ v1, err := newTestV1Client(dbTxtar)
+ if err != nil {
+ t.Fatal(err)
+ }
+
legacy := newTestLegacyClient([]*osv.Entry{&testOSV1, &testOSV2, &testOSV3})

t.Run("legacy", func(t *testing.T) {
test(t, legacy)
})
+
+ t.Run("v1", func(t *testing.T) {
+ test(t, v1)
+ })
}
diff --git a/internal/vuln/client_v1.go b/internal/vuln/client_v1.go
new file mode 100644
index 0000000..b4060e3
--- /dev/null
+++ b/internal/vuln/client_v1.go
@@ -0,0 +1,253 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package vuln
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "path/filepath"
+ "sort"
+ "strings"
+ "sync"
+
+ "golang.org/x/mod/semver"
+ "golang.org/x/pkgsite/internal/derrors"
+ "golang.org/x/sync/errgroup"
+ "golang.org/x/vuln/osv"
+)
+
+// client is a client for the v1 vulnerability database.
+type client struct {
+ src source
+}
+
+// ByPackage returns the OSV entries matching the package request.
+func (c *client) ByPackage(ctx context.Context, req *PackageRequest) (_ []*osv.Entry, err error) {
+ derrors.Wrap(&err, "ByPackage(%v)", req)
+
+ b, err := c.modules(ctx)
+ if err != nil {
+ return nil, err
+ }
+
+ dec, err := newStreamDecoder(b)
+ if err != nil {
+ return nil, err
+ }
+
+ var ids []string
+ for dec.More() {
+ var m ModuleMeta
+ err := dec.Decode(&m)
+ if err != nil {
+ return nil, err
+ }
+ if m.Path == req.Module {
+ for _, v := range m.Vulns {
+ // We need to download the full entry if there is no fix,
+ // or the requested version is less than the vuln's
+ // highest fixed version.
+ if v.Fixed == "" || less(req.Version, v.Fixed) {
+ ids = append(ids, v.ID)
+ }
+ }
+ // We found the requested module, so skip the rest.
+ break
+ }
+ }
+
+ if len(ids) == 0 {
+ return nil, nil
+ }
+
+ // Fetch all the entries in parallel, and create a slice
+ // containing all the actually affected entries.
+ g, gctx := errgroup.WithContext(ctx)
+ var mux sync.Mutex
+ g.SetLimit(10)
+ entries := make([]*osv.Entry, 0, len(ids))
+ for _, id := range ids {
+ id := id
+ g.Go(func() error {
+ entry, err := c.ByID(gctx, id)
+ if err != nil {
+ return err
+ }
+
+ if entry == nil {
+ return fmt.Errorf("vulnerability %s was found in %s but could not be retrieved", id, modulesEndpoint)
+ }
+
+ if isAffected(entry, req) {
+ mux.Lock()
+ entries = append(entries, entry)
+ mux.Unlock()
+ }
+
+ return nil
+ })
+ }
+ if err := g.Wait(); err != nil {
+ return nil, err
+ }
+
+ sort.SliceStable(entries, func(i, j int) bool {
+ return entries[i].ID < entries[j].ID
+ })
+
+ return entries, nil
+}
+
+// less returns whether v1 < v2, where v1 and v2 are
+// semver versions with either a "v", "go" or no prefix.
+func less(v1, v2 string) bool {
+ return semver.Compare(canonicalizeSemver(v1), canonicalizeSemver(v2)) < 0
+}
+
+// canonicalizeSemver turns a SEMVER string into the canonical
+// representation using the 'v' prefix as used by the "semver" package.
+// Input may be a bare SEMVER ("1.2.3"), Go prefixed SEMVER ("go1.2.3"),
+// or already canonical SEMVER ("v1.2.3").
+func canonicalizeSemver(s string) string {
+ // Remove "go" prefix if needed.
+ s = strings.TrimPrefix(s, "go")
+ // Add "v" prefix if needed.
+ if !strings.HasPrefix(s, "v") {
+ s = "v" + s
+ }
+ return s
+}
+
+// ByID returns the OSV entry with the given ID or (nil, nil)
+// if there isn't one.
+func (c *client) ByID(ctx context.Context, id string) (_ *osv.Entry, err error) {
+ derrors.Wrap(&err, "ByID(%s)", id)
+
+ b, err := c.entry(ctx, id)
+ if err != nil {
+ // entry only fails if the entry is not found, so do not return
+ // the error.
+ return nil, nil
+ }
+
+ var entry osv.Entry
+ if err := json.Unmarshal(b, &entry); err != nil {
+ return nil, err
+ }
+
+ return &entry, nil
+}
+
+// ByAlias returns the OSV entries that have the given alias, or (nil, nil)
+// if there are none.
+// It returns a list for compatibility with the legacy implementation,
+// but the list always contains at most one element.
+func (c *client) ByAlias(ctx context.Context, alias string) (_ []*osv.Entry, err error) {
+ derrors.Wrap(&err, "ByAlias(%s)", alias)
+
+ b, err := c.vulns(ctx)
+ if err != nil {
+ return nil, err
+ }
+
+ dec, err := newStreamDecoder(b)
+ if err != nil {
+ return nil, err
+ }
+
+ var id string
+ for dec.More() {
+ var v VulnMeta
+ err := dec.Decode(&v)
+ if err != nil {
+ return nil, err
+ }
+ for _, vAlias := range v.Aliases {
+ if alias == vAlias {
+ id = v.ID
+ break
+ }
+ }
+ if id != "" {
+ break
+ }
+ }
+
+ if id == "" {
+ return nil, nil
+ }
+
+ entry, err := c.ByID(ctx, id)
+ if err != nil {
+ return nil, err
+ }
+
+ if entry == nil {
+ return nil, fmt.Errorf("vulnerability %s was found in %s but could not be retrieved", id, vulnsEndpoint)
+ }
+
+ return []*osv.Entry{entry}, nil
+}
+
+// IDs returns a list of the IDs of all the entries in the database.
+func (c *client) IDs(ctx context.Context) (_ []string, err error) {
+ derrors.Wrap(&err, "IDs()")
+
+ b, err := c.vulns(ctx)
+ if err != nil {
+ return nil, err
+ }
+
+ dec, err := newStreamDecoder(b)
+ if err != nil {
+ return nil, err
+ }
+
+ var ids []string
+ for dec.More() {
+ var v VulnMeta
+ err := dec.Decode(&v)
+ if err != nil {
+ return nil, err
+ }
+ ids = append(ids, v.ID)
+ }
+
+ return ids, nil
+}
+
+// newStreamDecoder returns a decoder that can be used
+// to read an array of JSON objects.
+func newStreamDecoder(b []byte) (*json.Decoder, error) {
+ dec := json.NewDecoder(bytes.NewBuffer(b))
+
+ // skip open bracket
+ _, err := dec.Token()
+ if err != nil {
+ return nil, err
+ }
+
+ return dec, nil
+}
+
+var (
+ idDir = "ID"
+ modulesEndpoint = "index/modules"
+ vulnsEndpoint = "index/vulns"
+)
+
+func (c *client) modules(ctx context.Context) ([]byte, error) {
+ return c.src.get(ctx, modulesEndpoint)
+}
+
+func (c *client) vulns(ctx context.Context) ([]byte, error) {
+ return c.src.get(ctx, vulnsEndpoint)
+}
+
+func (c *client) entry(ctx context.Context, id string) ([]byte, error) {
+ return c.src.get(ctx, filepath.Join(idDir, id))
+}
diff --git a/internal/vuln/schema.go b/internal/vuln/schema.go
new file mode 100644
index 0000000..3113f5b
--- /dev/null
+++ b/internal/vuln/schema.go
@@ -0,0 +1,47 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package vuln
+
+import "time"
+
+// ModuleMeta contains metadata about a Go module that has one
+// or more vulnerabilities in the database.
+//
+// Found in the "index/modules" endpoint of the vulnerability database.
+type ModuleMeta struct {
+ // Path is the module path.
+ Path string `json:"path"`
+ // Vulns is a list of vulnerabilities that affect this module.
+ Vulns []ModuleVuln `json:"vulns"`
+}
+
+// ModuleVuln contains metadata about a vulnerability that affects
+// a certain module.
+type ModuleVuln struct {
+ // ID is a unique identifier for the vulnerability.
+ // The Go vulnerability database issues IDs of the form
+ // GO-<YEAR>-<ENTRYID>.
+ ID string `json:"id"`
+ // Modified is the time the vuln was last modified.
+ Modified time.Time `json:"modified"`
+ // Fixed is the latest version that introduces a fix for the
+ // vulnerability, in SemVer 2.0.0 format, with no leading "v" prefix.
+ Fixed string `json:"fixed,omitempty"`
+}
+
+// VulnMeta contains metadata about a vulnerability in the database.
+//
+// Found in the "index/vulns" endpoint of the vulnerability database.
+type VulnMeta struct {
+ // ID is a unique identifier for the vulnerability.
+ // The Go vulnerability database issues IDs of the form
+ // GO-<YEAR>-<ENTRYID>.
+ ID string `json:"id"`
+ // Modified is the time the vulnerability was last modified.
+ Modified time.Time `json:"modified"`
+ // Aliases is a list of IDs for the same vulnerability in other
+ // databases.
+ Aliases []string `json:"aliases,omitempty"`
+}
diff --git a/internal/vuln/source.go b/internal/vuln/source.go
new file mode 100644
index 0000000..ed699f6
--- /dev/null
+++ b/internal/vuln/source.go
@@ -0,0 +1,109 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package vuln
+
+import (
+ "compress/gzip"
+ "context"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "os"
+ "path/filepath"
+)
+
+type source interface {
+ // get returns the raw, uncompressed bytes at the
+ // requested endpoint, which should be bare with no file extensions
+ // (e.g., "index/modules" instead of "index/modules.json.gz").
+ // It errors if the endpoint cannot be reached or does not exist
+ // in the expected form.
+ get(ctx context.Context, endpoint string) ([]byte, error)
+}
+
+// NewSource returns a source interface from a http:// or file:// prefixed
+// url src. It errors if the given url is invalid or does not exist.
+func NewSource(src string) (source, error) {
+ uri, err := url.Parse(src)
+ if err != nil {
+ return nil, err
+ }
+ switch uri.Scheme {
+ case "http", "https":
+ return &httpSource{url: uri.String(), c: http.DefaultClient}, nil
+ case "file":
+ dir, err := URLToFilePath(uri)
+ if err != nil {
+ return nil, err
+ }
+ fi, err := os.Stat(dir)
+ if err != nil {
+ return nil, err
+ }
+ if !fi.IsDir() {
+ return nil, fmt.Errorf("%s is not a directory", dir)
+ }
+ return &localSource{dir: dir}, nil
+ default:
+ return nil, fmt.Errorf("src %q has unsupported scheme", uri)
+ }
+}
+
+// httpSource reads databases from an http(s) source.
+// Intended for use in production.
+type httpSource struct {
+ url string
+ c *http.Client
+}
+
+func (hs *httpSource) get(ctx context.Context, endpoint string) ([]byte, error) {
+ reqURL := fmt.Sprintf("%s/%s", hs.url, endpoint+".json.gz")
+ req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil)
+ if err != nil {
+ return nil, err
+ }
+ resp, err := hs.c.Do(req)
+ if err != nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("GET %s: unexpected status code: %d", req.URL, resp.StatusCode)
+ }
+
+ // Uncompress the result.
+ r, err := gzip.NewReader(resp.Body)
+ if err != nil {
+ return nil, err
+ }
+ defer r.Close()
+
+ return io.ReadAll(r)
+}
+
+// localSource reads databases from a local directory.
+// Intended for use in unit tests and screentests.
+type localSource struct {
+ dir string
+}
+
+func (db *localSource) get(ctx context.Context, endpoint string) ([]byte, error) {
+ return os.ReadFile(filepath.Join(db.dir, endpoint+".json"))
+}
+
+// inMemorySource reads databases from an in-memory map.
+// Intended for use in unit tests.
+type inMemorySource struct {
+ data map[string][]byte
+}
+
+func (db *inMemorySource) get(ctx context.Context, endpoint string) ([]byte, error) {
+ b, ok := db.data[endpoint]
+ if !ok {
+ return nil, fmt.Errorf("no data found at endpoint %q", endpoint)
+ }
+ return b, nil
+}
diff --git a/internal/vuln/source_test.go b/internal/vuln/source_test.go
new file mode 100644
index 0000000..28ee485
--- /dev/null
+++ b/internal/vuln/source_test.go
@@ -0,0 +1,124 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package vuln
+
+import (
+ "bytes"
+ "compress/gzip"
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "testing"
+)
+
+func TestNewSource(t *testing.T) {
+ t.Run("https", func(t *testing.T) {
+ url := "https://vuln.go.dev"
+ s, err := NewSource(url)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, ok := s.(*httpSource); !ok {
+ t.Errorf("NewSource(%s) = %#v, want type *httpSource ", url, s)
+ }
+ })
+
+ t.Run("file", func(t *testing.T) {
+ fileURL := "file:///" + t.TempDir()
+ s, err := NewSource(fileURL)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, ok := s.(*localSource); !ok {
+ t.Errorf("NewSource(%s) = %#v, want type *localSource", fileURL, s)
+ }
+ })
+}
+
+func TestHTTPSource(t *testing.T) {
+ want := []byte("some data")
+ gzipped, err := gzipped(want)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+ if req.URL.Path == "/test/endpoint.json.gz" {
+ if _, err := rw.Write(gzipped); err != nil {
+ rw.WriteHeader(http.StatusInternalServerError)
+ }
+ return
+ }
+ rw.WriteHeader(http.StatusNotFound)
+ }))
+ defer server.Close()
+
+ src := httpSource{
+ url: server.URL,
+ c: server.Client(),
+ }
+ got, err := src.get(context.Background(), "test/endpoint")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if string(got) != string(want) {
+ t.Errorf("httpSource.get = %s, want %s", got, want)
+ }
+}
+
+func TestLocalSource(t *testing.T) {
+ temp := t.TempDir()
+ if err := os.Mkdir(filepath.Join(temp, "test"), 0755); err != nil {
+ t.Fatal(err)
+ }
+
+ want := []byte("some data")
+ if err := os.WriteFile(filepath.Join(temp, "test/endpoint.json"), want, 0644); err != nil {
+ t.Fatal(err)
+ }
+
+ src := localSource{
+ dir: temp,
+ }
+ got, err := src.get(context.Background(), "test/endpoint")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if string(got) != string(want) {
+ t.Errorf("localSource.get = %s, want %s", got, want)
+ }
+}
+
+func TestInMemorySource(t *testing.T) {
+ want := []byte("some data")
+ src := inMemorySource{
+ data: map[string][]byte{
+ "test/endpoint": want,
+ },
+ }
+
+ got, err := src.get(context.Background(), "test/endpoint")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if string(got) != string(want) {
+ t.Errorf("inMemorySource.get = %s, want %s", got, want)
+ }
+}
+
+func gzipped(data []byte) ([]byte, error) {
+ var b bytes.Buffer
+ w := gzip.NewWriter(&b)
+ defer w.Close()
+ if _, err := w.Write(data); err != nil {
+ return nil, err
+ }
+ if err := w.Close(); err != nil {
+ return nil, err
+ }
+ return b.Bytes(), nil
+}
diff --git a/internal/vuln/test_client.go b/internal/vuln/test_client.go
index 12d88e0..c2b2f5a 100644
--- a/internal/vuln/test_client.go
+++ b/internal/vuln/test_client.go
@@ -7,15 +7,39 @@
import (
"context"

+ "golang.org/x/tools/txtar"
vulnc "golang.org/x/vuln/client"
"golang.org/x/vuln/osv"
)

-// NewTestClient creates an in-memory client for use in tests.
+// NewTestClient creates an in-memory client for use in tests,
+// It's logic is different from the real client, so it should not be used to
+// test the client itself, but can be used to test code that depends on the
+// client.
func NewTestClient(entries []*osv.Entry) *Client {
return &Client{legacy: newTestLegacyClient(entries)}
}

+// newTestV1Client creates an in-memory client for use in tests.
+// It uses all the logic of the real v1 client, except that it reads
+// raw database data from the given txtar file instead of making HTTP
+// requests.
+// It can be used to test core functionality of the v1 client.
+func newTestV1Client(txtarFile string) (*client, error) {
+ data := make(map[string][]byte)
+
+ ar, err := txtar.ParseFile(txtarFile)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, f := range ar.Files {
+ data[f.Name] = f.Data
+ }
+
+ return &client{&inMemorySource{data: data}}, nil
+}
+
func newTestLegacyClient(entries []*osv.Entry) *legacyClient {
c := &testVulnClient{
entries: entries,
diff --git a/internal/vuln/testdata/db.txtar b/internal/vuln/testdata/db.txtar
new file mode 100644
index 0000000..a165bb9
--- /dev/null
+++ b/internal/vuln/testdata/db.txtar
@@ -0,0 +1,229 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+//
+// Test database for the Go vulnerability database v1 schema with
+// three entries.
+
+-- index/db --
+{
+ "modified": "2003-01-01T00:00:00Z"
+}
+
+-- index/vulns --
+[
+ {
+ "id": "GO-1999-0001",
+ "modified": "2000-01-01T00:00:00Z",
+ "aliases": [
+ "CVE-1999-1111"
+ ]
+ },
+ {
+ "id": "GO-2000-0002",
+ "modified": "2002-01-01T00:00:00Z",
+ "aliases": [
+ "CVE-1999-2222"
+ ]
+ },
+ {
+ "id": "GO-2000-0003",
+ "modified": "2003-01-01T00:00:00Z",
+ "aliases": [
+ "CVE-1999-3333",
+ "GHSA-xxxx-yyyy-zzzz"
+ ]
+ }
+]
+
+-- index/modules --
+[
+ {
+ "path": "example.com/module",
+ "vulns": [
+ {
+ "id": "GO-2000-0002",
+ "modified": "2002-01-01T00:00:00Z",
+ "fixed": "1.2.0"
+ },
+ {
+ "id": "GO-2000-0003",
+ "modified": "2003-01-01T00:00:00Z",
+ "fixed": "1.1.0"
+ }
+ ]
+ },
+ {
+ "path": "stdlib",
+ "vulns": [
+ {
+ "id": "GO-1999-0001",
+ "modified": "2000-01-01T00:00:00Z",
+ "fixed": "1.2.2"
+ }
+ ]
+ }
+]
+
+-- ID/GO-1999-0001 --
+{
+ "id": "GO-1999-0001",
+ "published": "1999-01-01T00:00:00Z",
+ "modified": "2000-01-01T00:00:00Z",
+ "aliases": [
+ "CVE-1999-1111"
+ ],
+ "details": "Some details",
+ "affected": [
+ {
+ "package": {
+ "name": "stdlib",
+ "ecosystem": "Go"
+ },
+ "ranges": [
+ {
+ "type": "SEMVER",
+ "events": [
+ {
+ "introduced": "0"
+ },
+ {
+ "fixed": "1.1.0"
+ },
+ {
+ "introduced": "1.2.0"
+ },
+ {
+ "fixed": "1.2.2"
+ }
+ ]
+ }
+ ],
+ "database_specific": {
+ "url": "https://pkg.go.dev/vuln/GO-1999-0001"
+ },
+ "ecosystem_specific": {
+ "imports": [
+ {
+ "path": "package",
+ "symbols": [
+ "Symbol"
+ ]
+ }
+ ]
+ }
+ }
+ ],
+ "references": [
+ {
+ "type": "FIX",
+ "url": "https://example.com/cl/123"
+ }
+ ]
+}
+
+-- ID/GO-2000-0002 --
+{
+ "id": "GO-2000-0002",
+ "published": "2000-01-01T00:00:00Z",
+ "modified": "2002-01-01T00:00:00Z",
+ "aliases": [
+ "CVE-1999-2222"
+ ],
+ "details": "Some details",
+ "affected": [
+ {
+ "package": {
+ "name": "example.com/module",
+ "ecosystem": "Go"
+ },
+ "ranges": [
+ {
+ "type": "SEMVER",
+ "events": [
+ {
+ "introduced": "0"
+ },
+ {
+ "fixed": "1.2.0"
+ }
+ ]
+ }
+ ],
+ "database_specific": {
+ "url": "https://pkg.go.dev/vuln/GO-2000-0002"
+ },
+ "ecosystem_specific": {
+ "imports": [
+ {
+ "path": "example.com/module/package",
+ "symbols": [
+ "Symbol"
+ ]
+ }
+ ]
+ }
+ }
+ ],
+ "references": [
+ {
+ "type": "FIX",
+ "url": "https://example.com/cl/543"
+ }
+ ]
+}
+
+-- ID/GO-2000-0003 --
+{
+ "id": "GO-2000-0003",
+ "published": "2000-01-01T00:00:00Z",
+ "modified": "2003-01-01T00:00:00Z",
+ "aliases": [
+ "CVE-1999-3333",
+ "GHSA-xxxx-yyyy-zzzz"
+ ],
+ "details": "Some details",
+ "affected": [
+ {
+ "package": {
+ "name": "example.com/module",
+ "ecosystem": "Go"
+ },
+ "ranges": [
+ {
+ "type": "SEMVER",
+ "events": [
+ {
+ "introduced": "0"
+ },
+ {
+ "fixed": "1.1.0"
+ }
+ ]
+ }
+ ],
+ "database_specific": {
+ "url": "https://pkg.go.dev/vuln/GO-2000-0003"
+ },
+ "ecosystem_specific": {
+ "imports": [
+ {
+ "path": "example.com/module/package",
+ "symbols": [
+ "Symbol"
+ ]
+ },
+ {
+ "path": "example.com/module/package2"
+ }
+ ]
+ }
+ }
+ ],
+ "references": [
+ {
+ "type": "FIX",
+ "url": "https://example.com/cl/000"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/internal/vuln/testdata/db2.txtar b/internal/vuln/testdata/db2.txtar
new file mode 100644
index 0000000..6de6152
--- /dev/null
+++ b/internal/vuln/testdata/db2.txtar
@@ -0,0 +1,25 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+//
+// Test database for the Go vulnerability database v1 schema with
+// three entries.
+// This is meant for testing the VulnsForPackage function.
+
+-- index/db --
+{"modified":"0001-01-01T00:00:00Z"}
+
+-- index/vulns --
+[{"id":"GO-1","modified":"0001-01-01T00:00:00Z"},{"id":"GO-2","modified":"0001-01-01T00:00:00Z"},{"id":"GO-3","modified":"0001-01-01T00:00:00Z"}]
+
+-- index/modules --
+[{"path":"bad.com","vulns":[{"id":"GO-1","modified":"0001-01-01T00:00:00Z","fixed":"1.2.3"},{"id":"GO-2","modified":"0001-01-01T00:00:00Z","fixed":"1.2.0"}]},{"path":"stdlib","vulns":[{"id":"GO-3","modified":"0001-01-01T00:00:00Z","fixed":"1.19.4"}]},{"path":"unfixable.com","vulns":[{"id":"GO-1","modified":"0001-01-01T00:00:00Z"}]}]
+
+-- ID/GO-1 --
+{"id":"GO-1","published":"0001-01-01T00:00:00Z","modified":"0001-01-01T00:00:00Z","details":"","affected":[{"package":{"name":"bad.com","ecosystem":""},"ranges":[{"type":"SEMVER","events":[{"introduced":"0"},{"fixed":"1.2.3"}]}],"database_specific":{"url":""},"ecosystem_specific":{"imports":[{"path":"bad.com"},{"path":"bad.com/bad"}]}},{"package":{"name":"unfixable.com","ecosystem":""},"ranges":[{"type":"SEMVER","events":[{"introduced":"0"}]}],"database_specific":{"url":""},"ecosystem_specific":{"imports":[{"path":"unfixable.com"}]}}]}
+
+-- ID/GO-2 --
+{"id":"GO-2","published":"0001-01-01T00:00:00Z","modified":"0001-01-01T00:00:00Z","details":"","affected":[{"package":{"name":"bad.com","ecosystem":""},"ranges":[{"type":"SEMVER","events":[{"introduced":"0"},{"fixed":"1.2.0"}]}],"database_specific":{"url":""},"ecosystem_specific":{"imports":[{"path":"bad.com/pkg"}]}}]}
+
+-- ID/GO-3 --
+{"id":"GO-3","published":"0001-01-01T00:00:00Z","modified":"0001-01-01T00:00:00Z","details":"","affected":[{"package":{"name":"stdlib","ecosystem":""},"ranges":[{"type":"SEMVER","events":[{"introduced":"0"},{"fixed":"1.19.4"}]}],"database_specific":{"url":""},"ecosystem_specific":{"imports":[{"path":"net/http"}]}}]}
\ No newline at end of file
diff --git a/internal/vuln/url.go b/internal/vuln/url.go
new file mode 100644
index 0000000..60e2d1e
--- /dev/null
+++ b/internal/vuln/url.go
@@ -0,0 +1,57 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Code adapted from
+// https://github.com/golang/go/blob/2ebe77a2fda1ee9ff6fd9a3e08933ad1ebaea039/src/cmd/go/internal/web/url.go
+// TODO(go.dev/issue/32456): if accepted, use the new API.
+
+package vuln
+
+import (
+ "errors"
+ "net/url"
+ "path/filepath"
+ "runtime"
+)
+
+var errNotAbsolute = errors.New("path is not absolute")
+
+// URLToFilePath converts a file-scheme url to a file path.
+func URLToFilePath(u *url.URL) (string, error) {
+ if u.Scheme != "file" {
+ return "", errors.New("non-file URL")
+ }
+
+ checkAbs := func(path string) (string, error) {
+ if !filepath.IsAbs(path) {
+ return "", errNotAbsolute
+ }
+ return path, nil
+ }
+
+ if u.Path == "" {
+ if u.Host != "" || u.Opaque == "" {
+ return "", errors.New("file URL missing path")
+ }
+ return checkAbs(filepath.FromSlash(u.Opaque))
+ }
+
+ path, err := convertFileURLPath(u.Host, u.Path)
+ if err != nil {
+ return path, err
+ }
+ return checkAbs(path)
+}
+
+func convertFileURLPath(host, path string) (string, error) {
+ if runtime.GOOS == "windows" {
+ return "", errors.New("windows not supported")
+ }
+ switch host {
+ case "", "localhost":
+ default:
+ return "", errors.New("file URL specifies non-local host")
+ }
+ return filepath.FromSlash(path), nil
+}
diff --git a/internal/vuln/url_test.go b/internal/vuln/url_test.go
new file mode 100644
index 0000000..37b5bc7
--- /dev/null
+++ b/internal/vuln/url_test.go
@@ -0,0 +1,73 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package vuln
+
+import (
+ "net/url"
+ "testing"
+)
+
+func TestURLToFilePath(t *testing.T) {
+ for _, tc := range urlTests {
+ if tc.url == "" {
+ continue
+ }
+ tc := tc
+
+ t.Run(tc.url, func(t *testing.T) {
+ u, err := url.Parse(tc.url)
+ if err != nil {
+ t.Fatalf("url.Parse(%q): %v", tc.url, err)
+ }
+
+ path, err := URLToFilePath(u)
+ if err != nil {
+ if err.Error() == tc.wantErr {
+ return
+ }
+ if tc.wantErr == "" {
+ t.Fatalf("urlToFilePath(%v): %v; want <nil>", u, err)
+ } else {
+ t.Fatalf("urlToFilePath(%v): %v; want %s", u, err, tc.wantErr)
+ }
+ }
+
+ if path != tc.filePath || tc.wantErr != "" {
+ t.Fatalf("urlToFilePath(%v) = %q, <nil>; want %q, %s", u, path, tc.filePath, tc.wantErr)
+ }
+ })
+ }
+}
+
+type urlTest struct {
+ url string
+ filePath string
+ canonicalURL string // If empty, assume equal to url.
+ wantErr string
+}
+
+var urlTests = []urlTest{
+ // Examples from RFC 8089:
+ {
+ url: `file:///path/to/file`,
+ filePath: `/path/to/file`,
+ },
+ {
+ url: `file:/path/to/file`,
+ filePath: `/path/to/file`,
+ canonicalURL: `file:///path/to/file`,
+ },
+ {
+ url: `file://localhost/path/to/file`,
+ filePath: `/path/to/file`,
+ canonicalURL: `file:///path/to/file`,
+ },
+
+ // We reject non-local files.
+ {
+ url: `file://host.example.com/path/to/file`,
+ wantErr: "file URL specifies non-local host",
+ },
+}
diff --git a/internal/vuln/vulns_test.go b/internal/vuln/vulns_test.go
index 10e79c5..0eb8d38 100644
--- a/internal/vuln/vulns_test.go
+++ b/internal/vuln/vulns_test.go
@@ -77,61 +77,99 @@
}},
}

- vc := NewTestClient([]*osv.Entry{&e, &e2, &stdlib})
+ legacyClient := newTestLegacyClient([]*osv.Entry{&e, &e2, &stdlib})
+ v1Client, err := newTestV1Client("testdata/db2.txtar")
+ if err != nil {
+ t.Fatal(err)
+ }

testCases := []struct {
+ name string
mod, pkg, version string
want []Vuln
}{
// Vulnerabilities for a package
{
- "good.com", "good.com", "v1.0.0", nil,
+ name: "no match - all",
+ mod: "good.com", pkg: "good.com", version: "v1.0.0",
+ want: nil,
},
{
- "bad.com", "bad.com", "v1.0.0", []Vuln{{ID: "GO-1"}},
+ name: "match - same mod/pkg",
+ mod: "bad.com", pkg: "bad.com", version: "v1.0.0",
+ want: []Vuln{{ID: "GO-1"}},
},
{
- "bad.com", "bad.com/bad", "v1.0.0", []Vuln{{ID: "GO-1"}},
+ name: "match - different mod/pkg",
+ mod: "bad.com", pkg: "bad.com/bad", version: "v1.0.0",
+ want: []Vuln{{ID: "GO-1"}},
},
{
- "bad.com", "bad.com/ok", "v1.0.0", nil, // bad.com/ok isn't affected.
+ name: "no match - pkg",
+ mod: "bad.com", pkg: "bad.com/ok", version: "v1.0.0",
+ want: nil, // bad.com/ok isn't affected.
},
{
- "bad.com", "bad.com", "v1.3.0", nil,
+ name: "no match - version",
+ mod: "bad.com", pkg: "bad.com", version: "v1.3.0",
+ want: nil, // version 1.3.0 isn't affected
},
{
- "unfixable.com", "unfixable.com", "v1.999.999", []Vuln{{ID: "GO-1"}},
+ name: "match - pkg with no fix",
+ mod: "unfixable.com", pkg: "unfixable.com", version: "v1.999.999", want: []Vuln{{ID: "GO-1"}},
},
// Vulnerabilities for a module (package == "")
{
- "good.com", "", "v1.0.0", nil,
+ name: "no match - module only",
+ mod: "good.com", pkg: "", version: "v1.0.0", want: nil,
},
{
- "bad.com", "", "v1.0.0", []Vuln{{ID: "GO-1"}, {ID: "GO-2"}},
+ name: "match - module only",
+ mod: "bad.com", pkg: "", version: "v1.0.0", want: []Vuln{{ID: "GO-1"}, {ID: "GO-2"}},
},
{
- "bad.com", "", "v1.3.0", nil,
+ name: "no match - module but not version",
+ mod: "bad.com", pkg: "", version: "v1.3.0",
+ want: nil,
},
{
- "unfixable.com", "", "v1.999.999", []Vuln{{ID: "GO-1"}},
+ name: "match - module only, no fix",
+ mod: "unfixable.com", pkg: "", version: "v1.999.999", want: []Vuln{{ID: "GO-1"}},
},
// Vulns for stdlib
{
- "std", "net/http", "go1.19.3", []Vuln{{ID: "GO-3"}},
+ name: "match - stdlib",
+ mod: "std", pkg: "net/http", version: "go1.19.3",
+ want: []Vuln{{ID: "GO-3"}},
},
{
- "std", "net/http", "v0.0.0-20230104211531-bae7d772e800", nil,
+ name: "no match - stdlib pseudoversion",
+ mod: "std", pkg: "net/http", version: "v0.0.0-20230104211531-bae7d772e800", want: nil,
},
{
- "std", "net/http", "go1.20", nil,
+ name: "no match - stdlib version past fix",
+ mod: "std", pkg: "net/http", version: "go1.20", want: nil,
},
}
- for _, tc := range testCases {
- got := VulnsForPackage(ctx, tc.mod, tc.version, tc.pkg, vc)
- if diff := cmp.Diff(tc.want, got); diff != "" {
- t.Errorf("VulnsForPackage(mod=%q, v=%q, pkg=%q) = %+v, want %+v, diff (-want, +got):\n%s", tc.mod, tc.version, tc.pkg, got, tc.want, diff)
+ test := func(t *testing.T, c *Client) {
+ for _, tc := range testCases {
+ {
+ t.Run(tc.name, func(t *testing.T) {
+ got := VulnsForPackage(ctx, tc.mod, tc.version, tc.pkg, c)
+ if diff := cmp.Diff(tc.want, got); diff != "" {
+ t.Errorf("VulnsForPackage(mod=%q, v=%q, pkg=%q) = %+v, want %+v, diff (-want, +got):\n%s", tc.mod, tc.version, tc.pkg, got, tc.want, diff)
+ }
+ })
+ }
}
}
+ t.Run("legacy", func(t *testing.T) {
+ test(t, &Client{legacy: legacyClient})
+ })
+
+ t.Run("v1", func(t *testing.T) {
+ test(t, &Client{v1: v1Client})
+ })
}

func TestCollectRangePairs(t *testing.T) {

To view, visit change 476555. To unsubscribe, or for help writing mail filters, visit settings.

Gerrit-Project: pkgsite
Gerrit-Branch: master
Gerrit-Change-Id: Icd4491aeb98a7f7e3bf10301c71ec620cf5cdea8
Gerrit-Change-Number: 476555
Gerrit-PatchSet: 18
Gerrit-Owner: Tatiana Bradley <tatiana...@google.com>
Gerrit-Reviewer: Julie Qiu <juli...@google.com>
Gerrit-Reviewer: Tatiana Bradley <tatiana...@google.com>
Gerrit-Reviewer: kokoro <noreply...@google.com>
Gerrit-MessageType: merged
Reply all
Reply to author
Forward
0 new messages