[mod] sumdb: client and server protocol for checksum database
94 views
Skip to first unread message
Russ Cox (Gerrit)
May 10, 2019, 10:59:55 AM5/10/19
to Hyang-Ah Hana Kim, Ian Lance Taylor, goph...@pubsubhelper.golang.org, Russ Cox, golang-co...@googlegroups.com
Russ Cox would like Hyang-Ah Hana Kim to review this change.
sumdb: client and server protocol for checksum database
Copied from golang.org/x/exp/sumdb/internal/sumweb
with types cleaned up:
Conn -> Client
Client -> ClientOps
Handler -> Server
Server -> ServerOps
For golang/go#31761.
Change-Id: If0e004e6c9cab69c82de428810d67aba074aa843
---
A sumdb/cache.go
A sumdb/client.go
A sumdb/client_test.go
A sumdb/server.go
A sumdb/test.go
5 files changed, 1,503 insertions(+), 0 deletions(-)
diff --git a/sumdb/cache.go b/sumdb/cache.go
new file mode 100644
index 0000000..629e591
--- /dev/null
+++ b/sumdb/cache.go
@@ -0,0 +1,59 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Parallel cache.
+// This file is copied from cmd/go/internal/par.
+
+package sumdb
+
+import (
+ "sync"
+ "sync/atomic"
+)
+
+// parCache runs an action once per key and caches the result.
+type parCache struct {
+ m sync.Map
+}
+
+type cacheEntry struct {
+ done uint32
+ mu sync.Mutex
+ result interface{}
+}
+
+// Do calls the function f if and only if Do is being called for the first time with this key.
+// No call to Do with a given key returns until the one call to f returns.
+// Do returns the value returned by the one call to f.
+func (c *parCache) Do(key interface{}, f func() interface{}) interface{} {
+ entryIface, ok := c.m.Load(key)
+ if !ok {
+ entryIface, _ = c.m.LoadOrStore(key, new(cacheEntry))
+ }
+ e := entryIface.(*cacheEntry)
+ if atomic.LoadUint32(&e.done) == 0 {
+ e.mu.Lock()
+ if atomic.LoadUint32(&e.done) == 0 {
+ e.result = f()
+ atomic.StoreUint32(&e.done, 1)
+ }
+ e.mu.Unlock()
+ }
+ return e.result
+}
+
+// Get returns the cached result associated with key.
+// It returns nil if there is no such result.
+// If the result for key is being computed, Get does not wait for the computation to finish.
+func (c *parCache) Get(key interface{}) interface{} {
+ entryIface, ok := c.m.Load(key)
+ if !ok {
+ return nil
+ }
+ e := entryIface.(*cacheEntry)
+ if atomic.LoadUint32(&e.done) == 0 {
+ return nil
+ }
+ return e.result
+}
diff --git a/sumdb/client.go b/sumdb/client.go
new file mode 100644
index 0000000..f8da72b
--- /dev/null
+++ b/sumdb/client.go
@@ -0,0 +1,663 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package sumdb
+
+import (
+ "bytes"
+ "errors"
+ "fmt"
+ "path"
+ "strings"
+ "sync"
+ "sync/atomic"
+
+ "golang.org/x/mod/module"
+ "golang.org/x/mod/sumdb/note"
+ "golang.org/x/mod/sumdb/tlog"
+)
+
+// A ClientOps provides the external operations
+// (file caching, HTTP fetches, and so on) needed by the Client.
+// The methods must be safe for concurrent use by multiple goroutines.
+type ClientOps interface {
+ // ReadRemote reads and returns the content served at the given path
+ // on the remote database server. The path begins with "/lookup" or "/tile/".
+ // It is the implementation's responsibility to turn that path into a full URL
+ // and make the HTTP request. ReadRemote should return an error for
+ // any non-200 HTTP response status.
+ ReadRemote(path string) ([]byte, error)
+
+ // ReadConfig reads and returns the content of the named configuration file.
+ // There are only a fixed set of configuration files.
+ //
+ // "key" returns a file containing the verifier key for the server.
+ //
+ // serverName + "/latest" returns a file containing the latest known
+ // signed tree from the server. It is read and written (using WriteConfig).
+ // To signal that the client wishes to start with an "empty" signed tree,
+ // ReadConfig can return a successful empty result (0 bytes of data).
+ ReadConfig(file string) ([]byte, error)
+
+ // WriteConfig updates the content of the named configuration file,
+ // changing it from the old []byte to the new []byte.
+ // If the old []byte does not match the stored configuration,
+ // WriteConfig must return ErrWriteConflict.
+ // Otherwise, WriteConfig should atomically replace old with new.
+ WriteConfig(file string, old, new []byte) error
+
+ // ReadCache reads and returns the content of the named cache file.
+ // Any returned error will be treated as equivalent to the file not existing.
+ // There can be arbitrarily many cache files, such as:
+ // serverName/lookup/pkg@version
+ // serverName/tile/8/1/x123/456
+ ReadCache(file string) ([]byte, error)
+
+ // WriteCache writes the named cache file.
+ WriteCache(file string, data []byte)
+
+ // Log prints the given log message (such as with log.Print)
+ Log(msg string)
+
+ // SecurityError prints the given security error log message.
+ // The Client returns ErrSecurity from any operation that invokes SecurityError,
+ // but the return value is mainly for testing. In a real program,
+ // SecurityError should typically print the message and call log.Fatal or os.Exit.
+ SecurityError(msg string)
+}
+
+// ErrWriteConflict signals a write conflict during Client.WriteConfig.
+var ErrWriteConflict = errors.New("write conflict")
+
+// ErrSecurity is returned by Client operations that invoke Client.SecurityError.
+var ErrSecurity = errors.New("security error: misbehaving server")
+
+// A Client is a client connection to a checksum database.
+// All the methods are safe for simultaneous use by multiple goroutines.
+type Client struct {
+ ops ClientOps // access to operations in the external world
+
+ didLookup uint32
+
+ // one-time initialized data
+ initOnce sync.Once
+ initErr error // init error, if any
+ name string // name of accepted verifier
+ verifiers note.Verifiers // accepted verifiers (just one, but Verifiers for note.Open)
+ tileReader tileReader
+ tileHeight int
+ nosumdb string
+
+ record parCache // cache of record lookup, keyed by path@vers
+ tileCache parCache // cache of c.readTile, keyed by tile
+
+ latestMu sync.Mutex
+ latest tlog.Tree // latest known tree head
+ latestMsg []byte // encoded signed note for latest
+
+ tileSavedMu sync.Mutex
+ tileSaved map[tlog.Tile]bool // which tiles have been saved using c.ops.WriteCache already
+}
+
+// NewClient returns a new Client using the given Client.
+func NewClient(ops ClientOps) *Client {
+ return &Client{
+ ops: ops,
+ }
+}
+
+// init initiailzes the client (if not already initialized)
+// and returns any initialization error.
+func (c *Client) init() error {
+ c.initOnce.Do(c.initWork)
+ return c.initErr
+}
+
+// initWork does the actual initialization work.
+func (c *Client) initWork() {
+ defer func() {
+ if c.initErr != nil {
+ c.initErr = fmt.Errorf("initializing sumdb.Client: %v", c.initErr)
+ }
+ }()
+
+ c.tileReader.c = c
+ if c.tileHeight == 0 {
+ c.tileHeight = 8
+ }
+ c.tileSaved = make(map[tlog.Tile]bool)
+
+ vkey, err := c.ops.ReadConfig("key")
+ if err != nil {
+ c.initErr = err
+ return
+ }
+ verifier, err := note.NewVerifier(strings.TrimSpace(string(vkey)))
+ if err != nil {
+ c.initErr = err
+ return
+ }
+ c.verifiers = note.VerifierList(verifier)
+ c.name = verifier.Name()
+
+ data, err := c.ops.ReadConfig(c.name + "/latest")
+ if err != nil {
+ c.initErr = err
+ return
+ }
+ if err := c.mergeLatest(data); err != nil {
+ c.initErr = err
+ return
+ }
+}
+
+// SetTileHeight sets the tile height for the Client.
+// Any call to SetTileHeight must happen before the first call to Lookup.
+// If SetTileHeight is not called, the Client defaults to tile height 8.
+func (c *Client) SetTileHeight(height int) {
+ if atomic.LoadUint32(&c.didLookup) != 0 {
+ panic("SetTileHeight used after Lookup")
+ }
+ if c.tileHeight != 0 {
+ panic("multiple calls to SetTileHeight")
+ }
+ c.tileHeight = height
+}
+
+// SetGONOSUMDB sets the list of comma-separated GONOSUMDB patterns for the Client.
+// For any module path matching one of the patterns,
+// Lookup will return ErrGONOSUMDB.
+// Any call to SetGONOSUMDB must happen before the first call to Lookup.
+func (c *Client) SetGONOSUMDB(list string) {
+ if atomic.LoadUint32(&c.didLookup) != 0 {
+ panic("SetGONOSUMDB used after Lookup")
+ }
+ if c.nosumdb != "" {
+ panic("multiple calls to SetGONOSUMDB")
+ }
+ c.nosumdb = list
+}
+
+// ErrGONOSUMDB is returned by Lookup for paths that match
+// a pattern listed in the GONOSUMDB list (set by SetGONOSUMDB,
+// usually from the environment variable).
+var ErrGONOSUMDB = errors.New("skipped (listed in GONOSUMDB)")
+
+func (c *Client) skip(target string) bool {
+ return globsMatchPath(c.nosumdb, target)
+}
+
+// globsMatchPath reports whether any path prefix of target
+// matches one of the glob patterns (as defined by path.Match)
+// in the comma-separated globs list.
+// It ignores any empty or malformed patterns in the list.
+func globsMatchPath(globs, target string) bool {
+ for globs != "" {
+ // Extract next non-empty glob in comma-separated list.
+ var glob string
+ if i := strings.Index(globs, ","); i >= 0 {
+ glob, globs = globs[:i], globs[i+1:]
+ } else {
+ glob, globs = globs, ""
+ }
+ if glob == "" {
+ continue
+ }
+
+ // A glob with N+1 path elements (N slashes) needs to be matched
+ // against the first N+1 path elements of target,
+ // which end just before the N+1'th slash.
+ n := strings.Count(glob, "/")
+ prefix := target
+ // Walk target, counting slashes, truncating at the N+1'th slash.
+ for i := 0; i < len(target); i++ {
+ if target[i] == '/' {
+ if n == 0 {
+ prefix = target[:i]
+ break
+ }
+ n--
+ }
+ }
+ if n > 0 {
+ // Not enough prefix elements.
+ continue
+ }
+ matched, _ := path.Match(glob, prefix)
+ if matched {
+ return true
+ }
+ }
+ return false
+}
+
+// Lookup returns the go.sum lines for the given module path and version.
+// The version may end in a /go.mod suffix, in which case Lookup returns
+// the go.sum lines for the module's go.mod-only hash.
+func (c *Client) Lookup(path, vers string) (lines []string, err error) {
+ atomic.StoreUint32(&c.didLookup, 1)
+
+ if c.skip(path) {
+ return nil, ErrGONOSUMDB
+ }
+
+ defer func() {
+ if err != nil {
+ err = fmt.Errorf("%s@%s: %v", path, vers, err)
+ }
+ }()
+
+ if err := c.init(); err != nil {
+ return nil, err
+ }
+
+ // Prepare encoded cache filename / URL.
+ epath, err := module.EncodePath(path)
+ if err != nil {
+ return nil, err
+ }
+ evers, err := module.EncodeVersion(strings.TrimSuffix(vers, "/go.mod"))
+ if err != nil {
+ return nil, err
+ }
+ file := c.name + "/lookup/" + epath + "@" + evers
+ remotePath := "/lookup/" + epath + "@" + evers
+
+ // Fetch the data.
+ // The lookupCache avoids redundant ReadCache/GetURL operations
+ // (especially since go.sum lines tend to come in pairs for a given
+ // path and version) and also avoids having multiple of the same
+ // request in flight at once.
+ type cached struct {
+ data []byte
+ err error
+ }
+ result := c.record.Do(file, func() interface{} {
+ // Try the on-disk cache, or else get from web.
+ writeCache := false
+ data, err := c.ops.ReadCache(file)
+ if err != nil {
+ data, err = c.ops.ReadRemote(remotePath)
+ if err != nil {
+ return cached{nil, err}
+ }
+ writeCache = true
+ }
+
+ // Validate the record before using it for anything.
+ id, text, treeMsg, err := tlog.ParseRecord(data)
+ if err != nil {
+ return cached{nil, err}
+ }
+ if err := c.mergeLatest(treeMsg); err != nil {
+ return cached{nil, err}
+ }
+ if err := c.checkRecord(id, text); err != nil {
+ return cached{nil, err}
+ }
+
+ // Now that we've validated the record,
+ // save it to the on-disk cache (unless that's where it came from).
+ if writeCache {
+ c.ops.WriteCache(file, data)
+ }
+
+ return cached{data, nil}
+ }).(cached)
+ if result.err != nil {
+ return nil, result.err
+ }
+
+ // Extract the lines for the specific version we want
+ // (with or without /go.mod).
+ prefix := path + " " + vers + " "
+ var hashes []string
+ for _, line := range strings.Split(string(result.data), "\n") {
+ if strings.HasPrefix(line, prefix) {
+ hashes = append(hashes, line)
+ }
+ }
+ return hashes, nil
+}
+
+// mergeLatest merges the tree head in msg
+// with the Client's current latest tree head,
+// ensuring the result is a consistent timeline.
+// If the result is inconsistent, mergeLatest calls c.ops.SecurityError
+// with a detailed security error message and then
+// (only if c.ops.SecurityError does not exit the program) returns ErrSecurity.
+// If the Client's current latest tree head moves forward,
+// mergeLatest updates the underlying configuration file as well,
+// taking care to merge any independent updates to that configuration.
+func (c *Client) mergeLatest(msg []byte) error {
+ // Merge msg into our in-memory copy of the latest tree head.
+ when, err := c.mergeLatestMem(msg)
+ if err != nil {
+ return err
+ }
+ if when != msgFuture {
+ // msg matched our present or was in the past.
+ // No change to our present, so no update of config file.
+ return nil
+ }
+
+ // Flush our extended timeline back out to the configuration file.
+ // If the configuration file has been updated in the interim,
+ // we need to merge any updates made there as well.
+ // Note that writeConfig is an atomic compare-and-swap.
+ for {
+ msg, err := c.ops.ReadConfig(c.name + "/latest")
+ if err != nil {
+ return err
+ }
+ when, err := c.mergeLatestMem(msg)
+ if err != nil {
+ return err
+ }
+ if when != msgPast {
+ // msg matched our present or was from the future,
+ // and now our in-memory copy matches.
+ return nil
+ }
+
+ // msg (== config) is in the past, so we need to update it.
+ c.latestMu.Lock()
+ latestMsg := c.latestMsg
+ c.latestMu.Unlock()
+ if err := c.ops.WriteConfig(c.name+"/latest", msg, latestMsg); err != ErrWriteConflict {
+ // Success or a non-write-conflict error.
+ return err
+ }
+ }
+}
+
+const (
+ msgPast = 1 + iota
+ msgNow
+ msgFuture
+)
+
+// mergeLatestMem is like mergeLatest but is only concerned with
+// updating the in-memory copy of the latest tree head (c.latest)
+// not the configuration file.
+// The when result explains when msg happened relative to our
+// previous idea of c.latest:
+// msgPast means msg was from before c.latest,
+// msgNow means msg was exactly c.latest, and
+// msgFuture means msg was from after c.latest, which has now been updated.
+func (c *Client) mergeLatestMem(msg []byte) (when int, err error) {
+ if len(msg) == 0 {
+ // Accept empty msg as the unsigned, empty timeline.
+ c.latestMu.Lock()
+ latest := c.latest
+ c.latestMu.Unlock()
+ if latest.N == 0 {
+ return msgNow, nil
+ }
+ return msgPast, nil
+ }
+
+ note, err := note.Open(msg, c.verifiers)
+ if err != nil {
+ return 0, fmt.Errorf("reading tree note: %v\nnote:\n%s", err, msg)
+ }
+ tree, err := tlog.ParseTree([]byte(note.Text))
+ if err != nil {
+ return 0, fmt.Errorf("reading tree: %v\ntree:\n%s", err, note.Text)
+ }
+
+ // Other lookups may be calling mergeLatest with other heads,
+ // so c.latest is changing underfoot. We don't want to hold the
+ // c.mu lock during tile fetches, so loop trying to update c.latest.
+ c.latestMu.Lock()
+ latest := c.latest
+ latestMsg := c.latestMsg
+ c.latestMu.Unlock()
+
+ for {
+ // If the tree head looks old, check that it is on our timeline.
+ if tree.N <= latest.N {
+ if err := c.checkTrees(tree, msg, latest, latestMsg); err != nil {
+ return 0, err
+ }
+ if tree.N < latest.N {
+ return msgPast, nil
+ }
+ return msgNow, nil
+ }
+
+ // The tree head looks new. Check that we are on its timeline and try to move our timeline forward.
+ if err := c.checkTrees(latest, latestMsg, tree, msg); err != nil {
+ return 0, err
+ }
+
+ // Install our msg if possible.
+ // Otherwise we will go around again.
+ c.latestMu.Lock()
+ installed := false
+ if c.latest == latest {
+ installed = true
+ c.latest = tree
+ c.latestMsg = msg
+ } else {
+ latest = c.latest
+ latestMsg = c.latestMsg
+ }
+ c.latestMu.Unlock()
+
+ if installed {
+ return msgFuture, nil
+ }
+ }
+}
+
+// checkTrees checks that older (from olderNote) is contained in newer (from newerNote).
+// If an error occurs, such as malformed data or a network problem, checkTrees returns that error.
+// If on the other hand checkTrees finds evidence of misbehavior, it prepares a detailed
+// message and calls log.Fatal.
+func (c *Client) checkTrees(older tlog.Tree, olderNote []byte, newer tlog.Tree, newerNote []byte) error {
+ thr := tlog.TileHashReader(newer, &c.tileReader)
+ h, err := tlog.TreeHash(older.N, thr)
+ if err != nil {
+ if older.N == newer.N {
+ return fmt.Errorf("checking tree#%d: %v", older.N, err)
+ }
+ return fmt.Errorf("checking tree#%d against tree#%d: %v", older.N, newer.N, err)
+ }
+ if h == older.Hash {
+ return nil
+ }
+
+ // Detected a fork in the tree timeline.
+ // Start by reporting the inconsistent signed tree notes.
+ var buf bytes.Buffer
+ fmt.Fprintf(&buf, "SECURITY ERROR\n")
+ fmt.Fprintf(&buf, "go.sum database server misbehavior detected!\n\n")
+ indent := func(b []byte) []byte {
+ return bytes.Replace(b, []byte("\n"), []byte("\n\t"), -1)
+ }
+ fmt.Fprintf(&buf, "old database:\n\t%s\n", indent(olderNote))
+ fmt.Fprintf(&buf, "new database:\n\t%s\n", indent(newerNote))
+
+ // The notes alone are not enough to prove the inconsistency.
+ // We also need to show that the newer note's tree hash for older.N
+ // does not match older.Hash. The consumer of this report could
+ // of course consult the server to try to verify the inconsistency,
+ // but we are holding all the bits we need to prove it right now,
+ // so we might as well print them and make the report not depend
+ // on the continued availability of the misbehaving server.
+ // Preparing this data only reuses the tiled hashes needed for
+ // tlog.TreeHash(older.N, thr) above, so assuming thr is caching tiles,
+ // there are no new access to the server here, and these operations cannot fail.
+ fmt.Fprintf(&buf, "proof of misbehavior:\n\t%v", h)
+ if p, err := tlog.ProveTree(newer.N, older.N, thr); err != nil {
+ fmt.Fprintf(&buf, "\tinternal error: %v\n", err)
+ } else if err := tlog.CheckTree(p, newer.N, newer.Hash, older.N, h); err != nil {
+ fmt.Fprintf(&buf, "\tinternal error: generated inconsistent proof\n")
+ } else {
+ for _, h := range p {
+ fmt.Fprintf(&buf, "\n\t%v", h)
+ }
+ }
+ c.ops.SecurityError(buf.String())
+ return ErrSecurity
+}
+
+// checkRecord checks that record #id's hash matches data.
+func (c *Client) checkRecord(id int64, data []byte) error {
+ c.latestMu.Lock()
+ latest := c.latest
+ c.latestMu.Unlock()
+
+ if id >= latest.N {
+ return fmt.Errorf("cannot validate record %d in tree of size %d", id, latest.N)
+ }
+ hashes, err := tlog.TileHashReader(latest, &c.tileReader).ReadHashes([]int64{tlog.StoredHashIndex(0, id)})
+ if err != nil {
+ return err
+ }
+ if hashes[0] == tlog.RecordHash(data) {
+ return nil
+ }
+ return fmt.Errorf("cannot authenticate record data in server response")
+}
+
+// tileReader is a *Client wrapper that implements tlog.TileReader.
+// The separate type avoids exposing the ReadTiles and SaveTiles
+// methods on Client itself.
+type tileReader struct {
+ c *Client
+}
+
+func (r *tileReader) Height() int {
+ return r.c.tileHeight
+}
+
+// ReadTiles reads and returns the requested tiles,
+// either from the on-disk cache or the server.
+func (r *tileReader) ReadTiles(tiles []tlog.Tile) ([][]byte, error) {
+ // Read all the tiles in parallel.
+ data := make([][]byte, len(tiles))
+ errs := make([]error, len(tiles))
+ var wg sync.WaitGroup
+ for i, tile := range tiles {
+ wg.Add(1)
+ go func(i int, tile tlog.Tile) {
+ defer wg.Done()
+ data[i], errs[i] = r.c.readTile(tile)
+ }(i, tile)
+ }
+ wg.Wait()
+
+ for _, err := range errs {
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ return data, nil
+}
+
+// tileCacheKey returns the cache key for the tile.
+func (c *Client) tileCacheKey(tile tlog.Tile) string {
+ return c.name + "/" + tile.Path()
+}
+
+// tileRemotePath returns the remote path for the tile.
+func (c *Client) tileRemotePath(tile tlog.Tile) string {
+ return "/" + tile.Path()
+}
+
+// readTile reads a single tile, either from the on-disk cache or the server.
+func (c *Client) readTile(tile tlog.Tile) ([]byte, error) {
+ type cached struct {
+ data []byte
+ err error
+ }
+
+ result := c.tileCache.Do(tile, func() interface{} {
+ // Try the requested tile in on-disk cache.
+ data, err := c.ops.ReadCache(c.tileCacheKey(tile))
+ if err == nil {
+ c.markTileSaved(tile)
+ return cached{data, nil}
+ }
+
+ // Try the full tile in on-disk cache (if requested tile not already full).
+ // We only save authenticated tiles to the on-disk cache,
+ // so the recreated prefix is equally authenticated.
+ full := tile
+ full.W = 1 << uint(tile.H)
+ if tile != full {
+ data, err := c.ops.ReadCache(c.tileCacheKey(full))
+ if err == nil {
+ c.markTileSaved(tile) // don't save tile later; we already have full
+ return cached{data[:len(data)/full.W*tile.W], nil}
+ }
+ }
+
+ // Try requested tile from server.
+ data, err = c.ops.ReadRemote(c.tileRemotePath(tile))
+ if err == nil {
+ return cached{data, nil}
+ }
+
+ // Try full tile on server.
+ // If the partial tile does not exist, it should be because
+ // the tile has been completed and only the complete one
+ // is available.
+ if tile != full {
+ data, err := c.ops.ReadRemote(c.tileRemotePath(full))
+ if err == nil {
+ // Note: We could save the full tile in the on-disk cache here,
+ // but we don't know if it is valid yet, and we will only find out
+ // about the partial data, not the full data. So let SaveTiles
+ // save the partial tile, and we'll just refetch the full tile later
+ // once we can validate more (or all) of it.
+ return cached{data[:len(data)/full.W*tile.W], nil}
+ }
+ }
+
+ // Nothing worked.
+ // Return the error from the server fetch for the requested (not full) tile.
+ return cached{nil, err}
+ }).(cached)
+
+ return result.data, result.err
+}
+
+// markTileSaved records that tile is already present in the on-disk cache,
+// so that a future SaveTiles for that tile can be ignored.
+func (c *Client) markTileSaved(tile tlog.Tile) {
+ c.tileSavedMu.Lock()
+ c.tileSaved[tile] = true
+ c.tileSavedMu.Unlock()
+}
+
+// SaveTiles saves the now validated tiles.
+func (r *tileReader) SaveTiles(tiles []tlog.Tile, data [][]byte) {
+ c := r.c
+
+ // Determine which tiles need saving.
+ // (Tiles that came from the cache need not be saved back.)
+ save := make([]bool, len(tiles))
+ c.tileSavedMu.Lock()
+ for i, tile := range tiles {
+ if !c.tileSaved[tile] {
+ save[i] = true
+ c.tileSaved[tile] = true
+ }
+ }
+ c.tileSavedMu.Unlock()
+
+ for i, tile := range tiles {
+ if save[i] {
+ // If WriteCache fails here (out of disk space? i/o error?),
+ // c.tileSaved[tile] is still true and we will not try to write it again.
+ // Next time we run maybe we'll redownload it again and be
+ // more successful.
+ c.ops.WriteCache(c.name+"/"+tile.Path(), data[i])
+ }
+ }
+}
diff --git a/sumdb/client_test.go b/sumdb/client_test.go
new file mode 100644
index 0000000..0f3c481
--- /dev/null
+++ b/sumdb/client_test.go
@@ -0,0 +1,460 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package sumdb
+
+import (
+ "bytes"
+ "fmt"
+ "strings"
+ "sync"
+ "testing"
+
+ "golang.org/x/mod/sumdb/note"
+ "golang.org/x/mod/sumdb/tlog"
+)
+
+const (
+ testName = "localhost.localdev/sumdb"
+ testVerifierKey = "localhost.localdev/sumdb+00000c67+AcTrnkbUA+TU4heY3hkjiSES/DSQniBqIeQ/YppAUtK6"
+ testSignerKey = "PRIVATE+KEY+localhost.localdev/sumdb+00000c67+AXu6+oaVaOYuQOFrf1V59JK1owcFlJcHwwXHDfDGxSPk"
+)
+
+func TestClientLookup(t *testing.T) {
+ tc := newTestClient(t)
+ tc.mustHaveLatest(1)
+
+ // Basic lookup.
+ tc.mustLookup("rsc.io/sampler", "v1.3.0", "rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4=")
+ tc.mustHaveLatest(3)
+
+ // Everything should now be cached, both for the original package and its /go.mod.
+ tc.getOK = false
+ tc.mustLookup("rsc.io/sampler", "v1.3.0", "rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4=")
+ tc.mustLookup("rsc.io/sampler", "v1.3.0/go.mod", "rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=")
+ tc.mustHaveLatest(3)
+ tc.getOK = true
+ tc.getTileOK = false // the cache has what we need
+
+ // Lookup with multiple returned lines.
+ tc.mustLookup("rsc.io/quote", "v1.5.2", "rsc.io/quote v1.5.2 h1:w5fcysjrx7yqtD/aO+QwRjYZOKnaM9Uh2b40tElTs3Y=\nrsc.io/quote v1.5.2 h2:xyzzy")
+ tc.mustHaveLatest(3)
+
+ // Lookup with need for !-encoding.
+ // rsc.io/Quote is the only record written after rsc.io/samper,
+ // so it is the only one that should need more tiles.
+ tc.getTileOK = true
+ tc.mustLookup("rsc.io/Quote", "v1.5.2", "rsc.io/Quote v1.5.2 h1:uppercase!=")
+ tc.mustHaveLatest(4)
+}
+
+func TestClientBadTiles(t *testing.T) {
+ tc := newTestClient(t)
+
+ flipBits := func() {
+ for url, data := range tc.remote {
+ if strings.Contains(url, "/tile/") {
+ for i := range data {
+ data[i] ^= 0x80
+ }
+ }
+ }
+ }
+
+ // Bad tiles in initial download.
+ tc.mustHaveLatest(1)
+ flipBits()
+ _, err := tc.client.Lookup("rsc.io/sampler", "v1.3.0")
+ tc.mustError(err, "rsc.io/sam...@v1.3.0: initializing sumdb.Client: checking tree#1: downloaded inconsistent tile")
+ flipBits()
+ tc.newClient()
+ tc.mustLookup("rsc.io/sampler", "v1.3.0", "rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4=")
+
+ // Bad tiles after initial download.
+ flipBits()
+ _, err = tc.client.Lookup("rsc.io/Quote", "v1.5.2")
+ tc.mustError(err, "rsc.io/Qu...@v1.5.2: checking tree#3 against tree#4: downloaded inconsistent tile")
+ flipBits()
+ tc.newClient()
+ tc.mustLookup("rsc.io/Quote", "v1.5.2", "rsc.io/Quote v1.5.2 h1:uppercase!=")
+
+ // Bad starting tree hash looks like bad tiles.
+ tc.newClient()
+ text := tlog.FormatTree(tlog.Tree{N: 1, Hash: tlog.Hash{}})
+ data, err := note.Sign(¬e.Note{Text: string(text)}, tc.signer)
+ if err != nil {
+ tc.t.Fatal(err)
+ }
+ tc.config[testName+"/latest"] = data
+ _, err = tc.client.Lookup("rsc.io/sampler", "v1.3.0")
+ tc.mustError(err, "rsc.io/sam...@v1.3.0: initializing sumdb.Client: checking tree#1: downloaded inconsistent tile")
+}
+
+func TestClientFork(t *testing.T) {
+ tc := newTestClient(t)
+ tc2 := tc.fork()
+
+ tc.addRecord("rsc.io/pk...@v1.5.2", `rsc.io/pkg1 v1.5.2 h1:hash!=
+`)
+ tc.addRecord("rsc.io/pk...@v1.5.4", `rsc.io/pkg1 v1.5.4 h1:hash!=
+`)
+ tc.mustLookup("rsc.io/pkg1", "v1.5.2", "rsc.io/pkg1 v1.5.2 h1:hash!=")
+
+ tc2.addRecord("rsc.io/pk...@v1.5.3", `rsc.io/pkg1 v1.5.3 h1:hash!=
+`)
+ tc2.addRecord("rsc.io/pk...@v1.5.4", `rsc.io/pkg1 v1.5.4 h1:hash!=
+`)
+ tc2.mustLookup("rsc.io/pkg1", "v1.5.4", "rsc.io/pkg1 v1.5.4 h1:hash!=")
+
+ key := "/lookup/rsc.io/pk...@v1.5.2"
+ tc2.remote[key] = tc.remote[key]
+ _, err := tc2.client.Lookup("rsc.io/pkg1", "v1.5.2")
+ tc2.mustError(err, ErrSecurity.Error())
+
+ /*
+ SECURITY ERROR
+ go.sum database server misbehavior detected!
+
+ old database:
+ go.sum database tree!
+ 5
+ nWzN20+pwMt62p7jbv1/NlN95ePTlHijabv5zO/s36w=
+
+ — localhost.localdev/sumdb AAAMZ5/2FVAdMH58kmnz/0h299pwyskEbzDzoa2/YaPdhvLya4YWDFQQxu2TQb5GpwAH4NdWnTwuhILafisyf3CNbgg=
+
+ new database:
+ go.sum database tree
+ 6
+ wc4SkQt52o5W2nQ8To2ARs+mWuUJjss+sdleoiqxMmM=
+
+ — localhost.localdev/sumdb AAAMZ6oRNswlEZ6ZZhxrCvgl1MBy+nusq4JU+TG6Fe2NihWLqOzb+y2c2kzRLoCr4tvw9o36ucQEnhc20e4nA4Qc/wc=
+
+ proof of misbehavior:
+ T7i+H/8ER4nXOiw4Bj0koZOkGjkxoNvlI34GpvhHhQg=
+ Nsuejv72de9hYNM5bqFv8rv3gm3zJQwv/DT/WNbLDLA=
+ mOmqqZ1aI/lzS94oq/JSbj7pD8Rv9S+xDyi12BtVSHo=
+ /7Aw5jVSMM9sFjQhaMg+iiDYPMk6decH7QLOGrL9Lx0=
+ */
+
+ wants := []string{
+ "SECURITY ERROR",
+ "go.sum database server misbehavior detected!",
+ "old database:\n\tgo.sum database tree\n\t5\n",
+ "— localhost.localdev/sumdb AAAMZ5/2FVAd",
+ "new database:\n\tgo.sum database tree\n\t6\n",
+ "— localhost.localdev/sumdb AAAMZ6oRNswl",
+ "proof of misbehavior:\n\tT7i+H/8ER4nXOiw4Bj0k",
+ }
+ text := tc2.security.String()
+ for _, want := range wants {
+ if !strings.Contains(text, want) {
+ t.Fatalf("cannot find %q in security text:\n%s", want, text)
+ }
+ }
+}
+
+func TestClientGONOSUMDB(t *testing.T) {
+ tc := newTestClient(t)
+ tc.client.SetGONOSUMDB("p,*/q")
+ tc.client.Lookup("rsc.io/sampler", "v1.3.0") // initialize before we turn off network
+ tc.getOK = false
+
+ ok := []string{
+ "abc",
+ "a/p",
+ "pq",
+ "q",
+ "n/o/p/q",
+ }
+ skip := []string{
+ "p",
+ "p/x",
+ "x/q",
+ "x/q/z",
+ }
+
+ for _, path := range ok {
+ _, err := tc.client.Lookup(path, "v1.0.0")
+ if err == ErrGONOSUMDB {
+ t.Errorf("Lookup(%q): ErrGONOSUMDB, wanted failed actual lookup", path)
+ }
+ }
+ for _, path := range skip {
+ _, err := tc.client.Lookup(path, "v1.0.0")
+ if err != ErrGONOSUMDB {
+ t.Errorf("Lookup(%q): %v, wanted ErrGONOSUMDB", path, err)
+ }
+ }
+}
+
+// A testClient is a self-contained client-side testing environment.
+type testClient struct {
+ t *testing.T // active test
+ client *Client // client being tested
+ tileHeight int // tile height to use (default 2)
+ getOK bool // should tc.GetURL succeed?
+ getTileOK bool // should tc.GetURL of tiles succeed?
+ treeSize int64
+ hashes []tlog.Hash
+ remote map[string][]byte
+ signer note.Signer
+
+ // mu protects config, cache, log, security
+ // during concurrent use of the exported methods
+ // by the client itself (testClient is the Client's ClientOps,
+ // and the Client methods can both read and write these fields).
+ // Unexported methods invoked directly by the test
+ // (for example, addRecord) need not hold the mutex:
+ // for proper test execution those methods should only
+ // be called when the Client is idle and not using its ClientOps.
+ // Not holding the mutex in those methods ensures
+ // that if a mistake is made, go test -race will report it.
+ // (Holding the mutex would eliminate the race report but
+ // not the underlying problem.)
+ // Similarly, the get map is not protected by the mutex,
+ // because the Client methods only read it.
+ mu sync.Mutex // prot
+ config map[string][]byte
+ cache map[string][]byte
+ security bytes.Buffer
+}
+
+// newTestClient returns a new testClient that will call t.Fatal on error
+// and has a few records already available on the remote server.
+func newTestClient(t *testing.T) *testClient {
+ tc := &testClient{
+ t: t,
+ tileHeight: 2,
+ getOK: true,
+ getTileOK: true,
+ config: make(map[string][]byte),
+ cache: make(map[string][]byte),
+ remote: make(map[string][]byte),
+ }
+
+ tc.config["key"] = []byte(testVerifierKey + "\n")
+ var err error
+ tc.signer, err = note.NewSigner(testSignerKey)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ tc.newClient()
+
+ tc.addRecord("rsc.io/qu...@v1.5.2", `rsc.io/quote v1.5.2 h1:w5fcysjrx7yqtD/aO+QwRjYZOKnaM9Uh2b40tElTs3Y=
+rsc.io/quote v1.5.2/go.mod h1:LzX7hefJvL54yjefDEDHNONDjII0t9xZLPXsUe+TKr0=
+rsc.io/quote v1.5.2 h2:xyzzy
+`)
+
+ tc.addRecord("golang.org/x/te...@v0.0.0-20170915032832-14c0d48ead0c", `golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c h1:qgOY6WgZOaTkIIMiVjBQcw93ERBE4m30iBm00nkL0i8=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+`)
+ tc.addRecord("rsc.io/sam...@v1.3.0", `rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+`)
+ tc.config[testName+"/latest"] = tc.signTree(1)
+
+ tc.addRecord("rsc.io/!qu...@v1.5.2", `rsc.io/Quote v1.5.2 h1:uppercase!=
+`)
+ return tc
+}
+
+// newClient resets the Client associated with tc.
+// This clears any in-memory cache from the Client
+// but not tc's on-disk cache.
+func (tc *testClient) newClient() {
+ tc.client = NewClient(tc)
+ tc.client.SetTileHeight(tc.tileHeight)
+}
+
+// mustLookup does a lookup for path@vers and checks that the lines that come back match want.
+func (tc *testClient) mustLookup(path, vers, want string) {
+ tc.t.Helper()
+ lines, err := tc.client.Lookup(path, vers)
+ if err != nil {
+ tc.t.Fatal(err)
+ }
+ if strings.Join(lines, "\n") != want {
+ tc.t.Fatalf("Lookup(%q, %q):\n\t%s\nwant:\n\t%s", path, vers, strings.Join(lines, "\n\t"), strings.Replace(want, "\n", "\n\t", -1))
+ }
+}
+
+// mustHaveLatest checks that the on-disk configuration
+// for latest is a tree of size n.
+func (tc *testClient) mustHaveLatest(n int64) {
+ tc.t.Helper()
+
+ latest := tc.config[testName+"/latest"]
+ lines := strings.Split(string(latest), "\n")
+ if len(lines) < 2 || lines[1] != fmt.Sprint(n) {
+ tc.t.Fatalf("/latest should have tree %d, but has:\n%s", n, latest)
+ }
+}
+
+// mustError checks that err's error string contains the text.
+func (tc *testClient) mustError(err error, text string) {
+ tc.t.Helper()
+ if err == nil || !strings.Contains(err.Error(), text) {
+ tc.t.Fatalf("err = %v, want %q", err, text)
+ }
+}
+
+// fork returns a copy of tc.
+// Changes made to the new copy or to tc are not reflected in the other.
+func (tc *testClient) fork() *testClient {
+ tc2 := &testClient{
+ t: tc.t,
+ getOK: tc.getOK,
+ getTileOK: tc.getTileOK,
+ tileHeight: tc.tileHeight,
+ treeSize: tc.treeSize,
+ hashes: append([]tlog.Hash{}, tc.hashes...),
+ signer: tc.signer,
+ config: copyMap(tc.config),
+ cache: copyMap(tc.cache),
+ remote: copyMap(tc.remote),
+ }
+ tc2.newClient()
+ return tc2
+}
+
+func copyMap(m map[string][]byte) map[string][]byte {
+ m2 := make(map[string][]byte)
+ for k, v := range m {
+ m2[k] = v
+ }
+ return m2
+}
+
+// ReadHashes is tc's implementation of tlog.HashReader, for use with
+// tlog.TreeHash and so on.
+func (tc *testClient) ReadHashes(indexes []int64) ([]tlog.Hash, error) {
+ var list []tlog.Hash
+ for _, id := range indexes {
+ list = append(list, tc.hashes[id])
+ }
+ return list, nil
+}
+
+// addRecord adds a log record using the given (!-encoded) key and data.
+func (tc *testClient) addRecord(key, data string) {
+ tc.t.Helper()
+
+ // Create record, add hashes to log tree.
+ id := tc.treeSize
+ tc.treeSize++
+ rec, err := tlog.FormatRecord(id, []byte(data))
+ if err != nil {
+ tc.t.Fatal(err)
+ }
+ hashes, err := tlog.StoredHashesForRecordHash(id, tlog.RecordHash([]byte(data)), tc)
+ if err != nil {
+ tc.t.Fatal(err)
+ }
+ tc.hashes = append(tc.hashes, hashes...)
+
+ // Create lookup result.
+ tc.remote["/lookup/"+key] = append(rec, tc.signTree(tc.treeSize)...)
+
+ // Create new tiles.
+ tiles := tlog.NewTiles(tc.tileHeight, id, tc.treeSize)
+ for _, tile := range tiles {
+ data, err := tlog.ReadTileData(tile, tc)
+ if err != nil {
+ tc.t.Fatal(err)
+ }
+ tc.remote["/"+tile.Path()] = data
+ // TODO delete old partial tiles
+ }
+}
+
+// signTree returns the signed head for the tree of the given size.
+func (tc *testClient) signTree(size int64) []byte {
+ h, err := tlog.TreeHash(size, tc)
+ if err != nil {
+ tc.t.Fatal(err)
+ }
+ text := tlog.FormatTree(tlog.Tree{N: size, Hash: h})
+ data, err := note.Sign(¬e.Note{Text: string(text)}, tc.signer)
+ if err != nil {
+ tc.t.Fatal(err)
+ }
+ return data
+}
+
+// ReadRemote is for tc's implementation of Client.
+func (tc *testClient) ReadRemote(path string) ([]byte, error) {
+ // No mutex here because only the Client should be running
+ // and the Client cannot change tc.get.
+ if !tc.getOK {
+ return nil, fmt.Errorf("disallowed remote read %s", path)
+ }
+ if strings.Contains(path, "/tile/") && !tc.getTileOK {
+ return nil, fmt.Errorf("disallowed remote tile read %s", path)
+ }
+
+ data, ok := tc.remote[path]
+ if !ok {
+ return nil, fmt.Errorf("no remote path %s", path)
+ }
+ return data, nil
+}
+
+// ReadConfig is for tc's implementation of Client.
+func (tc *testClient) ReadConfig(file string) ([]byte, error) {
+ tc.mu.Lock()
+ defer tc.mu.Unlock()
+
+ data, ok := tc.config[file]
+ if !ok {
+ return nil, fmt.Errorf("no config %s", file)
+ }
+ return data, nil
+}
+
+// WriteConfig is for tc's implementation of Client.
+func (tc *testClient) WriteConfig(file string, old, new []byte) error {
+ tc.mu.Lock()
+ defer tc.mu.Unlock()
+
+ data := tc.config[file]
+ if !bytes.Equal(old, data) {
+ return ErrWriteConflict
+ }
+ tc.config[file] = new
+ return nil
+}
+
+// ReadCache is for tc's implementation of Client.
+func (tc *testClient) ReadCache(file string) ([]byte, error) {
+ tc.mu.Lock()
+ defer tc.mu.Unlock()
+
+ data, ok := tc.cache[file]
+ if !ok {
+ return nil, fmt.Errorf("no cache %s", file)
+ }
+ return data, nil
+}
+
+// WriteCache is for tc's implementation of Client.
+func (tc *testClient) WriteCache(file string, data []byte) {
+ tc.mu.Lock()
+ defer tc.mu.Unlock()
+
+ tc.cache[file] = data
+}
+
+// Log is for tc's implementation of Client.
+func (tc *testClient) Log(msg string) {
+ tc.t.Log(msg)
+}
+
+// SecurityError is for tc's implementation of Client.
+func (tc *testClient) SecurityError(msg string) {
+ tc.mu.Lock()
+ defer tc.mu.Unlock()
+
+ fmt.Fprintf(&tc.security, "%s\n", strings.TrimRight(msg, "\n"))
+}
diff --git a/sumdb/server.go b/sumdb/server.go
new file mode 100644
index 0000000..f9c3208
--- /dev/null
+++ b/sumdb/server.go
@@ -0,0 +1,188 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package sumdb implements the HTTP protocols for serving or accessing a module checksum database.
+package sumdb
+
+import (
+ "context"
+ "net/http"
+ "os"
+ "regexp"
+ "strings"
+
+ "golang.org/x/mod/module"
+ "golang.org/x/mod/sumdb/tlog"
+)
+
+// A ServerOps provides the external operations
+// (underlying database access and so on) needed by the Server.
+type ServerOps interface {
+ // NewContext returns the context to use for the request r.
+ NewContext(r *http.Request) (context.Context, error)
+
+ // Signed returns the signed hash of the latest tree.
+ Signed(ctx context.Context) ([]byte, error)
+
+ // ReadRecords returns the content for the n records id through id+n-1.
+ ReadRecords(ctx context.Context, id, n int64) ([][]byte, error)
+
+ // Lookup looks up a record by its associated key ("module@version"),
+ // returning the record ID.
+ Lookup(ctx context.Context, key string) (int64, error)
+
+ // ReadTileData reads the content of tile t.
+ // It is only invoked for hash tiles (t.L ≥ 0).
+ ReadTileData(ctx context.Context, t tlog.Tile) ([]byte, error)
+}
+
+// A Server is the checksum database HTTP server,
+// which implements http.Handler and should be invoked
+// to serve the paths listed in ServerPaths.
+type Server struct {
+ ops ServerOps
+}
+
+// NewServer returns a new Server using the given operations.
+func NewServer(ops ServerOps) *Server {
+ return &Server{ops: ops}
+}
+
+// ServerPaths are the URL paths the Server can (and should) serve.
+//
+// Typically a server will do:
+//
+// srv := sumdb.NewServer(ops)
+// for _, path := range sumdb.ServerPaths {
+// http.Handle(path, srv)
+// }
+//
+var ServerPaths = []string{
+ "/lookup/",
+ "/latest",
+ "/tile/",
+}
+
+var modVerRE = regexp.MustCompile(`^[^@]+@v[0-9]+\.[0-9]+\.[0-9]+(-[^@]*)?(\+incompatible)?$`)
+
+func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ ctx, err := s.ops.NewContext(r)
+ if err != nil {
+ http.Error(w, err.Error(), 500)
+ return
+ }
+
+ switch {
+ default:
+ http.NotFound(w, r)
+
+ case strings.HasPrefix(r.URL.Path, "/lookup/"):
+ mod := strings.TrimPrefix(r.URL.Path, "/lookup/")
+ if !modVerRE.MatchString(mod) {
+ http.Error(w, "invalid module@version syntax", http.StatusBadRequest)
+ return
+ }
+ i := strings.Index(mod, "@")
+ encPath, encVers := mod[:i], mod[i+1:]
+ path, err := module.DecodePath(encPath)
+ if err != nil {
+ reportError(w, r, err)
+ return
+ }
+ vers, err := module.DecodeVersion(encVers)
+ if err != nil {
+ reportError(w, r, err)
+ return
+ }
+ id, err := s.ops.Lookup(ctx, path+"@"+vers)
+ if err != nil {
+ reportError(w, r, err)
+ return
+ }
+ records, err := s.ops.ReadRecords(ctx, id, 1)
+ if err != nil {
+ // This should never happen - the lookup says the record exists.
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ if len(records) != 1 {
+ http.Error(w, "invalid record count returned by ReadRecords", http.StatusInternalServerError)
+ return
+ }
+ msg, err := tlog.FormatRecord(id, records[0])
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ signed, err := s.ops.Signed(ctx)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ w.Header().Set("Content-Type", "text/plain; charset=UTF-8")
+ w.Write(msg)
+ w.Write(signed)
+
+ case r.URL.Path == "/latest":
+ data, err := s.ops.Signed(ctx)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ w.Header().Set("Content-Type", "text/plain; charset=UTF-8")
+ w.Write(data)
+
+ case strings.HasPrefix(r.URL.Path, "/tile/"):
+ t, err := tlog.ParseTilePath(r.URL.Path[1:])
+ if err != nil {
+ http.Error(w, "invalid tile syntax", http.StatusBadRequest)
+ return
+ }
+ if t.L == -1 {
+ // Record data.
+ start := t.N << uint(t.H)
+ records, err := s.ops.ReadRecords(ctx, start, int64(t.W))
+ if err != nil {
+ reportError(w, r, err)
+ return
+ }
+ if len(records) != t.W {
+ http.Error(w, "invalid record count returned by ReadRecords", http.StatusInternalServerError)
+ return
+ }
+ var data []byte
+ for i, text := range records {
+ msg, err := tlog.FormatRecord(start+int64(i), text)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ }
+ data = append(data, msg...)
+ }
+ w.Header().Set("Content-Type", "text/plain; charset=UTF-8")
+ w.Write(data)
+ return
+ }
+
+ data, err := s.ops.ReadTileData(ctx, t)
+ if err != nil {
+ reportError(w, r, err)
+ return
+ }
+ w.Header().Set("Content-Type", "application/octet-stream")
+ w.Write(data)
+ }
+}
+
+// reportError reports err to w.
+// If it's a not-found, the reported error is 404.
+// Otherwise it is an internal server error.
+// The caller must only call reportError in contexts where
+// a not-found err should be reported as 404.
+func reportError(w http.ResponseWriter, r *http.Request, err error) {
+ if os.IsNotExist(err) {
+ http.Error(w, err.Error(), http.StatusNotFound)
+ return
+ }
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+}
diff --git a/sumdb/test.go b/sumdb/test.go
new file mode 100644
index 0000000..d293a70
--- /dev/null
+++ b/sumdb/test.go
@@ -0,0 +1,133 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package sumdb
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "strings"
+ "sync"
+
+ "golang.org/x/mod/sumdb/note"
+ "golang.org/x/mod/sumdb/tlog"
+)
+
+// NewTestServer constructs a new TestServer
+// that will sign its tree with the given signer key
+// (see golang.org/x/mod/sumdb/note)
+// and fetch new records as needed by calling gosum.
+func NewTestServer(signer string, gosum func(path, vers string) ([]byte, error)) *TestServer {
+ return &TestServer{signer: signer, gosum: gosum}
+}
+
+// A TestServer is an in-memory implementation of Server for testing.
+type TestServer struct {
+ signer string
+ gosum func(path, vers string) ([]byte, error)
+
+ mu sync.Mutex
+ hashes testHashes
+ records [][]byte
+ lookup map[string]int64
+}
+
+// testHashes implements tlog.HashReader, reading from a slice.
+type testHashes []tlog.Hash
+
+func (h testHashes) ReadHashes(indexes []int64) ([]tlog.Hash, error) {
+ var list []tlog.Hash
+ for _, id := range indexes {
+ list = append(list, h[id])
+ }
+ return list, nil
+}
+
+func (s *TestServer) NewContext(r *http.Request) (context.Context, error) {
+ return nil, nil
+}
+
+func (s *TestServer) Signed(ctx context.Context) ([]byte, error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ size := int64(len(s.records))
+ h, err := tlog.TreeHash(size, s.hashes)
+ if err != nil {
+ return nil, err
+ }
+ text := tlog.FormatTree(tlog.Tree{N: size, Hash: h})
+ signer, err := note.NewSigner(s.signer)
+ if err != nil {
+ return nil, err
+ }
+ return note.Sign(¬e.Note{Text: string(text)}, signer)
+}
+
+func (s *TestServer) ReadRecords(ctx context.Context, id, n int64) ([][]byte, error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ var list [][]byte
+ for i := int64(0); i < n; i++ {
+ if id+i >= int64(len(s.records)) {
+ return nil, fmt.Errorf("missing records")
+ }
+ list = append(list, s.records[id+i])
+ }
+ return list, nil
+}
+
+func (s *TestServer) Lookup(ctx context.Context, key string) (int64, error) {
+ s.mu.Lock()
+ id, ok := s.lookup[key]
+ s.mu.Unlock()
+ if ok {
+ return id, nil
+ }
+
+ // Look up module and compute go.sum lines.
+ i := strings.Index(key, "@")
+ if i < 0 {
+ return 0, fmt.Errorf("invalid lookup key %q", key)
+ }
+ path, vers := key[:i], key[i+1:]
+ data, err := s.gosum(path, vers)
+ if err != nil {
+ return 0, err
+ }
+
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ // We ran the fetch without the lock.
+ // If another fetch happened and committed, use it instead.
+ id, ok = s.lookup[key]
+ if ok {
+ return id, nil
+ }
+
+ // Add record.
+ id = int64(len(s.records))
+ s.records = append(s.records, data)
+ if s.lookup == nil {
+ s.lookup = make(map[string]int64)
+ }
+ s.lookup[key] = id
+ hashes, err := tlog.StoredHashesForRecordHash(id, tlog.RecordHash([]byte(data)), s.hashes)
+ if err != nil {
+ panic(err)
+ }
+ s.hashes = append(s.hashes, hashes...)
+
+ return id, nil
+}
+
+func (s *TestServer) ReadTileData(ctx context.Context, t tlog.Tile) ([]byte, error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ return tlog.ReadTileData(t, s.hashes)
+}
To view, visit change 176466. To unsubscribe, or for help writing mail filters, visit settings.
Gobot Gobot (Gerrit)
May 10, 2019, 11:01:19 AM5/10/19
to Russ Cox, goph...@pubsubhelper.golang.org, Hyang-Ah Hana Kim, golang-co...@googlegroups.com
TryBots beginning. Status page: https://farmer.golang.org/try?commit=6c788d1e
Gobot Gobot (Gerrit)
May 10, 2019, 11:04:34 AM5/10/19
to Russ Cox, goph...@pubsubhelper.golang.org, Hyang-Ah Hana Kim, golang-co...@googlegroups.com
TryBots are happy.
Patch set 1:TryBot-Result +1
Russ Cox (Gerrit)
May 10, 2019, 3:02:26 PM5/10/19
to Russ Cox, goph...@pubsubhelper.golang.org, Gobot Gobot, Hyang-Ah Hana Kim, golang-co...@googlegroups.com
Uploaded patch set 2.
Russ Cox (Gerrit)
May 10, 2019, 3:02:26 PM5/10/19
to Russ Cox, Gobot Gobot, Hyang-Ah Hana Kim, goph...@pubsubhelper.golang.org, golang-co...@googlegroups.com
Russ Cox uploaded patch set #2 to this change.
sumdb: client and server protocol for checksum database
Copied from golang.org/x/exp/sumdb/internal/sumweb
with types cleaned up:
Conn -> Client
Client -> ClientOps
Handler -> Server
Server -> ServerOps
For golang/go#31761.
Change-Id: If0e004e6c9cab69c82de428810d67aba074aa843
---
A sumdb/cache.go
A sumdb/client.go
A sumdb/client_test.go
A sumdb/server.go
A sumdb/test.go
5 files changed, 1,503 insertions(+), 0 deletions(-)
To view, visit change 176466. To unsubscribe, or for help writing mail filters, visit settings.
Gobot Gobot (Gerrit)
May 10, 2019, 3:02:36 PM5/10/19
to Russ Cox, goph...@pubsubhelper.golang.org, Hyang-Ah Hana Kim, golang-co...@googlegroups.com
Gobot Gobot (Gerrit)
May 10, 2019, 3:05:02 PM5/10/19
to Russ Cox, goph...@pubsubhelper.golang.org, Hyang-Ah Hana Kim, golang-co...@googlegroups.com
TryBots are happy.
Patch set 2:TryBot-Result +1
Russ Cox (Gerrit)
May 13, 2019, 2:39:16 PM5/13/19
to Russ Cox, goph...@pubsubhelper.golang.org, Gobot Gobot, Hyang-Ah Hana Kim, golang-co...@googlegroups.com
Uploaded patch set 3: Patch Set 2 was rebased.
Gobot Gobot (Gerrit)
May 13, 2019, 2:39:32 PM5/13/19
to Russ Cox, goph...@pubsubhelper.golang.org, Hyang-Ah Hana Kim, golang-co...@googlegroups.com
Gobot Gobot (Gerrit)
May 13, 2019, 2:40:42 PM5/13/19
to Russ Cox, goph...@pubsubhelper.golang.org, Hyang-Ah Hana Kim, golang-co...@googlegroups.com
TryBots are happy.
Patch set 3:TryBot-Result +1
Hyang-Ah Hana Kim (Gerrit)
May 16, 2019, 11:30:59 AM5/16/19
to Russ Cox, goph...@pubsubhelper.golang.org, Hyang-Ah Hana Kim, Gobot Gobot, golang-co...@googlegroups.com
8 comments:
Patch Set #3, Line 26:
"/lookup"can it assume the /lookup path already properly escaped?
Patch Set #3, Line 38:
It is read and written (using WriteConfig)Do you mean the value is written using WriteConfig? (I don't understand the 'read' part)
if c.tileHeight == 0 {
c.tileHeight = 8
}why not set this in NewClient and let SetTileHeight reject invalid values like height<=0?
and no more than once.
Patch Set #3, Line 264:
file := c.name + "/lookup/" + epath + "@" + evers
remotePath := "/lookup/" + epath + "@" + evers
file := c.name + remotePath
// NewContext returns the context to use for the request r.
NewContext(r *http.Request) (context.Context, error)
Why should this be in ServerOps?
is there any example implementation of ServerOps interface that does more than returning r.Context()?
Patch Set #3, Line 52:
ServerPathswhy should this be exported? Shouldn't the user use NewServer+ServeHTTP?
return r.Context(), nil
avoid nil context.
To view, visit change 176466. To unsubscribe, or for help writing mail filters, visit settings.
Russ Cox (Gerrit)
May 31, 2019, 12:12:36 PM5/31/19
to Russ Cox, Gobot Gobot, Hyang-Ah Hana Kim, goph...@pubsubhelper.golang.org, golang-co...@googlegroups.com
Russ Cox uploaded patch set #4 to this change.
sumdb: client and server protocol for checksum database
Copied from golang.org/x/exp/sumdb/internal/sumweb
with types cleaned up:
Conn -> Client
Client -> ClientOps
Handler -> Server
Server -> ServerOps
For golang/go#31761.
Change-Id: If0e004e6c9cab69c82de428810d67aba074aa843
---
A sumdb/cache.go
A sumdb/client.go
A sumdb/client_test.go
A sumdb/server.go
A sumdb/test.go
5 files changed, 1,499 insertions(+), 0 deletions(-)
To view, visit change 176466. To unsubscribe, or for help writing mail filters, visit settings.
Russ Cox (Gerrit)
May 31, 2019, 12:12:36 PM5/31/19
to Russ Cox, goph...@pubsubhelper.golang.org, Hyang-Ah Hana Kim, Gobot Gobot, golang-co...@googlegroups.com
Uploaded patch set 4.
(8 comments)
8 comments:
Patch Set #3, Line 26:
"/lookup"can it assume the /lookup path already properly escaped?
Yes. Updated.
Patch Set #3, Line 38:
It is read and written (using WriteConfig)Do you mean the value is written using WriteConfig? (I don't understand the 'read' part)
Removed this. The point was to draw a contrast with "key" which is not written, but I added that explicitly in the WriteConfig docs instead.
if c.tileHeight == 0 {
c.tileHeight = 8
}why not set this in NewClient and let SetTileHeight reject invalid values like height<=0?
Setting the default here avoids having a separate "didSetTileHeight bool". But I added the check for invalid height to SetTileHeight.
and no more than once.
Done
Patch Set #3, Line 264:
file := c.name + "/lookup/" + epath + "@" + eversremotePath := "/lookup/" + epath + "@" + evers […]
Done
// NewContext returns the context to use for the request r.
NewContext(r *http.Request) (context.Context, error)Why should this be in ServerOps? […]
I forgot r.Context existed. That should be fine. Removed.
Patch Set #3, Line 52:
ServerPathswhy should this be exported? Shouldn't the user use NewServer+ServeHTTP?
ServeHTTP is the name of the method you implement to be an http.Handler.
Users never call it directly.
The server needs to be registered at the expected paths
(maybe the overall web server does other stuff too).
These are the paths.The example below says
for _, path := range sumdb.ServerPaths {
http.Handle(path, srv)
}but a web server with multiple hosts that only wants to serve those on one host might do instead:
for _, path := range sumdb.ServerPaths {
http.Handle("sum.golang.org"+path, srv)
}And so on. Exposing the path list helps with that kind of thing.
return r.Context(), nil […]
Deleted.
To view, visit change 176466. To unsubscribe, or for help writing mail filters, visit settings.
Gobot Gobot (Gerrit)
May 31, 2019, 12:12:41 PM5/31/19
to Russ Cox, goph...@pubsubhelper.golang.org, Hyang-Ah Hana Kim, golang-co...@googlegroups.com
Gobot Gobot (Gerrit)
May 31, 2019, 12:15:55 PM5/31/19
to Russ Cox, goph...@pubsubhelper.golang.org, Hyang-Ah Hana Kim, golang-co...@googlegroups.com
TryBots are happy.
Patch set 4:TryBot-Result +1
Hyang-Ah Hana Kim (Gerrit)
May 31, 2019, 12:45:20 PM5/31/19
to Russ Cox, goph...@pubsubhelper.golang.org, Hyang-Ah Hana Kim, Gobot Gobot, golang-co...@googlegroups.com
Patch set 4:Code-Review +2
Russ Cox (Gerrit)
May 31, 2019, 2:24:36 PM5/31/19
to Russ Cox, goph...@pubsubhelper.golang.org, golang-...@googlegroups.com, Hyang-Ah Hana Kim, Gobot Gobot, golang-co...@googlegroups.com
Russ Cox merged this change.
Approvals:
Hyang-Ah Hana Kim: Looks good to me, approved
Russ Cox: Run TryBots
Gobot Gobot: TryBots succeeded
sumdb: client and server protocol for checksum database
Copied from golang.org/x/exp/sumdb/internal/sumweb
with types cleaned up:
Conn -> Client
Client -> ClientOps
Handler -> Server
Server -> ServerOps
For golang/go#31761.
Change-Id: If0e004e6c9cab69c82de428810d67aba074aa843
Reviewed-on: https://go-review.googlesource.com/c/mod/+/176466
Run-TryBot: Russ Cox <r...@golang.org>
TryBot-Result: Gobot Gobot <go...@golang.org>
Reviewed-by: Hyang-Ah Hana Kim <hya...@gmail.com>
---
A sumdb/cache.go
A sumdb/client.go
A sumdb/client_test.go
A sumdb/server.go
A sumdb/test.go
5 files changed, 1,499 insertions(+), 0 deletions(-)
index 0000000..70dd56f
--- /dev/null
+++ b/sumdb/client.go
@@ -0,0 +1,671 @@
+ // on the remote database server. The path begins with "/lookup" or "/tile/",
+ // and there is no need to parse the path in any way.
+ // It is the implementation's responsibility to turn that path into a full URL
+ // and make the HTTP request. ReadRemote should return an error for
+ // any non-200 HTTP response status.
+ ReadRemote(path string) ([]byte, error)
+
+ // ReadConfig reads and returns the content of the named configuration file.
+ // There are only a fixed set of configuration files.
+ //
+ // "key" returns a file containing the verifier key for the server.
+ //
+ // serverName + "/latest" returns a file containing the latest known
+ // signed tree from the server.
+ // To signal that the client wishes to start with an "empty" signed tree,
+ // ReadConfig can return a successful empty result (0 bytes of data).
+ ReadConfig(file string) ([]byte, error)
+
+ // WriteConfig updates the content of the named configuration file,
+ // changing it from the old []byte to the new []byte.
+ // If the old []byte does not match the stored configuration,
+ // WriteConfig must return ErrWriteConflict.
+ // Otherwise, WriteConfig should atomically replace old with new.
+ // The "key" configuration file is never written using WriteConfig.+// SetTileHeight can be called at most once,
+// and if so it must be called before the first call to Lookup.
+func (c *Client) SetTileHeight(height int) {
+ if atomic.LoadUint32(&c.didLookup) != 0 {
+ panic("SetTileHeight used after Lookup")
+ }
+ if height <= 0 {
+ panic("invalid call to SetTileHeight")
+ }
+ if c.tileHeight != 0 {
+ panic("multiple calls to SetTileHeight")
+ }
+ c.tileHeight = height
+}
+
+// SetGONOSUMDB sets the list of comma-separated GONOSUMDB patterns for the Client.
+// For any module path matching one of the patterns,
+// Lookup will return ErrGONOSUMDB.
+// SetGONOSUMDB can be called at most once,
+// and if so it must be called before the first call to Lookup.
+ epath, err := module.EscapePath(path)
+ if err != nil {
+ return nil, err
+ }
+ evers, err := module.EscapeVersion(strings.TrimSuffix(vers, "/go.mod"))
+ if err != nil {
+ return nil, err
+ }
+ remotePath := "/lookup/" + epath + "@" + evers
+ file := c.name + remotePathindex 0000000..acaf720
--- /dev/null
+++ b/sumdb/server.go
@@ -0,0 +1,181 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package sumdb implements the HTTP protocols for serving or accessing a module checksum database.
+package sumdb
+
+import (
+ "context"
+ "net/http"
+ "os"
+ "regexp"
+ "strings"
+
+ "golang.org/x/mod/module"
+ "golang.org/x/mod/sumdb/tlog"
+)
+
+// A ServerOps provides the external operations
+// (underlying database access and so on) needed by the Server.
+type ServerOps interface {
+ // Signed returns the signed hash of the latest tree.
+ Signed(ctx context.Context) ([]byte, error)
+
+ // ReadRecords returns the content for the n records id through id+n-1.
+ ReadRecords(ctx context.Context, id, n int64) ([][]byte, error)
+
+ // Lookup looks up a record for the given module,
+ // returning the record ID.
+ Lookup(ctx context.Context, m module.Version) (int64, error)+ ctx := r.Context()
+
+ switch {
+ default:
+ http.NotFound(w, r)
+
+ case strings.HasPrefix(r.URL.Path, "/lookup/"):
+ mod := strings.TrimPrefix(r.URL.Path, "/lookup/")
+ if !modVerRE.MatchString(mod) {
+ http.Error(w, "invalid module@version syntax", http.StatusBadRequest)
+ return
+ }
+ i := strings.Index(mod, "@")
+ escPath, escVers := mod[:i], mod[i+1:]
+ path, err := module.UnescapePath(escPath)
+ if err != nil {
+ reportError(w, r, err)
+ return
+ }
+ vers, err := module.UnescapeVersion(escVers)
+ if err != nil {
+ reportError(w, r, err)
+ return
+ }
+ id, err := s.ops.Lookup(ctx, module.Version{Path: path, Version: vers})
+ if err != nil {
index 0000000..534ca3e
--- /dev/null
+++ b/sumdb/test.go
@@ -0,0 +1,128 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package sumdb
+
+import (
+ "context"
+ "fmt"
Reply all
Reply to author
Forward
0 new messages