feat: upload list (#9)

resolves #5

Author: Alan Shaw

client.go

@@ -15,12 +15,15 @@ import (
 	w3http "github.com/web3-storage/go-w3s-client/http"
 )
 
+const clientName = "web3.storage/go"
+
 // Client is a HTTP API client to the web3.storage service.
 type Client interface {
 	Get(context.Context, cid.Cid) (*w3http.Web3Response, error)
 	Put(context.Context, fs.File, ...PutOption) (cid.Cid, error)
 	PutCar(context.Context, io.Reader) (cid.Cid, error)
 	Status(context.Context, cid.Cid) (*Status, error)
+	List(context.Context, ...ListOption) (*UploadIterator, error)
 }
 
 type clientConfig struct {
@@ -57,3 +60,5 @@ func NewClient(options ...Option) (Client, error) {
 	}
 	return &c, nil
 }
+
+var _ Client = (*client)(nil)

example/go.sum

@@ -1224,6 +1224,8 @@ github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpP
 github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
 github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e/go.mod h1:XDKHRm5ThF8YJjx001LtgelzsoaEcvnA7lVWz9EeX3g=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y=
+github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=

example/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"io"
 	"io/fs"
 	"os"
 
@@ -24,8 +25,9 @@ func main() {
 
 	// cid := putSingleFile(c)
 	// getStatusForCid(c, cid)
-
+	// getStatusForKnownCid(c)
 	getFiles(c)
+	// listUploads(c)
 }
 
 func putSingleFile(c w3s.Client) cid.Cid {
@@ -90,7 +92,7 @@ func getStatusForCid(c w3s.Client, cid cid.Cid) {
 }
 
 func getStatusForKnownCid(c w3s.Client) {
-	cid, _ := cid.Parse("bafybeig7qnlzyregxe2m63b4kkpx3ujqm5bwmn5wtvtftp7j27tmdtznji")
+	cid, _ := cid.Parse("bafybeiauyddeo2axgargy56kwxirquxaxso3nobtjtjvoqu552oqciudrm")
 	getStatusForCid(c, cid)
 }
 
@@ -125,3 +127,23 @@ func getFiles(c w3s.Client) {
 		fmt.Printf("%s (%d bytes)\n", cid.String(), info.Size())
 	}
 }
+
+func listUploads(c w3s.Client) {
+	uploads, err := c.List(context.Background())
+	if err != nil {
+		panic(err)
+	}
+
+	for {
+		u, err := uploads.Next()
+		if err != nil {
+			// finished successfully
+			if err == io.EOF {
+				break
+			}
+			panic(err)
+		}
+
+		fmt.Printf("%s	%s	Size: %d	Deals: %d	Pins: %d\n", u.Created.Format("2006-01-02 15:04:05"), u.Cid, u.DagSize, len(u.Deals), len(u.Pins))
+	}
+}

get.go

@@ -10,12 +10,12 @@ import (
 )
 
 func (c *client) Get(ctx context.Context, cid cid.Cid) (*w3http.Web3Response, error) {
-	req, err := http.NewRequest("GET", fmt.Sprintf("%s/car/%s", c.cfg.endpoint, cid), nil)
+	req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s/car/%s", c.cfg.endpoint, cid), nil)
 	if err != nil {
 		return nil, err
 	}
 	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", c.cfg.token))
-	req.Header.Add("X-Client", "web3.storage/go")
+	req.Header.Add("X-Client", clientName)
 	res, err := c.hc.Do(req)
 	return w3http.NewWeb3Response(res, c.bsvc), err
 }

go.mod

@@ -24,4 +24,5 @@ require (
 	github.com/ipld/go-codec-dagpb v1.3.0
 	github.com/ipld/go-ipld-prime v0.12.3
 	github.com/libp2p/go-libp2p-core v0.9.0
+	github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80
 )

go.sum

@@ -0,0 +1,140 @@
+package w3s
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/tomnomnom/linkheader"
+)
+
+const maxPageSize = 100
+
+type UploadIterator struct {
+	paginator *pageIterator
+	max       int
+	count     int
+	page      []*Status
+}
+
+// Next retrieves status information for the next upload in the list.
+func (li *UploadIterator) Next() (*Status, error) {
+	li.count++
+	if li.max > 0 && li.count > li.max {
+		return nil, io.EOF
+	}
+	if len(li.page) > 0 {
+		item := li.page[0]
+		li.page = li.page[1:]
+		return item, nil
+	}
+	res, err := li.paginator.Next()
+	if err != nil {
+		return nil, err
+	}
+	var page []*Status
+	d := json.NewDecoder(res.Body)
+	err = d.Decode(&page)
+	if err != nil {
+		return nil, err
+	}
+	li.page = page
+	if len(li.page) > 0 {
+		item := li.page[0]
+		li.page = li.page[1:]
+		return item, nil
+	}
+	return nil, io.EOF
+}
+
+type listConfig struct {
+	before     time.Time
+	maxResults int
+}
+
+// List retrieves the list of uploads to Web3.Storage.
+func (c *client) List(ctx context.Context, options ...ListOption) (*UploadIterator, error) {
+	var cfg listConfig
+	for _, opt := range options {
+		if err := opt(&cfg); err != nil {
+			return nil, err
+		}
+	}
+
+	fetchNextPage := func(url string) (*http.Response, error) {
+		req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s%s", c.cfg.endpoint, url), nil)
+		if err != nil {
+			return nil, err
+		}
+		req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", c.cfg.token))
+		req.Header.Add("Access-Control-Request-Headers", "Link")
+		req.Header.Add("X-Client", clientName)
+		res, err := c.hc.Do(req)
+		if err != nil {
+			return nil, err
+		}
+		if res.StatusCode != 200 {
+			return nil, fmt.Errorf("unexpected response status: %d", res.StatusCode)
+		}
+		return res, nil
+	}
+
+	var before string
+	if cfg.before.IsZero() {
+		before = time.Now().Format(iso8601)
+	} else {
+		before = cfg.before.Format(iso8601)
+	}
+
+	size := cfg.maxResults
+	if size > maxPageSize {
+		size = maxPageSize
+	}
+
+	var urlPath string
+	if size <= 0 {
+		urlPath = fmt.Sprintf("/user/uploads?before=%s", url.QueryEscape(before))
+	} else {
+		urlPath = fmt.Sprintf("/user/uploads?before=%s&size=%d", url.QueryEscape(before), size)
+	}
+
+	return &UploadIterator{
+		paginator: newPageIterator(urlPath, fetchNextPage),
+		max:       cfg.maxResults,
+	}, nil
+}
+
+type pageIterator struct {
+	nextURL       string
+	fetchNextPage func(string) (*http.Response, error)
+}
+
+func newPageIterator(url string, fetchNextPage func(string) (*http.Response, error)) *pageIterator {
+	return &pageIterator{
+		nextURL:       url,
+		fetchNextPage: fetchNextPage,
+	}
+}
+
+func (pi *pageIterator) Next() (*http.Response, error) {
+	res, err := pi.fetchNextPage(pi.nextURL)
+	if err != nil {
+		return nil, err
+	}
+	linkHdrs := res.Header["Link"]
+	if len(linkHdrs) == 0 {
+		return nil, io.EOF
+	}
+	links := linkheader.Parse(linkHdrs[0])
+	for _, l := range links {
+		if l.Rel == "next" {
+			pi.nextURL = links[0].URL
+			return res, nil
+		}
+	}
+	return nil, io.EOF
+}

list.go

@@ -2,6 +2,7 @@ package w3s
 
 import (
 	"io/fs"
+	"time"
 
 	ds "github.com/ipfs/go-datastore"
 )
@@ -64,3 +65,23 @@ func WithDirname(dirname string) PutOption {
 		return nil
 	}
 }
+
+// ListOption is an option configuring a call to List.
+type ListOption func(cfg *listConfig) error
+
+// WithBefore sets the time that items in the list were uploaded before.
+func WithBefore(before time.Time) ListOption {
+	return func(cfg *listConfig) error {
+		cfg.before = before
+		return nil
+	}
+}
+
+// WithMaxResults sets the maximum number of results that will be available from
+// the iterator.
+func WithMaxResults(maxResults int) ListOption {
+	return func(cfg *listConfig) error {
+		cfg.maxResults = maxResults
+		return nil
+	}
+}

opts.go

@@ -120,7 +120,7 @@ func (c *client) sendCar(ctx context.Context, r io.Reader) (cid.Cid, error) {
 	}
 	req.Header.Add("Content-Type", "application/car")
 	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", c.cfg.token))
-	req.Header.Add("X-Client", "web3.storage/go")
+	req.Header.Add("X-Client", clientName)
 	res, err := c.hc.Do(req)
 	if err != nil {
 		return cid.Undef, err

put.go

@@ -13,7 +13,7 @@ import (
 	peer "github.com/libp2p/go-libp2p-core/peer"
 )
 
-const iso8601 = "2006-01-02T15:04:05Z0700"
+const iso8601 = "2006-01-02T15:04:05.999Z07:00"
 
 type PinStatus int
 
@@ -204,12 +204,12 @@ func (s *Status) UnmarshalJSON(b []byte) error {
 }
 
 func (c *client) Status(ctx context.Context, cid cid.Cid) (*Status, error) {
-	req, err := http.NewRequest("GET", fmt.Sprintf("%s/status/%s", c.cfg.endpoint, cid), nil)
+	req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s/status/%s", c.cfg.endpoint, cid), nil)
 	if err != nil {
 		return nil, err
 	}
 	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", c.cfg.token))
-	req.Header.Add("X-Client", "web3.storage/go")
+	req.Header.Add("X-Client", clientName)
 	res, err := c.hc.Do(req)
 	if err != nil {
 		return nil, err