Progressbar writer

pmtiles/cluster.go

@@ -2,7 +2,6 @@ package pmtiles
 
 import (
 	"fmt"
-	"github.com/schollz/progressbar/v3"
 	"io"
 	"log"
 	"os"
@@ -41,7 +40,7 @@ func Cluster(logger *log.Logger, InputPMTiles string, deduplicate bool) error {
 		return err
 	}
 
-	bar := progressbar.Default(int64(header.TileEntriesCount))
+	bar := defaultProgressbar(logger, int64(header.TileEntriesCount))
 
 	err = IterateEntries(header,
 		func(offset uint64, length uint64) ([]byte, error) {

pmtiles/convert.go

@@ -16,7 +16,6 @@ import (
 	"time"
 
 	"github.com/RoaringBitmap/roaring/roaring64"
-	"github.com/schollz/progressbar/v3"
 	"zombiezen.com/go/sqlite"
 )
 
@@ -190,7 +189,7 @@ func convertMbtiles(logger *log.Logger, input string, output string, deduplicate
 	logger.Println("Pass 2: writing tiles")
 	resolve := newResolver(deduplicate, header.TileType == Mvt)
 	{
-		bar := progressbar.Default(int64(tileset.GetCardinality()))
+		bar := defaultProgressbar(logger, int64(tileset.GetCardinality()))
 		i := tileset.Iterator()
 		stmt := conn.Prep("SELECT tile_data FROM tiles WHERE zoom_level = ? AND tile_column = ? AND tile_row = ?")
 

pmtiles/edit.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"github.com/schollz/progressbar/v3"
 	"io"
 	"io/ioutil"
 	"log"
@@ -13,7 +12,7 @@ import (
 
 // Edit parts of the header or metadata.
 // works in-place if only the header is modified.
-func Edit(_ *log.Logger, inputArchive string, newHeaderJSONFile string, newMetadataFile string) error {
+func Edit(logger *log.Logger, inputArchive string, newHeaderJSONFile string, newMetadataFile string) error {
 	if newHeaderJSONFile == "" && newMetadataFile == "" {
 		return fmt.Errorf("must supply --header-json and/or --metadata to edit")
 	}
@@ -110,7 +109,8 @@ func Edit(_ *log.Logger, inputArchive string, newHeaderJSONFile string, newMetad
 	newHeader.LeafDirectoryOffset = newHeader.MetadataOffset + newHeader.MetadataLength
 	newHeader.TileDataOffset = newHeader.LeafDirectoryOffset + newHeader.LeafDirectoryLength
 
-	bar := progressbar.DefaultBytes(
+	bar := defaultBytesProgressbar(
+		logger,
 		int64(HeaderV3LenBytes+newHeader.RootLength+uint64(len(metadataBytes))+newHeader.LeafDirectoryLength+newHeader.TileDataLength),
 		"writing file",
 	)

pmtiles/extract.go

@@ -8,7 +8,6 @@ import (
 	"github.com/RoaringBitmap/roaring/roaring64"
 	"github.com/dustin/go-humanize"
 	"github.com/paulmach/orb"
-	"github.com/schollz/progressbar/v3"
 	"golang.org/x/sync/errgroup"
 	"io"
 	"io/ioutil"
@@ -249,7 +248,7 @@ func MergeRanges(ranges []srcDstRange, overfetch float32) (*list.List, uint64) {
 // 9. get and write the metadata.
 // 10. write the leaf directories (if any)
 // 11. Get all tiles, and write directly to the output.
-func Extract(_ *log.Logger, bucketURL string, key string, minzoom int8, maxzoom int8, regionFile string, bbox string, output string, downloadThreads int, overfetch float32, dryRun bool) error {
+func Extract(logger *log.Logger, bucketURL string, key string, minzoom int8, maxzoom int8, regionFile string, bbox string, output string, downloadThreads int, overfetch float32, dryRun bool) error {
 	// 1. fetch the header
 	start := time.Now()
 	ctx := context.Background()
@@ -491,7 +490,8 @@ func Extract(_ *log.Logger, bucketURL string, key string, minzoom int8, maxzoom
 			return err
 		}
 
-		bar := progressbar.DefaultBytes(
+		bar := defaultBytesProgressbar(
+			logger,
 			int64(totalBytes),
 			"fetching chunks",
 		)

pmtiles/makesync.go

@@ -6,7 +6,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/cespare/xxhash/v2"
-	"github.com/schollz/progressbar/v3"
 	"golang.org/x/sync/errgroup"
 	"io"
 	"log"
@@ -106,7 +105,8 @@ func Makesync(logger *log.Logger, cliVersion string, fileName string, blockSizeK
 
 	defer output.Close()
 
-	bar := progressbar.Default(
+	bar := defaultProgressbar(
+		logger,
 		int64(header.TileEntriesCount),
 		"writing syncfile",
 	)

pmtiles/progressbar.go

@@ -0,0 +1,52 @@
+package pmtiles
+
+import (
+	"fmt"
+	"github.com/schollz/progressbar/v3"
+	"log"
+	"time"
+)
+
+func defaultProgressbar(log *log.Logger, max int64, description ...string) *progressbar.ProgressBar {
+	desc := ""
+	if len(description) > 0 {
+		desc = description[0]
+	}
+	return progressbar.NewOptions64(
+		max,
+		progressbar.OptionSetDescription(desc),
+		progressbar.OptionSetWriter(log.Writer()),
+		progressbar.OptionSetWidth(10),
+		progressbar.OptionThrottle(65*time.Millisecond),
+		progressbar.OptionShowCount(),
+		progressbar.OptionShowIts(),
+		progressbar.OptionOnCompletion(func() {
+			fmt.Fprint(log.Writer(), "\n")
+		}),
+		progressbar.OptionSpinnerType(14),
+		progressbar.OptionFullWidth(),
+		progressbar.OptionSetRenderBlankState(true),
+	)
+}
+
+func defaultBytesProgressbar(log *log.Logger, maxBytes int64, description ...string) *progressbar.ProgressBar {
+	desc := ""
+	if len(description) > 0 {
+		desc = description[0]
+	}
+	return progressbar.NewOptions64(
+		maxBytes,
+		progressbar.OptionSetDescription(desc),
+		progressbar.OptionSetWriter(log.Writer()),
+		progressbar.OptionShowBytes(true),
+		progressbar.OptionSetWidth(10),
+		progressbar.OptionThrottle(65*time.Millisecond),
+		progressbar.OptionShowCount(),
+		progressbar.OptionOnCompletion(func() {
+			fmt.Fprint(log.Writer(), "\n")
+		}),
+		progressbar.OptionSpinnerType(14),
+		progressbar.OptionFullWidth(),
+		progressbar.OptionSetRenderBlankState(true),
+	)
+}

pmtiles/sync.go

@@ -7,7 +7,6 @@ import (
 	"fmt"
 	"github.com/cespare/xxhash/v2"
 	"github.com/dustin/go-humanize"
-	"github.com/schollz/progressbar/v3"
 	"golang.org/x/sync/errgroup"
 	"io"
 	"log"
@@ -85,7 +84,8 @@ func Sync(logger *log.Logger, oldVersion string, newVersion string, dryRun bool)
 	if err != nil {
 		return err
 	}
-	bar := progressbar.DefaultBytes(
+	bar := defaultBytesProgressbar(
+		logger,
 		resp.ContentLength,
 		"downloading syncfile",
 	)
@@ -122,7 +122,8 @@ func Sync(logger *log.Logger, oldVersion string, newVersion string, dryRun bool)
 		return fmt.Errorf("archive must be clustered for sync")
 	}
 
-	bar = progressbar.Default(
+	bar = defaultProgressbar(
+		logger,
 		int64(len(blocks)),
 		"calculating diff",
 	)
@@ -282,15 +283,17 @@ func Sync(logger *log.Logger, oldVersion string, newVersion string, dryRun bool)
 		req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", newHeader.LeafDirectoryOffset, newHeader.LeafDirectoryOffset+newHeader.LeafDirectoryLength-1))
 		resp, err = client.Do(req)
 
-		leafBar := progressbar.DefaultBytes(
+		leafBar := defaultBytesProgressbar(
+			logger,
 			int64(newHeader.LeafDirectoryLength),
 			"downloading leaf directories",
 		)
 		io.Copy(leafWriter, io.TeeReader(resp.Body, leafBar))
 		leafBar.Close()
 
 		fmt.Println(len(have), "local chunks")
-		bar := progressbar.DefaultBytes(
+		bar := defaultBytesProgressbar(
+			logger,
 			int64(totalRemoteBytes-toTransfer),
 			"copying local chunks",
 		)
@@ -308,7 +311,8 @@ func Sync(logger *log.Logger, oldVersion string, newVersion string, dryRun bool)
 		multiRanges := makeMultiRanges(ranges, int64(newHeader.TileDataOffset), 1048576-200)
 		fmt.Println("Batched into http requests", len(multiRanges))
 
-		bar = progressbar.DefaultBytes(
+		bar = defaultBytesProgressbar(
+			logger,
 			int64(toTransfer),
 			"fetching remote chunks",
 		)

pmtiles/upload.go

@@ -3,7 +3,6 @@ package pmtiles
 import (
 	"context"
 	"fmt"
-	"github.com/schollz/progressbar/v3"
 	"gocloud.dev/blob"
 	"io"
 	"log"
@@ -19,7 +18,7 @@ func partSizeBytes(totalSize int64) int {
 }
 
 // Upload a pmtiles archive to a bucket.
-func Upload(_ *log.Logger, InputPMTiles string, bucket string, RemotePMTiles string, maxConcurrency int) error {
+func Upload(logger *log.Logger, InputPMTiles string, bucket string, RemotePMTiles string, maxConcurrency int) error {
 	ctx := context.Background()
 
 	b, err := blob.OpenBucket(ctx, bucket)
@@ -49,7 +48,7 @@ func Upload(_ *log.Logger, InputPMTiles string, bucket string, RemotePMTiles str
 		return fmt.Errorf("Failed to obtain writer: %w", err)
 	}
 
-	bar := progressbar.Default(filestat.Size())
+	bar := defaultProgressbar(logger, filestat.Size())
 	io.Copy(io.MultiWriter(w, bar), f)
 
 	if err := w.Close(); err != nil {