more wip

main.go

@@ -55,7 +55,7 @@ var cli struct {
 
 	Makesync struct {
 		Input        string `arg:"" type:"existingfile"`
-		BlockSize    int    `default:100 help:"The block size, in # of tiles."`
+		BlockSize    int    `default:1000 help:"The block size, in # of tiles."`
 		HashFunction string `default:fnv1a help:"The hash function."`
 	} `cmd:"" help:"Generates an **experimental** sync control file (.pmtiles.sync) for a local archive."`
 
@@ -77,6 +77,14 @@ var cli struct {
 		PublicHostname string `help:"Public hostname of tile endpoint e.g. https://example.com"`
 	} `cmd:"" help:"Run an HTTP proxy server for Z/X/Y tiles."`
 
+	Download struct {
+		Output string `arg:"" type:"path"`
+		Key            string `arg:""`
+		MaxConcurrency int    `default:2 help:"# of download threads"`
+		Bucket         string `required:"" help:"Bucket to download from."`
+		Input          string `arg:"" type:"existingfile"`
+	} `cmd:"" help:"Download a archive to a local file. Equivalent to extract with no arguments."`
+
 	Upload struct {
 		Input          string `arg:"" type:"existingfile"`
 		Key            string `arg:""`

pmtiles/makesync.go

@@ -10,6 +10,14 @@ import (
 	"log"
 )
 
+// download algorithm
+// 1. read NEW header
+// download all NEW metadata, root, and leaf directories.
+// 1. read NEW syncfile.
+// open OLD directory, calculate all block hashes to find matches (length and hash equals)
+// start writing new file
+// the result should be byte-for-byte identical
+
 // todo: the hash of the entire file ?
 // is this robust against movements in tile coordinates? (i think so)
 func Makesync(logger *log.Logger, bucketURL string, file string, blocksize int) error {
@@ -88,6 +96,7 @@ func Makesync(logger *log.Logger, bucketURL string, file string, blocksize int)
 		fmt.Printf("%d,%d,%d,%x\n", current_block, block_start, block_length, h.Sum64())
 	}
 
+	// TODO: does not handle runlength deduplications
 	for _, entry := range total_entries {
 		if entry.TileId/uint64(blocksize) != current_block {
 			emit(current_block, block_start, block_length)