experimental makesync command

main.go

@@ -135,6 +135,11 @@ func main() {
 		if err != nil {
 			logger.Fatalf("Failed to stats archive, %v", err)
 		}
+	case "makesync <input>":
+		err := pmtiles.Makesync(logger, "file:///", cli.Makesync.Input, cli.Makesync.BlockSize)
+		if err != nil {
+			logger.Fatalf("Failed to makesync archive, %v", err)
+		}
 	case "convert <input> <output>":
 		path := cli.Convert.Input
 		output := cli.Convert.Output

pmtiles/makesync.go

@@ -0,0 +1,108 @@
+package pmtiles
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"gocloud.dev/blob"
+	"hash/fnv"
+	"io"
+	"log"
+)
+
+// todo: the hash of the entire file ?
+// is this robust against movements in tile coordinates? (i think so)
+func Makesync(logger *log.Logger, bucketURL string, file string, blocksize int) error {
+	// determine the byte range in the file for entries 0...10, ignoring any backreferences.
+
+	ctx := context.Background()
+	bucket, err := blob.OpenBucket(ctx, bucketURL)
+	if err != nil {
+		return fmt.Errorf("Failed to open bucket for %s, %w", bucketURL, err)
+	}
+	defer bucket.Close()
+
+	r, err := bucket.NewRangeReader(ctx, file, 0, HEADERV3_LEN_BYTES, nil)
+
+	if err != nil {
+		return fmt.Errorf("Failed to create range reader for %s, %w", file, err)
+	}
+	b, err := io.ReadAll(r)
+	if err != nil {
+		return fmt.Errorf("Failed to read %s, %w", file, err)
+	}
+	r.Close()
+
+	header, err := deserialize_header(b[0:HEADERV3_LEN_BYTES])
+	if err != nil {
+		// check to see if it's a V2 file
+		if string(b[0:2]) == "PM" {
+			spec_version := b[2]
+			return fmt.Errorf("PMTiles version %d detected; please use 'pmtiles convert' to upgrade to version 3.", spec_version)
+		}
+
+		return fmt.Errorf("Failed to read %s, %w", file, err)
+	}
+
+	total_entries := make([]EntryV3, 0)
+
+	var CollectEntries func(uint64, uint64)
+
+	// depth-first traversal of directories
+	CollectEntries = func(dir_offset uint64, dir_length uint64) {
+		dirbytes, err := bucket.NewRangeReader(ctx, file, int64(dir_offset), int64(dir_length), nil)
+		defer dirbytes.Close()
+		b, err = io.ReadAll(dirbytes)
+		if err != nil {
+			panic(fmt.Errorf("I/O Error"))
+		}
+
+		directory := deserialize_entries(bytes.NewBuffer(b))
+		for _, entry := range directory {
+			if entry.RunLength > 0 {
+				total_entries = append(total_entries, entry)
+			} else {
+				CollectEntries(header.LeafDirectoryOffset+entry.Offset, uint64(entry.Length))
+			}
+		}
+	}
+
+	CollectEntries(header.RootOffset, header.RootLength)
+
+	current_block := uint64(0)
+	block_start := uint64(0)
+	block_length := uint64(0)
+
+	fmt.Printf("blocksize=%d\n", blocksize)
+	fmt.Println("hash=fnv1a")
+
+	emit := func(blocknum uint64, offset uint64, length uint64) {
+		blockdata, err := bucket.NewRangeReader(ctx, file, int64(block_start), int64(block_length), nil)
+		defer blockdata.Close()
+		b, err = io.ReadAll(blockdata)
+		if err != nil {
+			panic(fmt.Errorf("I/O Error"))
+		}
+		h := fnv.New64a()
+		h.Write(b)
+		fmt.Printf("%d,%d,%d,%x\n", current_block, block_start, block_length, h.Sum64())
+	}
+
+	for _, entry := range total_entries {
+		if entry.TileId/uint64(blocksize) != current_block {
+			emit(current_block, block_start, block_length)
+			current_block = entry.TileId / uint64(blocksize)
+			block_start = entry.Offset
+			block_length = 0
+		}
+		if entry.Offset < block_start {
+			// ignore this deduplicated tile
+			continue
+		}
+		block_length += uint64(entry.Length)
+	}
+
+	emit(current_block, block_start, block_length)
+
+	return nil
+}