From 1a2a1a73f16a56ce7fd10086f16603755c81e3b8 Mon Sep 17 00:00:00 2001 From: Brandon Liu Date: Sun, 28 Jan 2024 11:25:05 +0800 Subject: [PATCH] =?UTF-8?q?Make=20all=20variable=20and=20field=20names=20c?= =?UTF-8?q?onsistent=20with=20style=20guide.=20[#20,=20=E2=80=A6=20(#123)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Make all variable and field names consistent with style guide. [#20, #46] * Use revive with the default configuration. * also do error string and code style cleanup (revive defaults) --- main.go | 16 +- pmtiles/bitmap.go | 72 ++++----- pmtiles/bitmap_test.go | 26 ++-- pmtiles/bucket.go | 12 +- pmtiles/convert.go | 302 +++++++++++++++++++------------------- pmtiles/convert_test.go | 60 ++++---- pmtiles/directory.go | 118 +++++++-------- pmtiles/directory_test.go | 64 ++++---- pmtiles/extract.go | 270 +++++++++++++++++----------------- pmtiles/extract_test.go | 8 +- pmtiles/makesync.go | 74 +++++----- pmtiles/readerv2.go | 80 +++++----- pmtiles/region.go | 10 +- pmtiles/server.go | 174 +++++++++++----------- pmtiles/server_test.go | 12 +- pmtiles/show.go | 86 +++++------ pmtiles/tile_id.go | 32 ++-- pmtiles/tile_id_test.go | 64 ++++---- pmtiles/tilejson.go | 22 +-- pmtiles/upload.go | 4 +- pmtiles/verify.go | 46 +++--- 21 files changed, 776 insertions(+), 776 deletions(-) diff --git a/main.go b/main.go index e315d92..4ae8081 100644 --- a/main.go +++ b/main.go @@ -36,7 +36,7 @@ var cli struct { Bucket string `help:"Remote bucket"` Metadata bool `help:"Print only the JSON metadata."` Tilejson bool `help:"Print the TileJSON."` - PublicUrl string `help:"Public base URL of tile endpoint for TileJSON e.g. https://example.com/tiles"` + PublicURL string `help:"Public base URL of tile endpoint for TileJSON e.g. https://example.com/tiles"` } `cmd:"" help:"Inspect a local or remote archive."` Tile struct { @@ -83,7 +83,7 @@ var cli struct { Cors string `help:"Value of HTTP CORS header."` CacheSize int `default:64 help:"Size of cache in Megabytes."` Bucket string `help:"Remote bucket"` - PublicUrl string `help:"Public base URL of tile endpoint for TileJSON e.g. https://example.com/tiles/"` + PublicURL string `help:"Public base URL of tile endpoint for TileJSON e.g. https://example.com/tiles/"` } `cmd:"" help:"Run an HTTP proxy server for Z/X/Y tiles."` Download struct { @@ -116,7 +116,7 @@ func main() { switch ctx.Command() { case "show ": - err := pmtiles.Show(logger, cli.Show.Bucket, cli.Show.Path, cli.Show.Metadata, cli.Show.Tilejson, cli.Show.PublicUrl, false, 0, 0, 0) + err := pmtiles.Show(logger, cli.Show.Bucket, cli.Show.Path, cli.Show.Metadata, cli.Show.Tilejson, cli.Show.PublicURL, false, 0, 0, 0) if err != nil { logger.Fatalf("Failed to show archive, %v", err) } @@ -126,7 +126,7 @@ func main() { logger.Fatalf("Failed to show tile, %v", err) } case "serve ": - server, err := pmtiles.NewServer(cli.Serve.Bucket, cli.Serve.Path, logger, cli.Serve.CacheSize, cli.Serve.Cors, cli.Serve.PublicUrl) + server, err := pmtiles.NewServer(cli.Serve.Bucket, cli.Serve.Path, logger, cli.Serve.CacheSize, cli.Serve.Cors, cli.Serve.PublicURL) if err != nil { logger.Fatalf("Failed to create new server, %v", err) @@ -136,11 +136,11 @@ func main() { http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { start := time.Now() - status_code, headers, body := server.Get(r.Context(), r.URL.Path) + statusCode, headers, body := server.Get(r.Context(), r.URL.Path) for k, v := range headers { w.Header().Set(k, v) } - w.WriteHeader(status_code) + w.WriteHeader(statusCode) w.Write(body) logger.Printf("served %s in %s", r.URL.Path, time.Since(start)) }) @@ -166,13 +166,13 @@ func main() { logger.Fatalf("Failed to create temp file, %v", err) } } else { - abs_tmproot, err := filepath.Abs(cli.Convert.Tmpdir) + absTemproot, err := filepath.Abs(cli.Convert.Tmpdir) if err != nil { logger.Fatalf("Failed to derive absolute path for %s, %v", cli.Convert.Tmpdir, err) } - tmpfile, err = os.CreateTemp(abs_tmproot, "pmtiles") + tmpfile, err = os.CreateTemp(absTemproot, "pmtiles") if err != nil { logger.Fatalf("Failed to create temp file, %v", err) diff --git a/pmtiles/bitmap.go b/pmtiles/bitmap.go index 05ba6c5..2fcc7fd 100644 --- a/pmtiles/bitmap.go +++ b/pmtiles/bitmap.go @@ -15,54 +15,54 @@ import ( ) func bitmapMultiPolygon(zoom uint8, multipolygon orb.MultiPolygon) (*roaring64.Bitmap, *roaring64.Bitmap) { - boundary_set := roaring64.New() + boundarySet := roaring64.New() for _, polygon := range multipolygon { for _, ring := range polygon { - boundary_tiles, _ := tilecover.Geometry(orb.LineString(ring), maptile.Zoom(zoom)) // TODO is this buffer-aware? - for tile := range boundary_tiles { - boundary_set.Add(ZxyToId(uint8(tile.Z), tile.X, tile.Y)) + boundaryTiles, _ := tilecover.Geometry(orb.LineString(ring), maptile.Zoom(zoom)) // TODO is this buffer-aware? + for tile := range boundaryTiles { + boundarySet.Add(ZxyToID(uint8(tile.Z), tile.X, tile.Y)) } } } - multipolygon_projected := project.MultiPolygon(multipolygon.Clone(), project.WGS84.ToMercator) + multipolygonProjected := project.MultiPolygon(multipolygon.Clone(), project.WGS84.ToMercator) - interior_set := roaring64.New() - i := boundary_set.Iterator() + interiorSet := roaring64.New() + i := boundarySet.Iterator() for i.HasNext() { id := i.Next() - if !boundary_set.Contains(id+1) && i.HasNext() { - z, x, y := IdToZxy(id + 1) + if !boundarySet.Contains(id+1) && i.HasNext() { + z, x, y := IDToZxy(id + 1) tile := maptile.New(x, y, maptile.Zoom(z)) - if planar.MultiPolygonContains(multipolygon_projected, project.Point(tile.Center(), project.WGS84.ToMercator)) { - interior_set.AddRange(id+1, i.PeekNext()) + if planar.MultiPolygonContains(multipolygonProjected, project.Point(tile.Center(), project.WGS84.ToMercator)) { + interiorSet.AddRange(id+1, i.PeekNext()) } } } - return boundary_set, interior_set + return boundarySet, interiorSet } func generalizeOr(r *roaring64.Bitmap, minzoom uint8) { if r.GetCardinality() == 0 { return } - max_z, _, _ := IdToZxy(r.ReverseIterator().Next()) + maxZ, _, _ := IDToZxy(r.ReverseIterator().Next()) var temp *roaring64.Bitmap - var to_iterate *roaring64.Bitmap + var toIterate *roaring64.Bitmap temp = roaring64.New() - to_iterate = r + toIterate = r - for current_z := int(max_z); current_z > int(minzoom); current_z-- { - iter := to_iterate.Iterator() + for currentZ := int(maxZ); currentZ > int(minzoom); currentZ-- { + iter := toIterate.Iterator() for iter.HasNext() { - parent_id := ParentId(iter.Next()) - temp.Add(parent_id) + parentID := ParentID(iter.Next()) + temp.Add(parentID) } - to_iterate = temp + toIterate = temp r.Or(temp) temp = roaring64.New() } @@ -72,32 +72,32 @@ func generalizeAnd(r *roaring64.Bitmap) { if r.GetCardinality() == 0 { return } - max_z, _, _ := IdToZxy(r.ReverseIterator().Next()) + maxZ, _, _ := IDToZxy(r.ReverseIterator().Next()) var temp *roaring64.Bitmap - var to_iterate *roaring64.Bitmap + var toIterate *roaring64.Bitmap temp = roaring64.New() - to_iterate = r + toIterate = r - for current_z := int(max_z); current_z > 0; current_z-- { - iter := to_iterate.Iterator() + for currentZ := int(maxZ); currentZ > 0; currentZ-- { + iter := toIterate.Iterator() filled := 0 current := uint64(0) // check me... for iter.HasNext() { id := iter.Next() - parent_id := ParentId(id) - if parent_id == current { - filled += 1 + parentID := ParentID(id) + if parentID == current { + filled++ if filled == 4 { - temp.Add(parent_id) + temp.Add(parentID) } } else { - current = parent_id + current = parentID filled = 1 } } - to_iterate = temp + toIterate = temp r.Or(temp) temp = roaring64.New() } @@ -107,8 +107,8 @@ func WriteImage(interior *roaring64.Bitmap, boundary *roaring64.Bitmap, exterior dim := 1 << zoom img := image.NewNRGBA(image.Rect(0, 0, dim, dim)) - min := ZxyToId(zoom, 0, 0) - max := ZxyToId(zoom+1, 0, 0) + min := ZxyToID(zoom, 0, 0) + max := ZxyToID(zoom+1, 0, 0) { iter := interior.Iterator() @@ -116,7 +116,7 @@ func WriteImage(interior *roaring64.Bitmap, boundary *roaring64.Bitmap, exterior for iter.HasNext() { id := iter.Next() if id >= min && id < max { - _, x, y := IdToZxy(id) + _, x, y := IDToZxy(id) img.Set(int(x), int(y), fill) } } @@ -127,7 +127,7 @@ func WriteImage(interior *roaring64.Bitmap, boundary *roaring64.Bitmap, exterior for iter.HasNext() { id := iter.Next() if id >= min && id < max { - _, x, y := IdToZxy(id) + _, x, y := IDToZxy(id) img.Set(int(x), int(y), fill) } } @@ -138,7 +138,7 @@ func WriteImage(interior *roaring64.Bitmap, boundary *roaring64.Bitmap, exterior for iter.HasNext() { id := iter.Next() if id >= min && id < max { - _, x, y := IdToZxy(id) + _, x, y := IDToZxy(id) img.Set(int(x), int(y), fill) } } diff --git a/pmtiles/bitmap_test.go b/pmtiles/bitmap_test.go index 1f412c8..79adf1d 100644 --- a/pmtiles/bitmap_test.go +++ b/pmtiles/bitmap_test.go @@ -11,17 +11,17 @@ func TestGeneralizeAnd(t *testing.T) { generalizeAnd(b) assert.Equal(t, uint64(0), b.GetCardinality()) b = roaring64.New() - b.Add(ZxyToId(3, 0, 0)) + b.Add(ZxyToID(3, 0, 0)) generalizeAnd(b) assert.Equal(t, uint64(1), b.GetCardinality()) b = roaring64.New() - b.Add(ZxyToId(3, 0, 0)) - b.Add(ZxyToId(3, 0, 1)) - b.Add(ZxyToId(3, 1, 0)) - b.Add(ZxyToId(3, 1, 1)) + b.Add(ZxyToID(3, 0, 0)) + b.Add(ZxyToID(3, 0, 1)) + b.Add(ZxyToID(3, 1, 0)) + b.Add(ZxyToID(3, 1, 1)) generalizeAnd(b) assert.Equal(t, uint64(5), b.GetCardinality()) - assert.True(t, b.Contains(ZxyToId(2, 0, 0))) + assert.True(t, b.Contains(ZxyToID(2, 0, 0))) } func TestGeneralizeOr(t *testing.T) { @@ -29,19 +29,19 @@ func TestGeneralizeOr(t *testing.T) { generalizeOr(b, 0) assert.Equal(t, uint64(0), b.GetCardinality()) b = roaring64.New() - b.Add(ZxyToId(3, 0, 0)) + b.Add(ZxyToID(3, 0, 0)) generalizeOr(b, 0) assert.Equal(t, uint64(4), b.GetCardinality()) - assert.True(t, b.Contains(ZxyToId(2, 0, 0))) - assert.True(t, b.Contains(ZxyToId(1, 0, 0))) - assert.True(t, b.Contains(ZxyToId(0, 0, 0))) + assert.True(t, b.Contains(ZxyToID(2, 0, 0))) + assert.True(t, b.Contains(ZxyToID(1, 0, 0))) + assert.True(t, b.Contains(ZxyToID(0, 0, 0))) } func TestGeneralizeOrMinZoom(t *testing.T) { b := roaring64.New() - b.Add(ZxyToId(3, 0, 0)) + b.Add(ZxyToID(3, 0, 0)) generalizeOr(b, 2) assert.Equal(t, uint64(2), b.GetCardinality()) - assert.True(t, b.Contains(ZxyToId(2, 0, 0))) - assert.False(t, b.Contains(ZxyToId(1, 0, 0))) + assert.True(t, b.Contains(ZxyToID(2, 0, 0))) + assert.False(t, b.Contains(ZxyToID(1, 0, 0))) } diff --git a/pmtiles/bucket.go b/pmtiles/bucket.go index d860288..7f46dac 100644 --- a/pmtiles/bucket.go +++ b/pmtiles/bucket.go @@ -18,11 +18,11 @@ type Bucket interface { NewRangeReader(ctx context.Context, key string, offset int64, length int64) (io.ReadCloser, error) } -type HttpBucket struct { +type HTTPBucket struct { baseURL string } -func (b HttpBucket) NewRangeReader(ctx context.Context, key string, offset, length int64) (io.ReadCloser, error) { +func (b HTTPBucket) NewRangeReader(ctx context.Context, key string, offset, length int64) (io.ReadCloser, error) { reqURL := b.baseURL + "/" + key req, err := http.NewRequest("GET", reqURL, nil) @@ -45,7 +45,7 @@ func (b HttpBucket) NewRangeReader(ctx context.Context, key string, offset, leng return resp.Body, nil } -func (b HttpBucket) Close() error { +func (b HTTPBucket) Close() error { return nil } @@ -102,7 +102,7 @@ func NormalizeBucketKey(bucket string, prefix string, key string) (string, strin func OpenBucket(ctx context.Context, bucketURL string, bucketPrefix string) (Bucket, error) { if strings.HasPrefix(bucketURL, "http") { - bucket := HttpBucket{bucketURL} + bucket := HTTPBucket{bucketURL} return bucket, nil } else { bucket, err := blob.OpenBucket(ctx, bucketURL) @@ -112,7 +112,7 @@ func OpenBucket(ctx context.Context, bucketURL string, bucketPrefix string) (Buc if bucketPrefix != "" && bucketPrefix != "/" && bucketPrefix != "." { bucket = blob.PrefixedBucket(bucket, path.Clean(bucketPrefix)+string(os.PathSeparator)) } - wrapped_bucket := BucketAdapter{bucket} - return wrapped_bucket, err + wrappedBucket := BucketAdapter{bucket} + return wrappedBucket, err } } diff --git a/pmtiles/convert.go b/pmtiles/convert.go index 1d53410..d13e35c 100644 --- a/pmtiles/convert.go +++ b/pmtiles/convert.go @@ -35,7 +35,7 @@ type Resolver struct { OffsetMap map[string]OffsetLen AddressedTiles uint64 // none of them can be empty compressor *gzip.Writer - compress_tmp *bytes.Buffer + compressTmp *bytes.Buffer hashfunc hash.Hash } @@ -48,51 +48,51 @@ func (r *Resolver) NumContents() uint64 { } // must be called in increasing tile_id order, uniquely -func (r *Resolver) AddTileIsNew(tile_id uint64, data []byte) (bool, []byte) { +func (r *Resolver) AddTileIsNew(tileID uint64, data []byte) (bool, []byte) { r.AddressedTiles++ var found OffsetLen var ok bool - var sum_string string + var sumString string if r.deduplicate { r.hashfunc.Reset() r.hashfunc.Write(data) var tmp []byte - sum_string = string(r.hashfunc.Sum(tmp)) - found, ok = r.OffsetMap[sum_string] + sumString = string(r.hashfunc.Sum(tmp)) + found, ok = r.OffsetMap[sumString] } if r.deduplicate && ok { - last_entry := r.Entries[len(r.Entries)-1] - if tile_id == last_entry.TileId+uint64(last_entry.RunLength) && last_entry.Offset == found.Offset { + lastEntry := r.Entries[len(r.Entries)-1] + if tileID == lastEntry.TileID+uint64(lastEntry.RunLength) && lastEntry.Offset == found.Offset { // RLE - if last_entry.RunLength+1 > math.MaxUint32 { + if lastEntry.RunLength+1 > math.MaxUint32 { panic("Maximum 32-bit run length exceeded") } r.Entries[len(r.Entries)-1].RunLength++ } else { - r.Entries = append(r.Entries, EntryV3{tile_id, found.Offset, found.Length, 1}) + r.Entries = append(r.Entries, EntryV3{tileID, found.Offset, found.Length, 1}) } return false, nil } else { - var new_data []byte + var newData []byte if !r.compress || (len(data) >= 2 && data[0] == 31 && data[1] == 139) { // the tile is already compressed - new_data = data + newData = data } else { - r.compress_tmp.Reset() - r.compressor.Reset(r.compress_tmp) + r.compressTmp.Reset() + r.compressor.Reset(r.compressTmp) r.compressor.Write(data) r.compressor.Close() - new_data = r.compress_tmp.Bytes() + newData = r.compressTmp.Bytes() } if r.deduplicate { - r.OffsetMap[sum_string] = OffsetLen{r.Offset, uint32(len(new_data))} + r.OffsetMap[sumString] = OffsetLen{r.Offset, uint32(len(newData))} } - r.Entries = append(r.Entries, EntryV3{tile_id, r.Offset, uint32(len(new_data)), 1}) - r.Offset += uint64(len(new_data)) - return true, new_data + r.Entries = append(r.Entries, EntryV3{tileID, r.Offset, uint32(len(newData)), 1}) + r.Offset += uint64(len(newData)) + return true, newData } } @@ -111,10 +111,10 @@ func Convert(logger *log.Logger, input string, output string, deduplicate bool, } } -func add_directoryv2_entries(dir DirectoryV2, entries *[]EntryV3, f *os.File) { +func addDirectoryV2Entries(dir DirectoryV2, entries *[]EntryV3, f *os.File) { for zxy, rng := range dir.Entries { - tile_id := ZxyToId(zxy.Z, zxy.X, zxy.Y) - *entries = append(*entries, EntryV3{tile_id, rng.Offset, uint32(rng.Length), 1}) + tileID := ZxyToID(zxy.Z, zxy.X, zxy.Y) + *entries = append(*entries, EntryV3{tileID, rng.Offset, uint32(rng.Length), 1}) } var unique = map[uint64]uint32{} @@ -126,18 +126,18 @@ func add_directoryv2_entries(dir DirectoryV2, entries *[]EntryV3, f *os.File) { for offset, length := range unique { f.Seek(int64(offset), 0) - leaf_bytes := make([]byte, length) - f.Read(leaf_bytes) - leaf_dir := ParseDirectoryV2(leaf_bytes) - add_directoryv2_entries(leaf_dir, entries, f) + leafBytes := make([]byte, length) + f.Read(leafBytes) + leafDir := ParseDirectoryV2(leafBytes) + addDirectoryV2Entries(leafDir, entries, f) } } -func set_zoom_center_defaults(header *HeaderV3, entries []EntryV3) { - min_z, _, _ := IdToZxy(entries[0].TileId) - header.MinZoom = min_z - max_z, _, _ := IdToZxy(entries[len(entries)-1].TileId) - header.MaxZoom = max_z +func setZoomCenterDefaults(header *HeaderV3, entries []EntryV3) { + minZ, _, _ := IDToZxy(entries[0].TileID) + header.MinZoom = minZ + maxZ, _, _ := IDToZxy(entries[len(entries)-1].TileID) + header.MaxZoom = maxZ if header.CenterZoom == 0 && header.CenterLonE7 == 0 && header.CenterLatE7 == 0 { header.CenterZoom = header.MinZoom @@ -156,13 +156,13 @@ func ConvertPmtilesV2(logger *log.Logger, input string, output string, deduplica buffer := make([]byte, 512000) io.ReadFull(f, buffer) if string(buffer[0:7]) == "PMTiles" && buffer[7] == 3 { - return fmt.Errorf("Archive is already the latest PMTiles version (3).") + return fmt.Errorf("archive is already the latest PMTiles version (3)") } - v2_json_bytes, dir := ParseHeaderV2(bytes.NewReader(buffer)) + v2JsonBytes, dir := ParseHeaderV2(bytes.NewReader(buffer)) - var v2_metadata map[string]interface{} - json.Unmarshal(v2_json_bytes, &v2_metadata) + var v2metadata map[string]interface{} + json.Unmarshal(v2JsonBytes, &v2metadata) // get the first 4 bytes at offset 512000 to attempt tile type detection @@ -173,18 +173,18 @@ func ConvertPmtilesV2(logger *log.Logger, input string, output string, deduplica return fmt.Errorf("Failed to read first 4, %w", err) } - header, json_metadata, err := v2_to_header_json(v2_metadata, first4) + header, jsonMetadata, err := v2ToHeaderJSON(v2metadata, first4) if err != nil { return fmt.Errorf("Failed to convert v2 to header JSON, %w", err) } entries := make([]EntryV3, 0) - add_directoryv2_entries(dir, &entries, f) + addDirectoryV2Entries(dir, &entries, f) // sort sort.Slice(entries, func(i, j int) bool { - return entries[i].TileId < entries[j].TileId + return entries[i].TileID < entries[j].TileID }) // re-use resolver, because even if archives are de-duplicated we may need to recompress. @@ -207,8 +207,8 @@ func ConvertPmtilesV2(logger *log.Logger, input string, output string, deduplica } } // TODO: enforce sorted order - if is_new, new_data := resolver.AddTileIsNew(entry.TileId, buf); is_new { - _, err = tmpfile.Write(new_data) + if isNew, newData := resolver.AddTileIsNew(entry.TileID, buf); isNew { + _, err = tmpfile.Write(newData) if err != nil { return fmt.Errorf("Failed to write to tempfile, %w", err) } @@ -216,7 +216,7 @@ func ConvertPmtilesV2(logger *log.Logger, input string, output string, deduplica bar.Add(1) } - err = finalize(logger, resolver, header, tmpfile, output, json_metadata) + err = finalize(logger, resolver, header, tmpfile, output, jsonMetadata) if err != nil { return err } @@ -233,7 +233,7 @@ func ConvertMbtiles(logger *log.Logger, input string, output string, deduplicate } defer conn.Close() - mbtiles_metadata := make([]string, 0) + mbtilesMetadata := make([]string, 0) { stmt, _, err := conn.PrepareTransient("SELECT name, value FROM metadata") if err != nil { @@ -249,11 +249,11 @@ func ConvertMbtiles(logger *log.Logger, input string, output string, deduplicate if !row { break } - mbtiles_metadata = append(mbtiles_metadata, stmt.ColumnText(0)) - mbtiles_metadata = append(mbtiles_metadata, stmt.ColumnText(1)) + mbtilesMetadata = append(mbtilesMetadata, stmt.ColumnText(0)) + mbtilesMetadata = append(mbtilesMetadata, stmt.ColumnText(1)) } } - header, json_metadata, err := mbtiles_to_header_json(mbtiles_metadata) + header, jsonMetadata, err := mbtilesToHeaderJSON(mbtilesMetadata) if err != nil { return fmt.Errorf("Failed to convert MBTiles to header JSON, %w", err) @@ -261,7 +261,7 @@ func ConvertMbtiles(logger *log.Logger, input string, output string, deduplicate logger.Println("Querying total tile count...") // determine the count - var total_tiles int64 + var totalTiles int64 { stmt, _, err := conn.PrepareTransient("SELECT count(*) FROM tiles") if err != nil { @@ -272,7 +272,7 @@ func ConvertMbtiles(logger *log.Logger, input string, output string, deduplicate if err != nil || !row { return fmt.Errorf("Failed to step row, %w", err) } - total_tiles = stmt.ColumnInt64(0) + totalTiles = stmt.ColumnInt64(0) } logger.Println("Pass 1: Assembling TileID set") @@ -285,7 +285,7 @@ func ConvertMbtiles(logger *log.Logger, input string, output string, deduplicate } defer stmt.Finalize() - bar := progressbar.Default(total_tiles) + bar := progressbar.Default(totalTiles) for { row, err := stmt.Step() @@ -298,15 +298,15 @@ func ConvertMbtiles(logger *log.Logger, input string, output string, deduplicate z := uint8(stmt.ColumnInt64(0)) x := uint32(stmt.ColumnInt64(1)) y := uint32(stmt.ColumnInt64(2)) - flipped_y := (1 << z) - 1 - y - id := ZxyToId(z, x, flipped_y) + flippedY := (1 << z) - 1 - y + id := ZxyToID(z, x, flippedY) tileset.Add(id) bar.Add(1) } } if tileset.GetCardinality() == 0 { - return fmt.Errorf("No tiles in MBTiles archive.") + return fmt.Errorf("no tiles in MBTiles archive") } logger.Println("Pass 2: writing tiles") @@ -316,33 +316,33 @@ func ConvertMbtiles(logger *log.Logger, input string, output string, deduplicate i := tileset.Iterator() stmt := conn.Prep("SELECT tile_data FROM tiles WHERE zoom_level = ? AND tile_column = ? AND tile_row = ?") - var raw_tile_tmp bytes.Buffer + var rawTileTmp bytes.Buffer for i.HasNext() { id := i.Next() - z, x, y := IdToZxy(id) - flipped_y := (1 << z) - 1 - y + z, x, y := IDToZxy(id) + flippedY := (1 << z) - 1 - y stmt.BindInt64(1, int64(z)) stmt.BindInt64(2, int64(x)) - stmt.BindInt64(3, int64(flipped_y)) + stmt.BindInt64(3, int64(flippedY)) - has_row, err := stmt.Step() + hasRow, err := stmt.Step() if err != nil { return fmt.Errorf("Failed to step statement, %w", err) } - if !has_row { + if !hasRow { return fmt.Errorf("Missing row") } reader := stmt.ColumnReader(0) - raw_tile_tmp.Reset() - raw_tile_tmp.ReadFrom(reader) - data := raw_tile_tmp.Bytes() + rawTileTmp.Reset() + rawTileTmp.ReadFrom(reader) + data := rawTileTmp.Bytes() if len(data) > 0 { - if is_new, new_data := resolver.AddTileIsNew(id, data); is_new { - _, err := tmpfile.Write(new_data) + if isNew, newData := resolver.AddTileIsNew(id, data); isNew { + _, err := tmpfile.Write(newData) if err != nil { return fmt.Errorf("Failed to write to tempfile: %s", err) } @@ -354,7 +354,7 @@ func ConvertMbtiles(logger *log.Logger, input string, output string, deduplicate bar.Add(1) } } - err = finalize(logger, resolver, header, tmpfile, output, json_metadata) + err = finalize(logger, resolver, header, tmpfile, output, jsonMetadata) if err != nil { return err } @@ -362,7 +362,7 @@ func ConvertMbtiles(logger *log.Logger, input string, output string, deduplicate return nil } -func finalize(logger *log.Logger, resolver *Resolver, header HeaderV3, tmpfile *os.File, output string, json_metadata map[string]interface{}) error { +func finalize(logger *log.Logger, resolver *Resolver, header HeaderV3, tmpfile *os.File, output string, jsonMetadata map[string]interface{}) error { logger.Println("# of addressed tiles: ", resolver.AddressedTiles) logger.Println("# of tile entries (after RLE): ", len(resolver.Entries)) logger.Println("# of tile contents: ", resolver.NumContents()) @@ -377,34 +377,34 @@ func finalize(logger *log.Logger, resolver *Resolver, header HeaderV3, tmpfile * return fmt.Errorf("Failed to create %s, %w", output, err) } - root_bytes, leaves_bytes, num_leaves := optimize_directories(resolver.Entries, 16384-HEADERV3_LEN_BYTES) + rootBytes, leavesBytes, numLeaves := optimizeDirectories(resolver.Entries, 16384-HeaderV3LenBytes) - if num_leaves > 0 { - logger.Println("Root dir bytes: ", len(root_bytes)) - logger.Println("Leaves dir bytes: ", len(leaves_bytes)) - logger.Println("Num leaf dirs: ", num_leaves) - logger.Println("Total dir bytes: ", len(root_bytes)+len(leaves_bytes)) - logger.Println("Average leaf dir bytes: ", len(leaves_bytes)/num_leaves) - logger.Printf("Average bytes per addressed tile: %.2f\n", float64(len(root_bytes)+len(leaves_bytes))/float64(resolver.AddressedTiles)) + if numLeaves > 0 { + logger.Println("Root dir bytes: ", len(rootBytes)) + logger.Println("Leaves dir bytes: ", len(leavesBytes)) + logger.Println("Num leaf dirs: ", numLeaves) + logger.Println("Total dir bytes: ", len(rootBytes)+len(leavesBytes)) + logger.Println("Average leaf dir bytes: ", len(leavesBytes)/numLeaves) + logger.Printf("Average bytes per addressed tile: %.2f\n", float64(len(rootBytes)+len(leavesBytes))/float64(resolver.AddressedTiles)) } else { - logger.Println("Total dir bytes: ", len(root_bytes)) - logger.Printf("Average bytes per addressed tile: %.2f\n", float64(len(root_bytes))/float64(resolver.AddressedTiles)) + logger.Println("Total dir bytes: ", len(rootBytes)) + logger.Printf("Average bytes per addressed tile: %.2f\n", float64(len(rootBytes))/float64(resolver.AddressedTiles)) } - var metadata_bytes []byte + var metadataBytes []byte { - metadata_bytes_uncompressed, err := json.Marshal(json_metadata) + metadataBytesUncompressed, err := json.Marshal(jsonMetadata) if err != nil { return fmt.Errorf("Failed to marshal metadata, %w", err) } var b bytes.Buffer w, _ := gzip.NewWriterLevel(&b, gzip.BestCompression) - w.Write(metadata_bytes_uncompressed) + w.Write(metadataBytesUncompressed) w.Close() - metadata_bytes = b.Bytes() + metadataBytes = b.Bytes() } - set_zoom_center_defaults(&header, resolver.Entries) + setZoomCenterDefaults(&header, resolver.Entries) header.Clustered = true header.InternalCompression = Gzip @@ -412,30 +412,30 @@ func finalize(logger *log.Logger, resolver *Resolver, header HeaderV3, tmpfile * header.TileCompression = Gzip } - header.RootOffset = HEADERV3_LEN_BYTES - header.RootLength = uint64(len(root_bytes)) + header.RootOffset = HeaderV3LenBytes + header.RootLength = uint64(len(rootBytes)) header.MetadataOffset = header.RootOffset + header.RootLength - header.MetadataLength = uint64(len(metadata_bytes)) + header.MetadataLength = uint64(len(metadataBytes)) header.LeafDirectoryOffset = header.MetadataOffset + header.MetadataLength - header.LeafDirectoryLength = uint64(len(leaves_bytes)) + header.LeafDirectoryLength = uint64(len(leavesBytes)) header.TileDataOffset = header.LeafDirectoryOffset + header.LeafDirectoryLength header.TileDataLength = resolver.Offset - header_bytes := serialize_header(header) + headerBytes := serializeHeader(header) - _, err = outfile.Write(header_bytes) + _, err = outfile.Write(headerBytes) if err != nil { return fmt.Errorf("Failed to write header to outfile, %w", err) } - _, err = outfile.Write(root_bytes) + _, err = outfile.Write(rootBytes) if err != nil { return fmt.Errorf("Failed to write header to outfile, %w", err) } - _, err = outfile.Write(metadata_bytes) + _, err = outfile.Write(metadataBytes) if err != nil { return fmt.Errorf("Failed to write header to outfile, %w", err) } - _, err = outfile.Write(leaves_bytes) + _, err = outfile.Write(leavesBytes) if err != nil { return fmt.Errorf("Failed to write header to outfile, %w", err) } @@ -451,40 +451,40 @@ func finalize(logger *log.Logger, resolver *Resolver, header HeaderV3, tmpfile * return nil } -func v2_to_header_json(v2_json_metadata map[string]interface{}, first4 []byte) (HeaderV3, map[string]interface{}, error) { +func v2ToHeaderJSON(v2JsonMetadata map[string]interface{}, first4 []byte) (HeaderV3, map[string]interface{}, error) { header := HeaderV3{} - if val, ok := v2_json_metadata["bounds"]; ok { - min_lon, min_lat, max_lon, max_lat, err := parse_bounds(val.(string)) + if val, ok := v2JsonMetadata["bounds"]; ok { + minLon, minLat, maxLon, maxLat, err := parseBounds(val.(string)) if err != nil { - return header, v2_json_metadata, err + return header, v2JsonMetadata, err } - header.MinLonE7 = min_lon - header.MinLatE7 = min_lat - header.MaxLonE7 = max_lon - header.MaxLatE7 = max_lat - delete(v2_json_metadata, "bounds") + header.MinLonE7 = minLon + header.MinLatE7 = minLat + header.MaxLonE7 = maxLon + header.MaxLatE7 = maxLat + delete(v2JsonMetadata, "bounds") } else { - return header, v2_json_metadata, errors.New("Archive is missing bounds.") + return header, v2JsonMetadata, errors.New("archive is missing bounds") } - if val, ok := v2_json_metadata["center"]; ok { - center_lon, center_lat, center_zoom, err := parse_center(val.(string)) + if val, ok := v2JsonMetadata["center"]; ok { + centerLon, centerLat, centerZoom, err := parseCenter(val.(string)) if err != nil { - return header, v2_json_metadata, err + return header, v2JsonMetadata, err } - header.CenterLonE7 = center_lon - header.CenterLatE7 = center_lat - header.CenterZoom = center_zoom - delete(v2_json_metadata, "center") + header.CenterLonE7 = centerLon + header.CenterLatE7 = centerLat + header.CenterZoom = centerZoom + delete(v2JsonMetadata, "center") } - if val, ok := v2_json_metadata["compression"]; ok { + if val, ok := v2JsonMetadata["compression"]; ok { switch val.(string) { case "gzip": header.TileCompression = Gzip default: - return header, v2_json_metadata, errors.New("Unknown compression type") + return header, v2JsonMetadata, errors.New("Unknown compression type") } } else { if first4[0] == 0x1f && first4[1] == 0x8b { @@ -492,7 +492,7 @@ func v2_to_header_json(v2_json_metadata map[string]interface{}, first4 []byte) ( } } - if val, ok := v2_json_metadata["format"]; ok { + if val, ok := v2JsonMetadata["format"]; ok { switch val.(string) { case "pbf": header.TileType = Mvt @@ -509,7 +509,7 @@ func v2_to_header_json(v2_json_metadata map[string]interface{}, first4 []byte) ( header.TileType = Avif header.TileCompression = NoCompression default: - return header, v2_json_metadata, errors.New("Unknown tile type") + return header, v2JsonMetadata, errors.New("Unknown tile type") } } else { if first4[0] == 0x89 && first4[1] == 0x50 && first4[2] == 0x4e && first4[3] == 0x47 { @@ -526,66 +526,66 @@ func v2_to_header_json(v2_json_metadata map[string]interface{}, first4 []byte) ( // deserialize embedded JSON and lift keys to top-level // to avoid "json-in-json" - if val, ok := v2_json_metadata["json"]; ok { - string_val := val.(string) + if val, ok := v2JsonMetadata["json"]; ok { + stringVal := val.(string) var inside map[string]interface{} - json.Unmarshal([]byte(string_val), &inside) + json.Unmarshal([]byte(stringVal), &inside) for k, v := range inside { - v2_json_metadata[k] = v + v2JsonMetadata[k] = v } - delete(v2_json_metadata, "json") + delete(v2JsonMetadata, "json") } - return header, v2_json_metadata, nil + return header, v2JsonMetadata, nil } -func parse_bounds(bounds string) (int32, int32, int32, int32, error) { +func parseBounds(bounds string) (int32, int32, int32, int32, error) { parts := strings.Split(bounds, ",") E7 := 10000000.0 - min_lon, err := strconv.ParseFloat(parts[0], 64) + minLon, err := strconv.ParseFloat(parts[0], 64) if err != nil { return 0, 0, 0, 0, err } - min_lat, err := strconv.ParseFloat(parts[1], 64) + minLat, err := strconv.ParseFloat(parts[1], 64) if err != nil { return 0, 0, 0, 0, err } - max_lon, err := strconv.ParseFloat(parts[2], 64) + maxLon, err := strconv.ParseFloat(parts[2], 64) if err != nil { return 0, 0, 0, 0, err } - max_lat, err := strconv.ParseFloat(parts[3], 64) + maxLat, err := strconv.ParseFloat(parts[3], 64) if err != nil { return 0, 0, 0, 0, err } - return int32(min_lon * E7), int32(min_lat * E7), int32(max_lon * E7), int32(max_lat * E7), nil + return int32(minLon * E7), int32(minLat * E7), int32(maxLon * E7), int32(maxLat * E7), nil } -func parse_center(center string) (int32, int32, uint8, error) { +func parseCenter(center string) (int32, int32, uint8, error) { parts := strings.Split(center, ",") E7 := 10000000.0 - center_lon, err := strconv.ParseFloat(parts[0], 64) + centerLon, err := strconv.ParseFloat(parts[0], 64) if err != nil { return 0, 0, 0, err } - center_lat, err := strconv.ParseFloat(parts[1], 64) + centerLat, err := strconv.ParseFloat(parts[1], 64) if err != nil { return 0, 0, 0, err } - center_zoom, err := strconv.ParseInt(parts[2], 10, 8) + centerZoom, err := strconv.ParseInt(parts[2], 10, 8) if err != nil { return 0, 0, 0, err } - return int32(center_lon * E7), int32(center_lat * E7), uint8(center_zoom), nil + return int32(centerLon * E7), int32(centerLat * E7), uint8(centerZoom), nil } -func mbtiles_to_header_json(mbtiles_metadata []string) (HeaderV3, map[string]interface{}, error) { +func mbtilesToHeaderJSON(mbtilesMetadata []string) (HeaderV3, map[string]interface{}, error) { header := HeaderV3{} - json_result := make(map[string]interface{}) + jsonResult := make(map[string]interface{}) boundsSet := false - for i := 0; i < len(mbtiles_metadata); i += 2 { - value := mbtiles_metadata[i+1] - switch key := mbtiles_metadata[i]; key { + for i := 0; i < len(mbtilesMetadata); i += 2 { + value := mbtilesMetadata[i+1] + switch key := mbtilesMetadata[i]; key { case "format": switch value { case "pbf": @@ -603,34 +603,34 @@ func mbtiles_to_header_json(mbtiles_metadata []string) (HeaderV3, map[string]int header.TileType = Avif header.TileCompression = NoCompression } - json_result["format"] = value + jsonResult["format"] = value case "bounds": - min_lon, min_lat, max_lon, max_lat, err := parse_bounds(value) + minLon, minLat, maxLon, maxLat, err := parseBounds(value) if err != nil { - return header, json_result, err + return header, jsonResult, err } - if min_lon >= max_lon || min_lat >= max_lat { - return header, json_result, fmt.Errorf("Error: zero-area bounds in mbtiles metadata.") + if minLon >= maxLon || minLat >= maxLat { + return header, jsonResult, fmt.Errorf("zero-area bounds in mbtiles metadata") } - header.MinLonE7 = min_lon - header.MinLatE7 = min_lat - header.MaxLonE7 = max_lon - header.MaxLatE7 = max_lat + header.MinLonE7 = minLon + header.MinLatE7 = minLat + header.MaxLonE7 = maxLon + header.MaxLatE7 = maxLat boundsSet = true case "center": - center_lon, center_lat, center_zoom, err := parse_center(value) + centerLon, centerLat, centerZoom, err := parseCenter(value) if err != nil { - return header, json_result, err + return header, jsonResult, err } - header.CenterLonE7 = center_lon - header.CenterLatE7 = center_lat - header.CenterZoom = center_zoom + header.CenterLonE7 = centerLon + header.CenterLatE7 = centerLat + header.CenterZoom = centerZoom case "json": - var mbtiles_json map[string]interface{} - json.Unmarshal([]byte(value), &mbtiles_json) - for k, v := range mbtiles_json { - json_result[k] = v + var mbtilesJSON map[string]interface{} + json.Unmarshal([]byte(value), &mbtilesJSON) + for k, v := range mbtilesJSON { + jsonResult[k] = v } case "compression": switch value { @@ -641,10 +641,10 @@ func mbtiles_to_header_json(mbtiles_metadata []string) (HeaderV3, map[string]int header.TileCompression = NoCompression } } - json_result["compression"] = value + jsonResult["compression"] = value // name, attribution, description, type, version default: - json_result[key] = value + jsonResult[key] = value } } @@ -656,5 +656,5 @@ func mbtiles_to_header_json(mbtiles_metadata []string) (HeaderV3, map[string]int header.MaxLatE7 = int32(85 * E7) } - return header, json_result, nil + return header, jsonResult, nil } diff --git a/pmtiles/convert_test.go b/pmtiles/convert_test.go index a0be0ae..dbb2eee 100644 --- a/pmtiles/convert_test.go +++ b/pmtiles/convert_test.go @@ -11,8 +11,8 @@ func TestResolver(t *testing.T) { assert.Equal(t, 1, len(resolver.Entries)) resolver.AddTileIsNew(2, []byte{0x1, 0x3}) assert.Equal(t, uint64(52), resolver.Offset) - is_new, _ := resolver.AddTileIsNew(3, []byte{0x1, 0x2}) - assert.False(t, is_new) + isNew, _ := resolver.AddTileIsNew(3, []byte{0x1, 0x2}) + assert.False(t, isNew) assert.Equal(t, uint64(52), resolver.Offset) resolver.AddTileIsNew(4, []byte{0x1, 0x2}) assert.Equal(t, 3, len(resolver.Entries)) @@ -21,18 +21,18 @@ func TestResolver(t *testing.T) { } func TestV2UpgradeBarebones(t *testing.T) { - header, json_metadata, err := v2_to_header_json(map[string]interface{}{ + header, jsonMetadata, err := v2ToHeaderJSON(map[string]interface{}{ "bounds": "-180.0,-85,178,83", "attribution": "abcd", }, []byte{0x1f, 0x8b, 0x0, 0x0}) assert.Nil(t, err) - _, ok := json_metadata["attribution"] + _, ok := jsonMetadata["attribution"] assert.True(t, ok) assert.Equal(t, int32(-180*10000000), header.MinLonE7) assert.Equal(t, int32(-85*10000000), header.MinLatE7) assert.Equal(t, int32(178*10000000), header.MaxLonE7) assert.Equal(t, int32(83*10000000), header.MaxLatE7) - _, ok = json_metadata["bounds"] + _, ok = jsonMetadata["bounds"] assert.False(t, ok) assert.Equal(t, Gzip, int(header.TileCompression)) assert.Equal(t, Mvt, int(header.TileType)) @@ -40,7 +40,7 @@ func TestV2UpgradeBarebones(t *testing.T) { func TestV2UpgradeExtra(t *testing.T) { // with the fields tippecanoe usually has - header, json_metadata, err := v2_to_header_json(map[string]interface{}{ + header, jsonMetadata, err := v2ToHeaderJSON(map[string]interface{}{ "bounds": "-180.0,-85,180,85", "center": "-122.1906,37.7599,11", "format": "pbf", @@ -51,9 +51,9 @@ func TestV2UpgradeExtra(t *testing.T) { assert.Equal(t, int32(-122.1906*10000000), header.CenterLonE7) assert.Equal(t, int32(37.7599*10000000), header.CenterLatE7) assert.Equal(t, uint8(11), header.CenterZoom) - _, ok := json_metadata["center"] + _, ok := jsonMetadata["center"] assert.False(t, ok) - _, ok = json_metadata["abc"] + _, ok = jsonMetadata["abc"] assert.True(t, ok) } @@ -65,9 +65,9 @@ func TestZoomCenterDefaults(t *testing.T) { header.MinLatE7 = 21 * 10000000 header.MaxLatE7 = 23 * 10000000 entries := make([]EntryV3, 0) - entries = append(entries, EntryV3{ZxyToId(3, 0, 0), 0, 0, 0}) - entries = append(entries, EntryV3{ZxyToId(4, 0, 0), 1, 1, 1}) - set_zoom_center_defaults(&header, entries) + entries = append(entries, EntryV3{ZxyToID(3, 0, 0), 0, 0, 0}) + entries = append(entries, EntryV3{ZxyToID(4, 0, 0), 1, 1, 1}) + setZoomCenterDefaults(&header, entries) assert.Equal(t, uint8(3), header.MinZoom) assert.Equal(t, uint8(4), header.MaxZoom) assert.Equal(t, uint8(3), header.CenterZoom) @@ -83,31 +83,31 @@ func TestZoomCenterDefaults(t *testing.T) { header.CenterLonE7 = header.MinLonE7 header.CenterLatE7 = header.MinLatE7 header.CenterZoom = 4 - set_zoom_center_defaults(&header, entries) + setZoomCenterDefaults(&header, entries) assert.Equal(t, uint8(4), header.CenterZoom) assert.Equal(t, int32(-45*10000000), header.CenterLonE7) assert.Equal(t, int32(21*10000000), header.CenterLatE7) } func TestV2UpgradeInfer(t *testing.T) { - header, _, err := v2_to_header_json(map[string]interface{}{ + header, _, err := v2ToHeaderJSON(map[string]interface{}{ "bounds": "-180.0,-85,180,85", }, []byte{0xff, 0xd8, 0xff, 0xe0}) assert.Nil(t, err) assert.Equal(t, Jpeg, int(header.TileType)) assert.Equal(t, NoCompression, int(header.TileCompression)) - header, _, err = v2_to_header_json(map[string]interface{}{ + header, _, err = v2ToHeaderJSON(map[string]interface{}{ "bounds": "-180.0,-85,180,85", }, []byte{0x89, 0x50, 0x4e, 0x47}) assert.Nil(t, err) assert.Equal(t, Png, int(header.TileType)) assert.Equal(t, NoCompression, int(header.TileCompression)) - header, _, err = v2_to_header_json(map[string]interface{}{ + header, _, err = v2ToHeaderJSON(map[string]interface{}{ "bounds": "-180.0,-85,180,85", }, []byte{0x00, 00, 00, 00}) assert.Equal(t, Mvt, int(header.TileType)) assert.Equal(t, UnknownCompression, header.TileCompression) - header, _, err = v2_to_header_json(map[string]interface{}{ + header, _, err = v2ToHeaderJSON(map[string]interface{}{ "bounds": "-180.0,-85,180,85", }, []byte{0x1f, 0x8b, 00, 00}) assert.Nil(t, err) @@ -116,7 +116,7 @@ func TestV2UpgradeInfer(t *testing.T) { } func TestMbtiles(t *testing.T) { - header, json_metadata, err := mbtiles_to_header_json([]string{ + header, jsonMetadata, err := mbtilesToHeaderJSON([]string{ "name", "test_name", "format", "pbf", "bounds", "-180.0,-85,180,85", @@ -141,43 +141,43 @@ func TestMbtiles(t *testing.T) { // assert removal of redundant fields - _, ok := json_metadata["center"] + _, ok := jsonMetadata["center"] assert.False(t, ok) - _, ok = json_metadata["bounds"] + _, ok = jsonMetadata["bounds"] assert.False(t, ok) // assert preservation of metadata fields for roundtrip - _, ok = json_metadata["name"] + _, ok = jsonMetadata["name"] assert.True(t, ok) - _, ok = json_metadata["format"] + _, ok = jsonMetadata["format"] assert.True(t, ok) - _, ok = json_metadata["attribution"] + _, ok = jsonMetadata["attribution"] assert.True(t, ok) - _, ok = json_metadata["description"] + _, ok = jsonMetadata["description"] assert.True(t, ok) - _, ok = json_metadata["type"] + _, ok = jsonMetadata["type"] assert.True(t, ok) - _, ok = json_metadata["version"] + _, ok = jsonMetadata["version"] assert.True(t, ok) - _, ok = json_metadata["compression"] + _, ok = jsonMetadata["compression"] assert.True(t, ok) // assert well-known json fields at top level - _, ok = json_metadata["vector_layers"] + _, ok = jsonMetadata["vector_layers"] assert.True(t, ok) - _, ok = json_metadata["tilestats"] + _, ok = jsonMetadata["tilestats"] assert.True(t, ok) } func TestMbtilesMissingBoundsCenter(t *testing.T) { - header, _, err := mbtiles_to_header_json([]string{ + header, _, err := mbtilesToHeaderJSON([]string{ "name", "test_name", "format", "pbf", "attribution", "
abc
", @@ -197,7 +197,7 @@ func TestMbtilesMissingBoundsCenter(t *testing.T) { } func TestMbtilesDegenerateBounds(t *testing.T) { - _, _, err := mbtiles_to_header_json([]string{ + _, _, err := mbtilesToHeaderJSON([]string{ "name", "test_name", "format", "pbf", "bounds", "0,0,0,0", diff --git a/pmtiles/directory.go b/pmtiles/directory.go index e40c529..3a7463a 100644 --- a/pmtiles/directory.go +++ b/pmtiles/directory.go @@ -29,7 +29,7 @@ const ( Avif = 5 ) -const HEADERV3_LEN_BYTES = 127 +const HeaderV3LenBytes = 127 type HeaderV3 struct { SpecVersion uint8 @@ -105,13 +105,13 @@ func headerContentEncoding(compression Compression) (string, bool) { } type EntryV3 struct { - TileId uint64 + TileID uint64 Offset uint64 Length uint32 RunLength uint32 } -func serialize_entries(entries []EntryV3) []byte { +func serializeEntries(entries []EntryV3) []byte { var b bytes.Buffer tmp := make([]byte, binary.MaxVarintLen64) w, _ := gzip.NewWriterLevel(&b, gzip.BestCompression) @@ -120,12 +120,12 @@ func serialize_entries(entries []EntryV3) []byte { n = binary.PutUvarint(tmp, uint64(len(entries))) w.Write(tmp[:n]) - lastId := uint64(0) + lastID := uint64(0) for _, entry := range entries { - n = binary.PutUvarint(tmp, uint64(entry.TileId)-lastId) + n = binary.PutUvarint(tmp, uint64(entry.TileID)-lastID) w.Write(tmp[:n]) - lastId = uint64(entry.TileId) + lastID = uint64(entry.TileID) } for _, entry := range entries { @@ -152,33 +152,33 @@ func serialize_entries(entries []EntryV3) []byte { return b.Bytes() } -func deserialize_entries(data *bytes.Buffer) []EntryV3 { +func deserializeEntries(data *bytes.Buffer) []EntryV3 { entries := make([]EntryV3, 0) reader, _ := gzip.NewReader(data) - byte_reader := bufio.NewReader(reader) + byteReader := bufio.NewReader(reader) - num_entries, _ := binary.ReadUvarint(byte_reader) + numEntries, _ := binary.ReadUvarint(byteReader) - last_id := uint64(0) - for i := uint64(0); i < num_entries; i++ { - tmp, _ := binary.ReadUvarint(byte_reader) - entries = append(entries, EntryV3{last_id + tmp, 0, 0, 0}) - last_id = last_id + tmp + lastID := uint64(0) + for i := uint64(0); i < numEntries; i++ { + tmp, _ := binary.ReadUvarint(byteReader) + entries = append(entries, EntryV3{lastID + tmp, 0, 0, 0}) + lastID = lastID + tmp } - for i := uint64(0); i < num_entries; i++ { - run_length, _ := binary.ReadUvarint(byte_reader) - entries[i].RunLength = uint32(run_length) + for i := uint64(0); i < numEntries; i++ { + runLength, _ := binary.ReadUvarint(byteReader) + entries[i].RunLength = uint32(runLength) } - for i := uint64(0); i < num_entries; i++ { - length, _ := binary.ReadUvarint(byte_reader) + for i := uint64(0); i < numEntries; i++ { + length, _ := binary.ReadUvarint(byteReader) entries[i].Length = uint32(length) } - for i := uint64(0); i < num_entries; i++ { - tmp, _ := binary.ReadUvarint(byte_reader) + for i := uint64(0); i < numEntries; i++ { + tmp, _ := binary.ReadUvarint(byteReader) if i > 0 && tmp == 0 { entries[i].Offset = entries[i-1].Offset + uint64(entries[i-1].Length) } else { @@ -189,12 +189,12 @@ func deserialize_entries(data *bytes.Buffer) []EntryV3 { return entries } -func find_tile(entries []EntryV3, tileId uint64) (EntryV3, bool) { +func findTile(entries []EntryV3, tileID uint64) (EntryV3, bool) { m := 0 n := len(entries) - 1 for m <= n { k := (n + m) >> 1 - cmp := int64(tileId) - int64(entries[k].TileId) + cmp := int64(tileID) - int64(entries[k].TileID) if cmp > 0 { m = k + 1 } else if cmp < 0 { @@ -209,15 +209,15 @@ func find_tile(entries []EntryV3, tileId uint64) (EntryV3, bool) { if entries[n].RunLength == 0 { return entries[n], true } - if tileId-entries[n].TileId < uint64(entries[n].RunLength) { + if tileID-entries[n].TileID < uint64(entries[n].RunLength) { return entries[n], true } } return EntryV3{}, false } -func serialize_header(header HeaderV3) []byte { - b := make([]byte, HEADERV3_LEN_BYTES) +func serializeHeader(header HeaderV3) []byte { + b := make([]byte, HeaderV3LenBytes) copy(b[0:7], "PMTiles") b[7] = 3 @@ -250,19 +250,19 @@ func serialize_header(header HeaderV3) []byte { return b } -func deserialize_header(d []byte) (HeaderV3, error) { +func deserializeHeader(d []byte) (HeaderV3, error) { h := HeaderV3{} - magic_number := d[0:7] - if string(magic_number) != "PMTiles" { - return h, fmt.Errorf("Magic number not detected. Are you sure this is a PMTiles archive?") + magicNumber := d[0:7] + if string(magicNumber) != "PMTiles" { + return h, fmt.Errorf("magic number not detected. confirm this is a PMTiles archive") } - spec_version := d[7] - if spec_version > uint8(3) { - return h, fmt.Errorf("Archive is spec version %d, but this program only supports version 3: upgrade your pmtiles program.", spec_version) + specVersion := d[7] + if specVersion > uint8(3) { + return h, fmt.Errorf("archive is spec version %d, but this program only supports version 3: upgrade your pmtiles program", specVersion) } - h.SpecVersion = spec_version + h.SpecVersion = specVersion h.RootOffset = binary.LittleEndian.Uint64(d[8 : 8+8]) h.RootLength = binary.LittleEndian.Uint64(d[16 : 16+8]) h.MetadataOffset = binary.LittleEndian.Uint64(d[24 : 24+8]) @@ -291,33 +291,33 @@ func deserialize_header(d []byte) (HeaderV3, error) { return h, nil } -func build_roots_leaves(entries []EntryV3, leaf_size int) ([]byte, []byte, int) { - root_entries := make([]EntryV3, 0) - leaves_bytes := make([]byte, 0) - num_leaves := 0 +func buildRootsLeaves(entries []EntryV3, leafSize int) ([]byte, []byte, int) { + rootEntries := make([]EntryV3, 0) + leavesBytes := make([]byte, 0) + numLeaves := 0 - for idx := 0; idx < len(entries); idx += leaf_size { - num_leaves++ - end := idx + leaf_size - if idx+leaf_size > len(entries) { + for idx := 0; idx < len(entries); idx += leafSize { + numLeaves++ + end := idx + leafSize + if idx+leafSize > len(entries) { end = len(entries) } - serialized := serialize_entries(entries[idx:end]) + serialized := serializeEntries(entries[idx:end]) - root_entries = append(root_entries, EntryV3{entries[idx].TileId, uint64(len(leaves_bytes)), uint32(len(serialized)), 0}) - leaves_bytes = append(leaves_bytes, serialized...) + rootEntries = append(rootEntries, EntryV3{entries[idx].TileID, uint64(len(leavesBytes)), uint32(len(serialized)), 0}) + leavesBytes = append(leavesBytes, serialized...) } - root_bytes := serialize_entries(root_entries) - return root_bytes, leaves_bytes, num_leaves + rootBytes := serializeEntries(rootEntries) + return rootBytes, leavesBytes, numLeaves } -func optimize_directories(entries []EntryV3, target_root_len int) ([]byte, []byte, int) { +func optimizeDirectories(entries []EntryV3, targetRootLen int) ([]byte, []byte, int) { if len(entries) < 16384 { - test_root_bytes := serialize_entries(entries) + testRootBytes := serializeEntries(entries) // Case1: the entire directory fits into the target len - if len(test_root_bytes) <= target_root_len { - return test_root_bytes, make([]byte, 0), 0 + if len(testRootBytes) <= targetRootLen { + return testRootBytes, make([]byte, 0), 0 } } @@ -326,18 +326,18 @@ func optimize_directories(entries []EntryV3, target_root_len int) ([]byte, []byt // case 3: root directory is leaf pointers only // use an iterative method, increasing the size of the leaf directory until the root fits - var leaf_size float32 - leaf_size = float32(len(entries)) / 3500 + var leafSize float32 + leafSize = float32(len(entries)) / 3500 - if leaf_size < 4096 { - leaf_size = 4096 + if leafSize < 4096 { + leafSize = 4096 } for { - root_bytes, leaves_bytes, num_leaves := build_roots_leaves(entries, int(leaf_size)) - if len(root_bytes) <= target_root_len { - return root_bytes, leaves_bytes, num_leaves + rootBytes, leavesBytes, numLeaves := buildRootsLeaves(entries, int(leafSize)) + if len(rootBytes) <= targetRootLen { + return rootBytes, leavesBytes, numLeaves } - leaf_size *= 1.2 + leafSize *= 1.2 } } diff --git a/pmtiles/directory_test.go b/pmtiles/directory_test.go index e93c7f6..a5a8399 100644 --- a/pmtiles/directory_test.go +++ b/pmtiles/directory_test.go @@ -13,18 +13,18 @@ func TestDirectoryRoundtrip(t *testing.T) { entries = append(entries, EntryV3{1, 1, 1, 1}) entries = append(entries, EntryV3{2, 2, 2, 2}) - serialized := serialize_entries(entries) - result := deserialize_entries(bytes.NewBuffer(serialized)) + serialized := serializeEntries(entries) + result := deserializeEntries(bytes.NewBuffer(serialized)) assert.Equal(t, 3, len(result)) - assert.Equal(t, uint64(0), result[0].TileId) + assert.Equal(t, uint64(0), result[0].TileID) assert.Equal(t, uint64(0), result[0].Offset) assert.Equal(t, uint32(0), result[0].Length) assert.Equal(t, uint32(0), result[0].RunLength) - assert.Equal(t, uint64(1), result[1].TileId) + assert.Equal(t, uint64(1), result[1].TileID) assert.Equal(t, uint64(1), result[1].Offset) assert.Equal(t, uint32(1), result[1].Length) assert.Equal(t, uint32(1), result[1].RunLength) - assert.Equal(t, uint64(2), result[2].TileId) + assert.Equal(t, uint64(2), result[2].TileID) assert.Equal(t, uint64(2), result[2].Offset) assert.Equal(t, uint32(2), result[2].Length) assert.Equal(t, uint32(2), result[2].RunLength) @@ -56,8 +56,8 @@ func TestHeaderRoundtrip(t *testing.T) { header.CenterZoom = 3 header.CenterLonE7 = 3.1 * 10000000 header.CenterLatE7 = 3.2 * 10000000 - b := serialize_header(header) - result, _ := deserialize_header(b) + b := serializeHeader(header) + result, _ := deserializeHeader(b) assert.Equal(t, uint64(1), result.RootOffset) assert.Equal(t, uint64(2), result.RootLength) assert.Equal(t, uint64(3), result.MetadataOffset) @@ -88,9 +88,9 @@ func TestOptimizeDirectories(t *testing.T) { rand.Seed(3857) entries := make([]EntryV3, 0) entries = append(entries, EntryV3{0, 0, 100, 1}) - _, leaves_bytes, num_leaves := optimize_directories(entries, 100) - assert.False(t, len(leaves_bytes) > 0) - assert.Equal(t, 0, num_leaves) + _, leavesBytes, numLeaves := optimizeDirectories(entries, 100) + assert.False(t, len(leavesBytes) > 0) + assert.Equal(t, 0, numLeaves) entries = make([]EntryV3, 0) var i uint64 @@ -101,54 +101,54 @@ func TestOptimizeDirectories(t *testing.T) { offset += uint64(randtilesize) } - root_bytes, leaves_bytes, num_leaves := optimize_directories(entries, 1024) + rootBytes, leavesBytes, numLeaves := optimizeDirectories(entries, 1024) - assert.False(t, len(root_bytes) > 1024) + assert.False(t, len(rootBytes) > 1024) - assert.False(t, num_leaves == 0) - assert.False(t, len(leaves_bytes) == 0) + assert.False(t, numLeaves == 0) + assert.False(t, len(leavesBytes) == 0) } func TestFindTileMissing(t *testing.T) { entries := make([]EntryV3, 0) - _, ok := find_tile(entries, 0) + _, ok := findTile(entries, 0) assert.False(t, ok) } func TestFindTileFirstEntry(t *testing.T) { - entries := []EntryV3{{TileId: 100, Offset: 1, Length: 1, RunLength: 1}} - entry, ok := find_tile(entries, 100) + entries := []EntryV3{{TileID: 100, Offset: 1, Length: 1, RunLength: 1}} + entry, ok := findTile(entries, 100) assert.Equal(t, true, ok) assert.Equal(t, uint64(1), entry.Offset) assert.Equal(t, uint32(1), entry.Length) - _, ok = find_tile(entries, 101) + _, ok = findTile(entries, 101) assert.Equal(t, false, ok) } func TestFindTileMultipleEntries(t *testing.T) { entries := []EntryV3{ - {TileId: 100, Offset: 1, Length: 1, RunLength: 2}, + {TileID: 100, Offset: 1, Length: 1, RunLength: 2}, } - entry, ok := find_tile(entries, 101) + entry, ok := findTile(entries, 101) assert.Equal(t, true, ok) assert.Equal(t, uint64(1), entry.Offset) assert.Equal(t, uint32(1), entry.Length) entries = []EntryV3{ - {TileId: 100, Offset: 1, Length: 1, RunLength: 1}, - {TileId: 150, Offset: 2, Length: 2, RunLength: 2}, + {TileID: 100, Offset: 1, Length: 1, RunLength: 1}, + {TileID: 150, Offset: 2, Length: 2, RunLength: 2}, } - entry, ok = find_tile(entries, 151) + entry, ok = findTile(entries, 151) assert.Equal(t, true, ok) assert.Equal(t, uint64(2), entry.Offset) assert.Equal(t, uint32(2), entry.Length) entries = []EntryV3{ - {TileId: 50, Offset: 1, Length: 1, RunLength: 2}, - {TileId: 100, Offset: 2, Length: 2, RunLength: 1}, - {TileId: 150, Offset: 3, Length: 3, RunLength: 1}, + {TileID: 50, Offset: 1, Length: 1, RunLength: 2}, + {TileID: 100, Offset: 2, Length: 2, RunLength: 1}, + {TileID: 150, Offset: 3, Length: 3, RunLength: 1}, } - entry, ok = find_tile(entries, 51) + entry, ok = findTile(entries, 51) assert.Equal(t, true, ok) assert.Equal(t, uint64(1), entry.Offset) assert.Equal(t, uint32(1), entry.Length) @@ -156,9 +156,9 @@ func TestFindTileMultipleEntries(t *testing.T) { func TestFindTileLeafSearch(t *testing.T) { entries := []EntryV3{ - {TileId: 100, Offset: 1, Length: 1, RunLength: 0}, + {TileID: 100, Offset: 1, Length: 1, RunLength: 0}, } - entry, ok := find_tile(entries, 150) + entry, ok := findTile(entries, 150) assert.Equal(t, true, ok) assert.Equal(t, uint64(1), entry.Offset) assert.Equal(t, uint32(1), entry.Length) @@ -166,8 +166,8 @@ func TestFindTileLeafSearch(t *testing.T) { func TestBuildRootsLeaves(t *testing.T) { entries := []EntryV3{ - {TileId: 100, Offset: 1, Length: 1, RunLength: 0}, + {TileID: 100, Offset: 1, Length: 1, RunLength: 0}, } - _, _, num_leaves := build_roots_leaves(entries, 1) - assert.Equal(t, 1, num_leaves) + _, _, numLeaves := buildRootsLeaves(entries, 1) + assert.Equal(t, 1, numLeaves) } diff --git a/pmtiles/extract.go b/pmtiles/extract.go index 6dc39d4..22978f4 100644 --- a/pmtiles/extract.go +++ b/pmtiles/extract.go @@ -31,7 +31,7 @@ type SrcDstRange struct { // return sorted slice of entries, and slice of all leaf entries // any runlengths > 1 will be "trimmed" to the relevance bitmap func RelevantEntries(bitmap *roaring64.Bitmap, maxzoom uint8, dir []EntryV3) ([]EntryV3, []EntryV3) { - last_tile := ZxyToId(maxzoom+1, 0, 0) + lastTile := ZxyToID(maxzoom+1, 0, 0) leaves := make([]EntryV3, 0) tiles := make([]EntryV3, 0) for idx, entry := range dir { @@ -40,39 +40,39 @@ func RelevantEntries(bitmap *roaring64.Bitmap, maxzoom uint8, dir []EntryV3) ([] // if this is the last thing in the directory, it needs to be bounded if idx == len(dir)-1 { - tmp.AddRange(entry.TileId, last_tile) + tmp.AddRange(entry.TileID, lastTile) } else { - tmp.AddRange(entry.TileId, dir[idx+1].TileId) + tmp.AddRange(entry.TileID, dir[idx+1].TileID) } if bitmap.Intersects(tmp) { leaves = append(leaves, entry) } } else if entry.RunLength == 1 { - if bitmap.Contains(entry.TileId) { + if bitmap.Contains(entry.TileID) { tiles = append(tiles, entry) } } else { // runlength > 1 - current_id := entry.TileId - current_runlength := uint32(0) - for y := entry.TileId; y < entry.TileId+uint64(entry.RunLength); y++ { + currentID := entry.TileID + currentRunLength := uint32(0) + for y := entry.TileID; y < entry.TileID+uint64(entry.RunLength); y++ { if bitmap.Contains(y) { - if current_runlength == 0 { - current_runlength = 1 - current_id = y + if currentRunLength == 0 { + currentRunLength = 1 + currentID = y } else { - current_runlength += 1 + currentRunLength++ } } else { - if current_runlength > 0 { - tiles = append(tiles, EntryV3{current_id, entry.Offset, entry.Length, current_runlength}) + if currentRunLength > 0 { + tiles = append(tiles, EntryV3{currentID, entry.Offset, entry.Length, currentRunLength}) } - current_runlength = 0 + currentRunLength = 0 } } - if current_runlength > 0 { - tiles = append(tiles, EntryV3{current_id, entry.Offset, entry.Length, current_runlength}) + if currentRunLength > 0 { + tiles = append(tiles, EntryV3{currentID, entry.Offset, entry.Length, currentRunLength}) } } } @@ -93,34 +93,34 @@ func RelevantEntries(bitmap *roaring64.Bitmap, maxzoom uint8, dir []EntryV3) ([] // - this might not be the last SrcDstRange new_offset + length, it's the highest offset (can be in the middle) func ReencodeEntries(dir []EntryV3) ([]EntryV3, []SrcDstRange, uint64, uint64, uint64) { reencoded := make([]EntryV3, 0, len(dir)) - seen_offsets := make(map[uint64]uint64) + seenOffsets := make(map[uint64]uint64) ranges := make([]SrcDstRange, 0) - addressed_tiles := uint64(0) + addressedTiles := uint64(0) - dst_offset := uint64(0) + dstOffset := uint64(0) for _, entry := range dir { - if val, ok := seen_offsets[entry.Offset]; ok { - reencoded = append(reencoded, EntryV3{entry.TileId, val, entry.Length, entry.RunLength}) + if val, ok := seenOffsets[entry.Offset]; ok { + reencoded = append(reencoded, EntryV3{entry.TileID, val, entry.Length, entry.RunLength}) } else { if len(ranges) > 0 { - last_range := ranges[len(ranges)-1] - if last_range.SrcOffset+last_range.Length == entry.Offset { + lastRange := ranges[len(ranges)-1] + if lastRange.SrcOffset+lastRange.Length == entry.Offset { ranges[len(ranges)-1].Length += uint64(entry.Length) } else { - ranges = append(ranges, SrcDstRange{entry.Offset, dst_offset, uint64(entry.Length)}) + ranges = append(ranges, SrcDstRange{entry.Offset, dstOffset, uint64(entry.Length)}) } } else { - ranges = append(ranges, SrcDstRange{entry.Offset, dst_offset, uint64(entry.Length)}) + ranges = append(ranges, SrcDstRange{entry.Offset, dstOffset, uint64(entry.Length)}) } - reencoded = append(reencoded, EntryV3{entry.TileId, dst_offset, entry.Length, entry.RunLength}) - seen_offsets[entry.Offset] = dst_offset - dst_offset += uint64(entry.Length) + reencoded = append(reencoded, EntryV3{entry.TileID, dstOffset, entry.Length, entry.RunLength}) + seenOffsets[entry.Offset] = dstOffset + dstOffset += uint64(entry.Length) } - addressed_tiles += uint64(entry.RunLength) + addressedTiles += uint64(entry.RunLength) } - return reencoded, ranges, dst_offset, addressed_tiles, uint64(len(seen_offsets)) + return reencoded, ranges, dstOffset, addressedTiles, uint64(len(seenOffsets)) } // "want the next N bytes, then discard N bytes" @@ -158,28 +158,28 @@ type OverfetchListItem struct { // until the overfetch budget is consumed. // The slice is sorted by Length func MergeRanges(ranges []SrcDstRange, overfetch float32) (*list.List, uint64) { - total_size := 0 + totalSize := 0 shortest := make([]*OverfetchListItem, len(ranges)) // create the heap items for i, rng := range ranges { - var bytes_to_next uint64 + var bytesToNext uint64 if i == len(ranges)-1 { - bytes_to_next = math.MaxUint64 + bytesToNext = math.MaxUint64 } else { - bytes_to_next = ranges[i+1].SrcOffset - (rng.SrcOffset + rng.Length) - if bytes_to_next < 0 { - bytes_to_next = math.MaxUint64 + bytesToNext = ranges[i+1].SrcOffset - (rng.SrcOffset + rng.Length) + if bytesToNext < 0 { + bytesToNext = math.MaxUint64 } } shortest[i] = &OverfetchListItem{ Rng: rng, - BytesToNext: bytes_to_next, + BytesToNext: bytesToNext, CopyDiscards: []CopyDiscard{{uint64(rng.Length), 0}}, } - total_size += int(rng.Length) + totalSize += int(rng.Length) } // make the list doubly-linked @@ -192,7 +192,7 @@ func MergeRanges(ranges []SrcDstRange, overfetch float32) (*list.List, uint64) { } } - overfetch_budget := int(float32(total_size) * overfetch) + overfetchBudget := int(float32(totalSize) * overfetch) // sort by ascending distance to next range sort.Slice(shortest, func(i, j int) bool { @@ -200,12 +200,12 @@ func MergeRanges(ranges []SrcDstRange, overfetch float32) (*list.List, uint64) { }) // while we haven't consumed the budget, merge ranges - for (len(shortest) > 1) && (overfetch_budget-int(shortest[0].BytesToNext) >= 0) { + for (len(shortest) > 1) && (overfetchBudget-int(shortest[0].BytesToNext) >= 0) { item := shortest[0] // merge this item into item.next - new_length := item.Rng.Length + item.BytesToNext + item.next.Rng.Length - item.next.Rng = SrcDstRange{item.Rng.SrcOffset, item.Rng.DstOffset, new_length} + newLength := item.Rng.Length + item.BytesToNext + item.next.Rng.Length + item.next.Rng = SrcDstRange{item.Rng.SrcOffset, item.Rng.DstOffset, newLength} item.next.prev = item.prev if item.prev != nil { item.prev.next = item.next @@ -215,24 +215,24 @@ func MergeRanges(ranges []SrcDstRange, overfetch float32) (*list.List, uint64) { shortest = shortest[1:] - overfetch_budget -= int(item.BytesToNext) + overfetchBudget -= int(item.BytesToNext) } sort.Slice(shortest, func(i, j int) bool { return shortest[i].Rng.Length > shortest[j].Rng.Length }) - total_bytes := uint64(0) + totalBytes := uint64(0) result := list.New() for _, x := range shortest { result.PushBack(OverfetchRange{ Rng: x.Rng, CopyDiscards: x.CopyDiscards, }) - total_bytes += x.Rng.Length + totalBytes += x.Rng.Length } - return result, total_bytes + return result, totalBytes } // 1. Get the root directory (check that it is clustered) @@ -249,7 +249,7 @@ func MergeRanges(ranges []SrcDstRange, overfetch float32) (*list.List, uint64) { // 10. write the leaf directories (if any) // 11. Get all tiles, and write directly to the output. -func Extract(logger *log.Logger, bucketURL string, key string, minzoom int8, maxzoom int8, region_file string, bbox string, output string, download_threads int, overfetch float32, dry_run bool) error { +func Extract(logger *log.Logger, bucketURL string, key string, minzoom int8, maxzoom int8, regionFile string, bbox string, output string, downloadThreads int, overfetch float32, dryRun bool) error { // 1. fetch the header start := time.Now() ctx := context.Background() @@ -271,7 +271,7 @@ func Extract(logger *log.Logger, bucketURL string, key string, minzoom int8, max } defer bucket.Close() - r, err := bucket.NewRangeReader(ctx, key, 0, HEADERV3_LEN_BYTES) + r, err := bucket.NewRangeReader(ctx, key, 0, HeaderV3LenBytes) if err != nil { return fmt.Errorf("Failed to create range reader for %s, %w", key, err) @@ -282,14 +282,14 @@ func Extract(logger *log.Logger, bucketURL string, key string, minzoom int8, max } r.Close() - header, err := deserialize_header(b[0:HEADERV3_LEN_BYTES]) + header, err := deserializeHeader(b[0:HeaderV3LenBytes]) if !header.Clustered { - return fmt.Errorf("Error: source archive must be clustered for extracts.") + return fmt.Errorf("source archive must be clustered for extracts") } - source_metadata_offset := header.MetadataOffset - source_tile_data_offset := header.TileDataOffset + sourceMetadataOffset := header.MetadataOffset + sourceTileDataOffset := header.TileDataOffset if minzoom == -1 || int8(header.MinZoom) > minzoom { minzoom = int8(header.MinZoom) @@ -300,19 +300,19 @@ func Extract(logger *log.Logger, bucketURL string, key string, minzoom int8, max } if minzoom > maxzoom { - return fmt.Errorf("Error: minzoom cannot be greater than maxzoom.") + return fmt.Errorf("minzoom cannot be greater than maxzoom") } - var relevant_set *roaring64.Bitmap - if region_file != "" || bbox != "" { - if region_file != "" && bbox != "" { - return fmt.Errorf("Only one of region and bbox can be specified.") + var relevantSet *roaring64.Bitmap + if regionFile != "" || bbox != "" { + if regionFile != "" && bbox != "" { + return fmt.Errorf("only one of region and bbox can be specified") } var multipolygon orb.MultiPolygon - if region_file != "" { - dat, _ := ioutil.ReadFile(region_file) + if regionFile != "" { + dat, _ := ioutil.ReadFile(regionFile) multipolygon, err = UnmarshalRegion(dat) if err != nil { @@ -329,10 +329,10 @@ func Extract(logger *log.Logger, bucketURL string, key string, minzoom int8, max bound := multipolygon.Bound() - boundary_set, interior_set := bitmapMultiPolygon(uint8(maxzoom), multipolygon) - relevant_set = boundary_set - relevant_set.Or(interior_set) - generalizeOr(relevant_set, uint8(minzoom)) + boundarySet, interiorSet := bitmapMultiPolygon(uint8(maxzoom), multipolygon) + relevantSet = boundarySet + relevantSet.Or(interiorSet) + generalizeOr(relevantSet, uint8(minzoom)) header.MinLonE7 = int32(bound.Left() * 10000000) header.MinLatE7 = int32(bound.Bottom() * 10000000) @@ -341,197 +341,197 @@ func Extract(logger *log.Logger, bucketURL string, key string, minzoom int8, max header.CenterLonE7 = int32(bound.Center().X() * 10000000) header.CenterLatE7 = int32(bound.Center().Y() * 10000000) } else { - relevant_set = roaring64.New() - relevant_set.AddRange(ZxyToId(uint8(minzoom), 0, 0), ZxyToId(uint8(maxzoom)+1, 0, 0)) + relevantSet = roaring64.New() + relevantSet.AddRange(ZxyToID(uint8(minzoom), 0, 0), ZxyToID(uint8(maxzoom)+1, 0, 0)) } // 3. get relevant entries from root - dir_offset := header.RootOffset - dir_length := header.RootLength + dirOffset := header.RootOffset + dirLength := header.RootLength - root_reader, err := bucket.NewRangeReader(ctx, key, int64(dir_offset), int64(dir_length)) + rootReader, err := bucket.NewRangeReader(ctx, key, int64(dirOffset), int64(dirLength)) if err != nil { return err } - defer root_reader.Close() - root_bytes, err := io.ReadAll(root_reader) + defer rootReader.Close() + rootBytes, err := io.ReadAll(rootReader) if err != nil { return err } - root_dir := deserialize_entries(bytes.NewBuffer(root_bytes)) + rootDir := deserializeEntries(bytes.NewBuffer(rootBytes)) - tile_entries, leaves := RelevantEntries(relevant_set, uint8(maxzoom), root_dir) + tileEntries, leaves := RelevantEntries(relevantSet, uint8(maxzoom), rootDir) // 4. get all relevant leaf entries - leaf_ranges := make([]SrcDstRange, 0) + leafRanges := make([]SrcDstRange, 0) for _, leaf := range leaves { - leaf_ranges = append(leaf_ranges, SrcDstRange{header.LeafDirectoryOffset + leaf.Offset, 0, uint64(leaf.Length)}) + leafRanges = append(leafRanges, SrcDstRange{header.LeafDirectoryOffset + leaf.Offset, 0, uint64(leaf.Length)}) } - overfetch_leaves, _ := MergeRanges(leaf_ranges, overfetch) - num_overfetch_leaves := overfetch_leaves.Len() - fmt.Printf("fetching %d dirs, %d chunks, %d requests\n", len(leaves), len(leaf_ranges), overfetch_leaves.Len()) + overfetchLeaves, _ := MergeRanges(leafRanges, overfetch) + numOverfetchLeaves := overfetchLeaves.Len() + fmt.Printf("fetching %d dirs, %d chunks, %d requests\n", len(leaves), len(leafRanges), overfetchLeaves.Len()) for { - if overfetch_leaves.Len() == 0 { + if overfetchLeaves.Len() == 0 { break } - or := overfetch_leaves.Remove(overfetch_leaves.Front()).(OverfetchRange) + or := overfetchLeaves.Remove(overfetchLeaves.Front()).(OverfetchRange) - slab_r, err := bucket.NewRangeReader(ctx, key, int64(or.Rng.SrcOffset), int64(or.Rng.Length)) + chunkReader, err := bucket.NewRangeReader(ctx, key, int64(or.Rng.SrcOffset), int64(or.Rng.Length)) if err != nil { return err } for _, cd := range or.CopyDiscards { - leaf_bytes := make([]byte, cd.Wanted) - _, err := io.ReadFull(slab_r, leaf_bytes) + leafBytes := make([]byte, cd.Wanted) + _, err := io.ReadFull(chunkReader, leafBytes) if err != nil { return err } - leafdir := deserialize_entries(bytes.NewBuffer(leaf_bytes)) - new_entries, new_leaves := RelevantEntries(relevant_set, uint8(maxzoom), leafdir) + leafdir := deserializeEntries(bytes.NewBuffer(leafBytes)) + newEntries, newLeaves := RelevantEntries(relevantSet, uint8(maxzoom), leafdir) - if len(new_leaves) > 0 { + if len(newLeaves) > 0 { panic("This doesn't support leaf level 2+.") } - tile_entries = append(tile_entries, new_entries...) + tileEntries = append(tileEntries, newEntries...) - _, err = io.CopyN(io.Discard, slab_r, int64(cd.Discard)) + _, err = io.CopyN(io.Discard, chunkReader, int64(cd.Discard)) if err != nil { return err } } - slab_r.Close() + chunkReader.Close() } - sort.Slice(tile_entries, func(i, j int) bool { - return tile_entries[i].TileId < tile_entries[j].TileId + sort.Slice(tileEntries, func(i, j int) bool { + return tileEntries[i].TileID < tileEntries[j].TileID }) - fmt.Printf("Region tiles %d, result tile entries %d\n", relevant_set.GetCardinality(), len(tile_entries)) + fmt.Printf("Region tiles %d, result tile entries %d\n", relevantSet.GetCardinality(), len(tileEntries)) // 6. create the new header and chunk list // we now need to re-encode this entry list using cumulative offsets - reencoded, tile_parts, tiledata_length, addressed_tiles, tile_contents := ReencodeEntries(tile_entries) + reencoded, tileParts, tiledataLength, addressedTiles, tileContents := ReencodeEntries(tileEntries) - overfetch_ranges, total_bytes := MergeRanges(tile_parts, overfetch) + overfetchRanges, totalBytes := MergeRanges(tileParts, overfetch) - num_overfetch_ranges := overfetch_ranges.Len() - fmt.Printf("fetching %d tiles, %d chunks, %d requests\n", len(reencoded), len(tile_parts), overfetch_ranges.Len()) + numOverfetchRanges := overfetchRanges.Len() + fmt.Printf("fetching %d tiles, %d chunks, %d requests\n", len(reencoded), len(tileParts), overfetchRanges.Len()) // TODO: takes up too much RAM // construct the directories - new_root_bytes, new_leaves_bytes, _ := optimize_directories(reencoded, 16384-HEADERV3_LEN_BYTES) + newRootBytes, newLeavesBytes, _ := optimizeDirectories(reencoded, 16384-HeaderV3LenBytes) // 7. write the modified header - header.RootOffset = HEADERV3_LEN_BYTES - header.RootLength = uint64(len(new_root_bytes)) + header.RootOffset = HeaderV3LenBytes + header.RootLength = uint64(len(newRootBytes)) header.MetadataOffset = header.RootOffset + header.RootLength header.LeafDirectoryOffset = header.MetadataOffset + header.MetadataLength - header.LeafDirectoryLength = uint64(len(new_leaves_bytes)) + header.LeafDirectoryLength = uint64(len(newLeavesBytes)) header.TileDataOffset = header.LeafDirectoryOffset + header.LeafDirectoryLength - header.TileDataLength = tiledata_length - header.AddressedTilesCount = addressed_tiles - header.TileEntriesCount = uint64(len(tile_entries)) - header.TileContentsCount = tile_contents + header.TileDataLength = tiledataLength + header.AddressedTilesCount = addressedTiles + header.TileEntriesCount = uint64(len(tileEntries)) + header.TileContentsCount = tileContents header.MaxZoom = uint8(maxzoom) - header_bytes := serialize_header(header) + headerBytes := serializeHeader(header) - total_actual_bytes := uint64(0) - for _, x := range tile_parts { - total_actual_bytes += x.Length + totalActualBytes := uint64(0) + for _, x := range tileParts { + totalActualBytes += x.Length } - if !dry_run { + if !dryRun { outfile, err := os.Create(output) defer outfile.Close() - outfile.Truncate(127 + int64(len(new_root_bytes)) + int64(header.MetadataLength) + int64(len(new_leaves_bytes)) + int64(total_actual_bytes)) + outfile.Truncate(127 + int64(len(newRootBytes)) + int64(header.MetadataLength) + int64(len(newLeavesBytes)) + int64(totalActualBytes)) - _, err = outfile.Write(header_bytes) + _, err = outfile.Write(headerBytes) if err != nil { return err } // 8. write the root directory - _, err = outfile.Write(new_root_bytes) + _, err = outfile.Write(newRootBytes) if err != nil { return err } // 9. get and write the metadata - metadata_reader, err := bucket.NewRangeReader(ctx, key, int64(source_metadata_offset), int64(header.MetadataLength)) + metadataReader, err := bucket.NewRangeReader(ctx, key, int64(sourceMetadataOffset), int64(header.MetadataLength)) if err != nil { return err } - metadata_bytes, err := io.ReadAll(metadata_reader) - defer metadata_reader.Close() + metadataBytes, err := io.ReadAll(metadataReader) + defer metadataReader.Close() if err != nil { return err } - outfile.Write(metadata_bytes) + outfile.Write(metadataBytes) // 10. write the leaf directories - _, err = outfile.Write(new_leaves_bytes) + _, err = outfile.Write(newLeavesBytes) if err != nil { return err } bar := progressbar.DefaultBytes( - int64(total_bytes), + int64(totalBytes), "fetching chunks", ) var mu sync.Mutex downloadPart := func(or OverfetchRange) error { - tile_r, err := bucket.NewRangeReader(ctx, key, int64(source_tile_data_offset+or.Rng.SrcOffset), int64(or.Rng.Length)) + tileReader, err := bucket.NewRangeReader(ctx, key, int64(sourceTileDataOffset+or.Rng.SrcOffset), int64(or.Rng.Length)) if err != nil { return err } - offset_writer := io.NewOffsetWriter(outfile, int64(header.TileDataOffset)+int64(or.Rng.DstOffset)) + offsetWriter := io.NewOffsetWriter(outfile, int64(header.TileDataOffset)+int64(or.Rng.DstOffset)) for _, cd := range or.CopyDiscards { - _, err := io.CopyN(io.MultiWriter(offset_writer, bar), tile_r, int64(cd.Wanted)) + _, err := io.CopyN(io.MultiWriter(offsetWriter, bar), tileReader, int64(cd.Wanted)) if err != nil { return err } - _, err = io.CopyN(bar, tile_r, int64(cd.Discard)) + _, err = io.CopyN(bar, tileReader, int64(cd.Discard)) if err != nil { return err } } - tile_r.Close() + tileReader.Close() return nil } errs, _ := errgroup.WithContext(ctx) - for i := 0; i < download_threads; i++ { - work_back := (i == 0 && download_threads > 1) + for i := 0; i < downloadThreads; i++ { + workBack := (i == 0 && downloadThreads > 1) errs.Go(func() error { done := false var or OverfetchRange for { mu.Lock() - if overfetch_ranges.Len() == 0 { + if overfetchRanges.Len() == 0 { done = true } else { - if work_back { - or = overfetch_ranges.Remove(overfetch_ranges.Back()).(OverfetchRange) + if workBack { + or = overfetchRanges.Remove(overfetchRanges.Back()).(OverfetchRange) } else { - or = overfetch_ranges.Remove(overfetch_ranges.Front()).(OverfetchRange) + or = overfetchRanges.Remove(overfetchRanges.Front()).(OverfetchRange) } } mu.Unlock() @@ -554,13 +554,13 @@ func Extract(logger *log.Logger, bucketURL string, key string, minzoom int8, max } } - fmt.Printf("Completed in %v with %v download threads (%v tiles/s).\n", time.Since(start), download_threads, float64(len(reencoded))/float64(time.Since(start).Seconds())) - total_requests := 2 // header + root - total_requests += num_overfetch_leaves // leaves - total_requests += 1 // metadata - total_requests += num_overfetch_ranges - fmt.Printf("Extract required %d total requests.\n", total_requests) - fmt.Printf("Extract transferred %s (overfetch %v) for an archive size of %s\n", humanize.Bytes(total_bytes), overfetch, humanize.Bytes(total_actual_bytes)) + fmt.Printf("Completed in %v with %v download threads (%v tiles/s).\n", time.Since(start), downloadThreads, float64(len(reencoded))/float64(time.Since(start).Seconds())) + totalRequests := 2 // header + root + totalRequests += numOverfetchLeaves // leaves + totalRequests++ // metadata + totalRequests += numOverfetchRanges + fmt.Printf("Extract required %d total requests.\n", totalRequests) + fmt.Printf("Extract transferred %s (overfetch %v) for an archive size of %s\n", humanize.Bytes(totalBytes), overfetch, humanize.Bytes(totalActualBytes)) return nil } diff --git a/pmtiles/extract_test.go b/pmtiles/extract_test.go index 403ccff..9eacb36 100644 --- a/pmtiles/extract_test.go +++ b/pmtiles/extract_test.go @@ -141,10 +141,10 @@ func TestMergeRanges(t *testing.T) { ranges = append(ranges, SrcDstRange{0, 0, 50}) ranges = append(ranges, SrcDstRange{60, 60, 60}) - result, total_transfer_bytes := MergeRanges(ranges, 0.1) + result, totalTransferBytes := MergeRanges(ranges, 0.1) assert.Equal(t, 1, result.Len()) - assert.Equal(t, uint64(120), total_transfer_bytes) + assert.Equal(t, uint64(120), totalTransferBytes) front := result.Front().Value.(OverfetchRange) assert.Equal(t, SrcDstRange{0, 0, 120}, front.Rng) assert.Equal(t, 2, len(front.CopyDiscards)) @@ -158,9 +158,9 @@ func TestMergeRangesMultiple(t *testing.T) { ranges = append(ranges, SrcDstRange{60, 60, 10}) ranges = append(ranges, SrcDstRange{80, 80, 10}) - result, total_transfer_bytes := MergeRanges(ranges, 0.3) + result, totalTransferBytes := MergeRanges(ranges, 0.3) front := result.Front().Value.(OverfetchRange) - assert.Equal(t, uint64(90), total_transfer_bytes) + assert.Equal(t, uint64(90), totalTransferBytes) assert.Equal(t, 1, result.Len()) assert.Equal(t, SrcDstRange{0, 0, 90}, front.Rng) assert.Equal(t, 3, len(front.CopyDiscards)) diff --git a/pmtiles/makesync.go b/pmtiles/makesync.go index a104bdd..f9e6a44 100644 --- a/pmtiles/makesync.go +++ b/pmtiles/makesync.go @@ -39,12 +39,12 @@ type Syncline struct { Hash uint64 } -func Makesync(logger *log.Logger, cli_version string, file string, block_size_kb int, checksum string) error { +func Makesync(logger *log.Logger, cliVersion string, file string, blockSizeKb int, checksum string) error { ctx := context.Background() start := time.Now() bucketURL, key, err := NormalizeBucketKey("", "", file) - block_size_bytes := uint64(1000 * block_size_kb) + blockSizeBytes := uint64(1000 * blockSizeKb) if err != nil { return err @@ -68,10 +68,10 @@ func Makesync(logger *log.Logger, cli_version string, file string, block_size_kb } r.Close() - header, err := deserialize_header(b[0:HEADERV3_LEN_BYTES]) + header, err := deserializeHeader(b[0:HeaderV3LenBytes]) if !header.Clustered { - return fmt.Errorf("Error: archive must be clustered for makesync.") + return fmt.Errorf("archive must be clustered for makesync") } var CollectEntries func(uint64, uint64, func(EntryV3)) @@ -87,7 +87,7 @@ func Makesync(logger *log.Logger, cli_version string, file string, block_size_kb panic(fmt.Errorf("I/O Error")) } - directory := deserialize_entries(bytes.NewBuffer(b)) + directory := deserializeEntries(bytes.NewBuffer(b)) for _, entry := range directory { if entry.RunLength > 0 { f(entry) @@ -102,7 +102,7 @@ func Makesync(logger *log.Logger, cli_version string, file string, block_size_kb panic(err) } defer output.Close() - output.Write([]byte(fmt.Sprintf("version=%s\n", cli_version))) + output.Write([]byte(fmt.Sprintf("version=%s\n", cliVersion))) if checksum == "md5" { localfile, err := os.Open(file) @@ -121,7 +121,7 @@ func Makesync(logger *log.Logger, cli_version string, file string, block_size_kb } output.Write([]byte("hash=fnv1a\n")) - output.Write([]byte(fmt.Sprintf("blocksize=%d\n", block_size_bytes))) + output.Write([]byte(fmt.Sprintf("blocksize=%d\n", blockSizeBytes))) bar := progressbar.Default( int64(header.TileEntriesCount), @@ -166,14 +166,14 @@ func Makesync(logger *log.Logger, cli_version string, file string, block_size_kb }) } - current_index := uint64(0) + currentIndex := uint64(0) blocks := 0 CollectEntries(header.RootOffset, header.RootLength, func(e EntryV3) { bar.Add(1) if current.Length == 0 { - current.Index = current_index - current.Start = e.TileId + current.Index = currentIndex + current.Start = e.TileID current.Offset = e.Offset current.Length = uint64(e.Length) } else if e.Offset < current.Offset+uint64(current.Length) { // todo: check max block length @@ -181,13 +181,13 @@ func Makesync(logger *log.Logger, cli_version string, file string, block_size_kb } else if e.Offset > current.Offset+uint64(current.Length) { panic("Invalid clustering of archive detected - check with verify") } else { - if current.Length+uint64(e.Length) > block_size_bytes { + if current.Length+uint64(e.Length) > blockSizeBytes { tasks <- Block{current.Index, current.Start, current.Offset, current.Length} - blocks += 1 + blocks++ - current_index += 1 - current.Index = current_index - current.Start = e.TileId + currentIndex++ + current.Index = currentIndex + current.Start = e.TileID current.Offset = e.Offset current.Length = uint64(e.Length) } else { @@ -197,7 +197,7 @@ func Makesync(logger *log.Logger, cli_version string, file string, block_size_kb }) tasks <- Block{current.Index, current.Start, current.Offset, current.Length} - blocks += 1 + blocks++ close(tasks) wg.Wait() @@ -220,13 +220,13 @@ func Makesync(logger *log.Logger, cli_version string, file string, block_size_kb func Sync(logger *log.Logger, file string, syncfile string) error { start := time.Now() - total_remote_bytes := uint64(0) + totalRemoteBytes := uint64(0) - by_start_id := make(map[uint64]Syncline) + byStartID := make(map[uint64]Syncline) sync, err := os.Open(syncfile) if err != nil { - return fmt.Errorf("Error opening syncfile: %v\n", err) + return fmt.Errorf("error opening syncfile: %v", err) } defer sync.Close() scanner := bufio.NewScanner(sync) @@ -237,12 +237,12 @@ func Sync(logger *log.Logger, file string, syncfile string) error { continue } - start_id, _ := strconv.ParseUint(parts[0], 10, 64) + startID, _ := strconv.ParseUint(parts[0], 10, 64) offset, _ := strconv.ParseUint(parts[1], 10, 64) length, _ := strconv.ParseUint(parts[2], 10, 64) - total_remote_bytes += length + totalRemoteBytes += length hash, _ := strconv.ParseUint(parts[3], 16, 64) - by_start_id[start_id] = Syncline{offset, length, hash} + byStartID[startID] = Syncline{offset, length, hash} } // open the existing archive @@ -273,10 +273,10 @@ func Sync(logger *log.Logger, file string, syncfile string) error { } r.Close() - header, err := deserialize_header(b[0:HEADERV3_LEN_BYTES]) + header, err := deserializeHeader(b[0:HeaderV3LenBytes]) if !header.Clustered { - return fmt.Errorf("Error: archive must be clustered for makesync.") + return fmt.Errorf("archive must be clustered for makesync") } GetHash := func(offset uint64, length uint64) uint64 { @@ -306,7 +306,7 @@ func Sync(logger *log.Logger, file string, syncfile string) error { panic(fmt.Errorf("I/O Error")) } - directory := deserialize_entries(bytes.NewBuffer(b)) + directory := deserializeEntries(bytes.NewBuffer(b)) for _, entry := range directory { if entry.RunLength > 0 { f(entry) @@ -321,31 +321,31 @@ func Sync(logger *log.Logger, file string, syncfile string) error { "calculating diff", ) - total_blocks := len(by_start_id) + totalBlocks := len(byStartID) hits := 0 CollectEntries(header.RootOffset, header.RootLength, func(e EntryV3) { bar.Add(1) - potential_match, ok := by_start_id[e.TileId] + potentialMatch, ok := byStartID[e.TileID] if ok { - hash_result := GetHash(e.Offset, potential_match.Length) - if hash_result == potential_match.Hash { - hits += 1 - delete(by_start_id, e.TileId) + hashResult := GetHash(e.Offset, potentialMatch.Length) + if hashResult == potentialMatch.Hash { + hits++ + delete(byStartID, e.TileID) } } }) - to_transfer := uint64(0) - for _, v := range by_start_id { - to_transfer += v.Length + toTransfer := uint64(0) + for _, v := range byStartID { + toTransfer += v.Length } - blocks_matched := float64(hits) / float64(total_blocks) * 100 - pct := float64(to_transfer) / float64(total_remote_bytes) * 100 + blocksMatched := float64(hits) / float64(totalBlocks) * 100 + pct := float64(toTransfer) / float64(totalRemoteBytes) * 100 - fmt.Printf("%d/%d blocks matched (%.1f%%), need to transfer %s/%s (%.1f%%).\n", hits, total_blocks, blocks_matched, humanize.Bytes(to_transfer), humanize.Bytes(total_remote_bytes), pct) + fmt.Printf("%d/%d blocks matched (%.1f%%), need to transfer %s/%s (%.1f%%).\n", hits, totalBlocks, blocksMatched, humanize.Bytes(toTransfer), humanize.Bytes(totalRemoteBytes), pct) fmt.Printf("Completed sync in %v.\n", time.Since(start)) return nil diff --git a/pmtiles/readerv2.go b/pmtiles/readerv2.go index 62c7dcc..c74aff9 100644 --- a/pmtiles/readerv2.go +++ b/pmtiles/readerv2.go @@ -36,61 +36,61 @@ func readUint48(b []byte) uint64 { } func GetParentTile(tile Zxy, level uint8) Zxy { - tile_diff := tile.Z - level - x := math.Floor(float64(tile.X / (1 << tile_diff))) - y := math.Floor(float64(tile.Y / (1 << tile_diff))) + tileDiff := tile.Z - level + x := math.Floor(float64(tile.X / (1 << tileDiff))) + y := math.Floor(float64(tile.Y / (1 << tileDiff))) return Zxy{Z: level, X: uint32(x), Y: uint32(y)} } func ParseEntryV2(b []byte) (uint8, Zxy, Range) { - z_raw := b[0] - x_raw := b[1:4] - y_raw := b[4:7] - offset_raw := b[7:13] - length_raw := b[13:17] - x := readUint24(x_raw) - y := readUint24(y_raw) - offset := readUint48(offset_raw) - length := uint64(binary.LittleEndian.Uint32(length_raw)) - if z_raw&0b10000000 == 0 { - return 0, Zxy{Z: uint8(z_raw), X: uint32(x), Y: uint32(y)}, Range{Offset: offset, Length: length} + zRaw := b[0] + xRaw := b[1:4] + yRaw := b[4:7] + offsetRaw := b[7:13] + lengthRaw := b[13:17] + x := readUint24(xRaw) + y := readUint24(yRaw) + offset := readUint48(offsetRaw) + length := uint64(binary.LittleEndian.Uint32(lengthRaw)) + if zRaw&0b10000000 == 0 { + return 0, Zxy{Z: uint8(zRaw), X: uint32(x), Y: uint32(y)}, Range{Offset: offset, Length: length} } else { - leaf_z := z_raw & 0b01111111 - return leaf_z, Zxy{Z: leaf_z, X: uint32(x), Y: uint32(y)}, Range{Offset: offset, Length: length} + leafZ := zRaw & 0b01111111 + return leafZ, Zxy{Z: leafZ, X: uint32(x), Y: uint32(y)}, Range{Offset: offset, Length: length} } } -func ParseDirectoryV2(dir_bytes []byte) DirectoryV2 { - the_dir := DirectoryV2{Entries: make(map[Zxy]Range), Leaves: make(map[Zxy]Range)} +func ParseDirectoryV2(dirBytes []byte) DirectoryV2 { + theDir := DirectoryV2{Entries: make(map[Zxy]Range), Leaves: make(map[Zxy]Range)} var maxz uint8 - for i := 0; i < len(dir_bytes)/17; i++ { - leaf_z, zxy, rng := ParseEntryV2(dir_bytes[i*17 : i*17+17]) - if leaf_z == 0 { - the_dir.Entries[zxy] = rng + for i := 0; i < len(dirBytes)/17; i++ { + leafZ, zxy, rng := ParseEntryV2(dirBytes[i*17 : i*17+17]) + if leafZ == 0 { + theDir.Entries[zxy] = rng } else { - maxz = leaf_z // todo check spec - the_dir.Leaves[zxy] = rng + maxz = leafZ // todo check spec + theDir.Leaves[zxy] = rng } } - the_dir.LeafZ = maxz - return the_dir + theDir.LeafZ = maxz + return theDir } func ParseHeaderV2(reader io.Reader) ([]byte, DirectoryV2) { - magic_num := make([]byte, 2) - io.ReadFull(reader, magic_num) + magicNum := make([]byte, 2) + io.ReadFull(reader, magicNum) version := make([]byte, 2) io.ReadFull(reader, version) - metadata_len_bytes := make([]byte, 4) - io.ReadFull(reader, metadata_len_bytes) - metadata_len := binary.LittleEndian.Uint32(metadata_len_bytes) - rootdir_len_bytes := make([]byte, 2) - io.ReadFull(reader, rootdir_len_bytes) - rootdir_len := int(binary.LittleEndian.Uint16(rootdir_len_bytes)) - metadata_bytes := make([]byte, metadata_len) - io.ReadFull(reader, metadata_bytes) - dir_bytes := make([]byte, rootdir_len*17) - io.ReadFull(reader, dir_bytes) - the_dir := ParseDirectoryV2(dir_bytes) - return metadata_bytes, the_dir + metadataLenBytes := make([]byte, 4) + io.ReadFull(reader, metadataLenBytes) + metadataLen := binary.LittleEndian.Uint32(metadataLenBytes) + rootDirLenBytes := make([]byte, 2) + io.ReadFull(reader, rootDirLenBytes) + rootDirLen := int(binary.LittleEndian.Uint16(rootDirLenBytes)) + metadataBytes := make([]byte, metadataLen) + io.ReadFull(reader, metadataBytes) + dirBytes := make([]byte, rootDirLen*17) + io.ReadFull(reader, dirBytes) + theDir := ParseDirectoryV2(dirBytes) + return metadataBytes, theDir } diff --git a/pmtiles/region.go b/pmtiles/region.go index 2ecc8bd..17bf7fc 100644 --- a/pmtiles/region.go +++ b/pmtiles/region.go @@ -10,23 +10,23 @@ import ( func BboxRegion(bbox string) (orb.MultiPolygon, error) { parts := strings.Split(bbox, ",") - min_lon, err := strconv.ParseFloat(parts[0], 64) + minLon, err := strconv.ParseFloat(parts[0], 64) if err != nil { return nil, err } - min_lat, err := strconv.ParseFloat(parts[1], 64) + minLat, err := strconv.ParseFloat(parts[1], 64) if err != nil { return nil, err } - max_lon, err := strconv.ParseFloat(parts[2], 64) + maxLon, err := strconv.ParseFloat(parts[2], 64) if err != nil { return nil, err } - max_lat, err := strconv.ParseFloat(parts[3], 64) + maxLat, err := strconv.ParseFloat(parts[3], 64) if err != nil { return nil, err } - return orb.MultiPolygon{{{{min_lon, max_lat}, {max_lon, max_lat}, {max_lon, min_lat}, {min_lon, min_lat}, {min_lon, max_lat}}}}, nil + return orb.MultiPolygon{{{{minLon, maxLat}, {maxLon, maxLat}, {maxLon, minLat}, {minLon, minLat}, {minLon, maxLat}}}}, nil } func UnmarshalRegion(data []byte) (orb.MultiPolygon, error) { diff --git a/pmtiles/server.go b/pmtiles/server.go index 06857b9..e5b7021 100644 --- a/pmtiles/server.go +++ b/pmtiles/server.go @@ -44,10 +44,10 @@ type Server struct { logger *log.Logger cacheSize int cors string - publicUrl string + publicURL string } -func NewServer(bucketURL string, prefix string, logger *log.Logger, cacheSize int, cors string, publicUrl string) (*Server, error) { +func NewServer(bucketURL string, prefix string, logger *log.Logger, cacheSize int, cors string, publicURL string) (*Server, error) { ctx := context.Background() @@ -63,10 +63,10 @@ func NewServer(bucketURL string, prefix string, logger *log.Logger, cacheSize in return nil, err } - return NewServerWithBucket(bucket, prefix, logger, cacheSize, cors, publicUrl) + return NewServerWithBucket(bucket, prefix, logger, cacheSize, cors, publicURL) } -func NewServerWithBucket(bucket Bucket, prefix string, logger *log.Logger, cacheSize int, cors string, publicUrl string) (*Server, error) { +func NewServerWithBucket(bucket Bucket, prefix string, logger *log.Logger, cacheSize int, cors string, publicURL string) (*Server, error) { reqs := make(chan Request, 8) @@ -76,7 +76,7 @@ func NewServerWithBucket(bucket Bucket, prefix string, logger *log.Logger, cache logger: logger, cacheSize: cacheSize, cors: cors, - publicUrl: publicUrl, + publicURL: publicURL, } return l, nil @@ -104,12 +104,12 @@ func (server *Server) Start() { inflight[key] = []Request{req} go func() { var result CachedValue - is_root := (key.offset == 0 && key.length == 0) + isRoot := (key.offset == 0 && key.length == 0) offset := int64(key.offset) length := int64(key.length) - if is_root { + if isRoot { offset = 0 length = 16384 } @@ -133,24 +133,24 @@ func (server *Server) Start() { return } - if is_root { - header, err := deserialize_header(b[0:HEADERV3_LEN_BYTES]) + if isRoot { + header, err := deserializeHeader(b[0:HeaderV3LenBytes]) if err != nil { server.logger.Printf("parsing header failed: %v", err) return } // populate the root first before header - root_entries := deserialize_entries(bytes.NewBuffer(b[header.RootOffset : header.RootOffset+header.RootLength])) - result2 := CachedValue{directory: root_entries, ok: true} + rootEntries := deserializeEntries(bytes.NewBuffer(b[header.RootOffset : header.RootOffset+header.RootLength])) + result2 := CachedValue{directory: rootEntries, ok: true} - root_key := CacheKey{name: key.name, offset: header.RootOffset, length: header.RootLength} - resps <- Response{key: root_key, value: result2, size: 24 * len(root_entries), ok: true} + rootKey := CacheKey{name: key.name, offset: header.RootOffset, length: header.RootLength} + resps <- Response{key: rootKey, value: result2, size: 24 * len(rootEntries), ok: true} result = CachedValue{header: header, ok: true} resps <- Response{key: key, value: result, size: 127, ok: true} } else { - directory := deserialize_entries(bytes.NewBuffer(b)) + directory := deserializeEntries(bytes.NewBuffer(b)) result = CachedValue{directory: directory, ok: true} resps <- Response{key: key, value: result, size: 24 * len(directory), ok: true} } @@ -190,13 +190,13 @@ func (server *Server) Start() { }() } -func (server *Server) get_header_metadata(ctx context.Context, name string) (error, bool, HeaderV3, []byte) { - root_req := Request{key: CacheKey{name: name, offset: 0, length: 0}, value: make(chan CachedValue, 1)} - server.reqs <- root_req - root_value := <-root_req.value - header := root_value.header +func (server *Server) getHeaderMetadata(ctx context.Context, name string) (error, bool, HeaderV3, []byte) { + rootReq := Request{key: CacheKey{name: name, offset: 0, length: 0}, value: make(chan CachedValue, 1)} + server.reqs <- rootReq + rootValue := <-rootReq.value + header := rootValue.header - if !root_value.ok { + if !rootValue.ok { return nil, false, HeaderV3{}, nil } @@ -206,111 +206,111 @@ func (server *Server) get_header_metadata(ctx context.Context, name string) (err } defer r.Close() - var metadata_bytes []byte + var metadataBytes []byte if header.InternalCompression == Gzip { - metadata_reader, _ := gzip.NewReader(r) - defer metadata_reader.Close() - metadata_bytes, err = io.ReadAll(metadata_reader) + metadataReader, _ := gzip.NewReader(r) + defer metadataReader.Close() + metadataBytes, err = io.ReadAll(metadataReader) } else if header.InternalCompression == NoCompression { - metadata_bytes, err = io.ReadAll(r) + metadataBytes, err = io.ReadAll(r) } else { return errors.New("Unknown compression"), true, HeaderV3{}, nil } - return nil, true, header, metadata_bytes + return nil, true, header, metadataBytes } -func (server *Server) get_tilejson(ctx context.Context, http_headers map[string]string, name string) (int, map[string]string, []byte) { - err, found, header, metadata_bytes := server.get_header_metadata(ctx, name) +func (server *Server) getTileJSON(ctx context.Context, httpHeaders map[string]string, name string) (int, map[string]string, []byte) { + err, found, header, metadataBytes := server.getHeaderMetadata(ctx, name) if err != nil { - return 500, http_headers, []byte("I/O Error") + return 500, httpHeaders, []byte("I/O Error") } if !found { - return 404, http_headers, []byte("Archive not found") + return 404, httpHeaders, []byte("Archive not found") } - var metadata_map map[string]interface{} - json.Unmarshal(metadata_bytes, &metadata_map) + var metadataMap map[string]interface{} + json.Unmarshal(metadataBytes, &metadataMap) - if server.publicUrl == "" { - return 501, http_headers, []byte("PUBLIC_URL must be set for TileJSON") + if server.publicURL == "" { + return 501, httpHeaders, []byte("PUBLIC_URL must be set for TileJSON") } - tilejson_bytes, err := CreateTilejson(header, metadata_bytes, server.publicUrl+"/"+name) + tilejsonBytes, err := CreateTilejson(header, metadataBytes, server.publicURL+"/"+name) if err != nil { - return 500, http_headers, []byte("Error generating tilejson") + return 500, httpHeaders, []byte("Error generating tilejson") } - http_headers["Content-Type"] = "application/json" + httpHeaders["Content-Type"] = "application/json" - return 200, http_headers, tilejson_bytes + return 200, httpHeaders, tilejsonBytes } -func (server *Server) get_metadata(ctx context.Context, http_headers map[string]string, name string) (int, map[string]string, []byte) { - err, found, _, metadata_bytes := server.get_header_metadata(ctx, name) +func (server *Server) getMetadata(ctx context.Context, httpHeaders map[string]string, name string) (int, map[string]string, []byte) { + err, found, _, metadataBytes := server.getHeaderMetadata(ctx, name) if err != nil { - return 500, http_headers, []byte("I/O Error") + return 500, httpHeaders, []byte("I/O Error") } if !found { - return 404, http_headers, []byte("Archive not found") + return 404, httpHeaders, []byte("Archive not found") } - http_headers["Content-Type"] = "application/json" - return 200, http_headers, metadata_bytes + httpHeaders["Content-Type"] = "application/json" + return 200, httpHeaders, metadataBytes } -func (server *Server) get_tile(ctx context.Context, http_headers map[string]string, name string, z uint8, x uint32, y uint32, ext string) (int, map[string]string, []byte) { - root_req := Request{key: CacheKey{name: name, offset: 0, length: 0}, value: make(chan CachedValue, 1)} - server.reqs <- root_req +func (server *Server) getTile(ctx context.Context, httpHeaders map[string]string, name string, z uint8, x uint32, y uint32, ext string) (int, map[string]string, []byte) { + rootReq := Request{key: CacheKey{name: name, offset: 0, length: 0}, value: make(chan CachedValue, 1)} + server.reqs <- rootReq // https://golang.org/doc/faq#atomic_maps - root_value := <-root_req.value - header := root_value.header + rootValue := <-rootReq.value + header := rootValue.header - if !root_value.ok { - return 404, http_headers, []byte("Archive not found") + if !rootValue.ok { + return 404, httpHeaders, []byte("Archive not found") } if z < header.MinZoom || z > header.MaxZoom { - return 404, http_headers, []byte("Tile not found") + return 404, httpHeaders, []byte("Tile not found") } switch header.TileType { case Mvt: if ext != "mvt" { - return 400, http_headers, []byte("path mismatch: archive is type MVT (.mvt)") + return 400, httpHeaders, []byte("path mismatch: archive is type MVT (.mvt)") } case Png: if ext != "png" { - return 400, http_headers, []byte("path mismatch: archive is type PNG (.png)") + return 400, httpHeaders, []byte("path mismatch: archive is type PNG (.png)") } case Jpeg: if ext != "jpg" { - return 400, http_headers, []byte("path mismatch: archive is type JPEG (.jpg)") + return 400, httpHeaders, []byte("path mismatch: archive is type JPEG (.jpg)") } case Webp: if ext != "webp" { - return 400, http_headers, []byte("path mismatch: archive is type WebP (.webp)") + return 400, httpHeaders, []byte("path mismatch: archive is type WebP (.webp)") } case Avif: if ext != "avif" { - return 400, http_headers, []byte("path mismatch: archive is type AVIF (.avif)") + return 400, httpHeaders, []byte("path mismatch: archive is type AVIF (.avif)") } } - tile_id := ZxyToId(z, x, y) - dir_offset, dir_len := header.RootOffset, header.RootLength + tileID := ZxyToID(z, x, y) + dirOffset, dirLen := header.RootOffset, header.RootLength for depth := 0; depth <= 3; depth++ { - dir_req := Request{key: CacheKey{name: name, offset: dir_offset, length: dir_len}, value: make(chan CachedValue, 1)} - server.reqs <- dir_req - dir_value := <-dir_req.value - directory := dir_value.directory - entry, ok := find_tile(directory, tile_id) + dirReq := Request{key: CacheKey{name: name, offset: dirOffset, length: dirLen}, value: make(chan CachedValue, 1)} + server.reqs <- dirReq + dirValue := <-dirReq.value + directory := dirValue.directory + entry, ok := findTile(directory, tileID) if !ok { break } @@ -318,34 +318,34 @@ func (server *Server) get_tile(ctx context.Context, http_headers map[string]stri if entry.RunLength > 0 { r, err := server.bucket.NewRangeReader(ctx, name+".pmtiles", int64(header.TileDataOffset+entry.Offset), int64(entry.Length)) if err != nil { - return 500, http_headers, []byte("Network error") + return 500, httpHeaders, []byte("Network error") } defer r.Close() b, err := io.ReadAll(r) if err != nil { - return 500, http_headers, []byte("I/O error") + return 500, httpHeaders, []byte("I/O error") } - if header_val, ok := headerContentType(header); ok { - http_headers["Content-Type"] = header_val + if headerVal, ok := headerContentType(header); ok { + httpHeaders["Content-Type"] = headerVal } - if header_val, ok := headerContentEncoding(header.TileCompression); ok { - http_headers["Content-Encoding"] = header_val + if headerVal, ok := headerContentEncoding(header.TileCompression); ok { + httpHeaders["Content-Encoding"] = headerVal } - return 200, http_headers, b + return 200, httpHeaders, b } else { - dir_offset = header.LeafDirectoryOffset + entry.Offset - dir_len = uint64(entry.Length) + dirOffset = header.LeafDirectoryOffset + entry.Offset + dirLen = uint64(entry.Length) } } - return 204, http_headers, nil + return 204, httpHeaders, nil } var tilePattern = regexp.MustCompile(`^\/([-A-Za-z0-9_\/!-_\.\*'\(\)']+)\/(\d+)\/(\d+)\/(\d+)\.([a-z]+)$`) var metadataPattern = regexp.MustCompile(`^\/([-A-Za-z0-9_\/!-_\.\*'\(\)']+)\/metadata$`) var tileJSONPattern = regexp.MustCompile(`^\/([-A-Za-z0-9_\/!-_\.\*'\(\)']+)\.json$`) -func parse_tile_path(path string) (bool, string, uint8, uint32, uint32, string) { +func parseTilePath(path string) (bool, string, uint8, uint32, uint32, string) { if res := tilePattern.FindStringSubmatch(path); res != nil { name := res[1] z, _ := strconv.ParseUint(res[2], 10, 8) @@ -357,7 +357,7 @@ func parse_tile_path(path string) (bool, string, uint8, uint32, uint32, string) return false, "", 0, 0, 0, "" } -func parse_tilejson_path(path string) (bool, string) { +func parseTilejsonPath(path string) (bool, string) { if res := tileJSONPattern.FindStringSubmatch(path); res != nil { name := res[1] return true, name @@ -365,7 +365,7 @@ func parse_tilejson_path(path string) (bool, string) { return false, "" } -func parse_metadata_path(path string) (bool, string) { +func parseMetadataPath(path string) (bool, string) { if res := metadataPattern.FindStringSubmatch(path); res != nil { name := res[1] return true, name @@ -374,24 +374,24 @@ func parse_metadata_path(path string) (bool, string) { } func (server *Server) Get(ctx context.Context, path string) (int, map[string]string, []byte) { - http_headers := make(map[string]string) + httpHeaders := make(map[string]string) if len(server.cors) > 0 { - http_headers["Access-Control-Allow-Origin"] = server.cors + httpHeaders["Access-Control-Allow-Origin"] = server.cors } - if ok, key, z, x, y, ext := parse_tile_path(path); ok { - return server.get_tile(ctx, http_headers, key, z, x, y, ext) + if ok, key, z, x, y, ext := parseTilePath(path); ok { + return server.getTile(ctx, httpHeaders, key, z, x, y, ext) } - if ok, key := parse_tilejson_path(path); ok { - return server.get_tilejson(ctx, http_headers, key) + if ok, key := parseTilejsonPath(path); ok { + return server.getTileJSON(ctx, httpHeaders, key) } - if ok, key := parse_metadata_path(path); ok { - return server.get_metadata(ctx, http_headers, key) + if ok, key := parseMetadataPath(path); ok { + return server.getMetadata(ctx, httpHeaders, key) } if path == "/" { - return 204, http_headers, []byte{} + return 204, httpHeaders, []byte{} } - return 404, http_headers, []byte("Path not found") + return 404, httpHeaders, []byte("Path not found") } diff --git a/pmtiles/server_test.go b/pmtiles/server_test.go index 0f639a9..bcbee7c 100644 --- a/pmtiles/server_test.go +++ b/pmtiles/server_test.go @@ -6,16 +6,16 @@ import ( ) func TestRegex(t *testing.T) { - ok, key, z, x, y, ext := parse_tile_path("/foo/0/0/0") + ok, key, z, x, y, ext := parseTilePath("/foo/0/0/0") assert.False(t, ok) - ok, key, z, x, y, ext = parse_tile_path("/foo/0/0/0.mvt") + ok, key, z, x, y, ext = parseTilePath("/foo/0/0/0.mvt") assert.True(t, ok) assert.Equal(t, key, "foo") assert.Equal(t, z, uint8(0)) assert.Equal(t, x, uint32(0)) assert.Equal(t, y, uint32(0)) assert.Equal(t, ext, "mvt") - ok, key, z, x, y, ext = parse_tile_path("/foo/bar/0/0/0.mvt") + ok, key, z, x, y, ext = parseTilePath("/foo/bar/0/0/0.mvt") assert.True(t, ok) assert.Equal(t, key, "foo/bar") assert.Equal(t, z, uint8(0)) @@ -23,17 +23,17 @@ func TestRegex(t *testing.T) { assert.Equal(t, y, uint32(0)) assert.Equal(t, ext, "mvt") // https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html - ok, key, z, x, y, ext = parse_tile_path("/!-_.*'()/0/0/0.mvt") + ok, key, z, x, y, ext = parseTilePath("/!-_.*'()/0/0/0.mvt") assert.True(t, ok) assert.Equal(t, key, "!-_.*'()") assert.Equal(t, z, uint8(0)) assert.Equal(t, x, uint32(0)) assert.Equal(t, y, uint32(0)) assert.Equal(t, ext, "mvt") - ok, key = parse_metadata_path("/!-_.*'()/metadata") + ok, key = parseMetadataPath("/!-_.*'()/metadata") assert.True(t, ok) assert.Equal(t, key, "!-_.*'()") - ok, key = parse_tilejson_path("/!-_.*'().json") + ok, key = parseTilejsonPath("/!-_.*'().json") assert.True(t, ok) assert.Equal(t, key, "!-_.*'()") } diff --git a/pmtiles/show.go b/pmtiles/show.go index c4655a2..8c2f861 100644 --- a/pmtiles/show.go +++ b/pmtiles/show.go @@ -13,7 +13,7 @@ import ( "os" ) -func Show(logger *log.Logger, bucketURL string, key string, show_metadata_only bool, show_tilejson bool, public_url string, show_tile bool, z int, x int, y int) error { +func Show(logger *log.Logger, bucketURL string, key string, showMetadataOnly bool, showTilejson bool, publicURL string, showTile bool, z int, x int, y int) error { ctx := context.Background() bucketURL, key, err := NormalizeBucketKey(bucketURL, "", key) @@ -40,75 +40,75 @@ func Show(logger *log.Logger, bucketURL string, key string, show_metadata_only b } r.Close() - header, err := deserialize_header(b[0:HEADERV3_LEN_BYTES]) + header, err := deserializeHeader(b[0:HeaderV3LenBytes]) if err != nil { // check to see if it's a V2 file if string(b[0:2]) == "PM" { - spec_version := b[2] - return fmt.Errorf("PMTiles version %d detected; please use 'pmtiles convert' to upgrade to version 3.", spec_version) + specVersion := b[2] + return fmt.Errorf("PMTiles version %d detected; please use 'pmtiles convert' to upgrade to version 3", specVersion) } return fmt.Errorf("Failed to read %s, %w", key, err) } - if !show_tile { - var tile_type string + if !showTile { + var tileType string switch header.TileType { case Mvt: - tile_type = "Vector Protobuf (MVT)" + tileType = "Vector Protobuf (MVT)" case Png: - tile_type = "Raster PNG" + tileType = "Raster PNG" case Jpeg: - tile_type = "Raster Jpeg" + tileType = "Raster Jpeg" case Webp: - tile_type = "Raster WebP" + tileType = "Raster WebP" case Avif: - tile_type = "Raster AVIF" + tileType = "Raster AVIF" default: - tile_type = "Unknown" + tileType = "Unknown" } - metadata_reader, err := bucket.NewRangeReader(ctx, key, int64(header.MetadataOffset), int64(header.MetadataLength)) + metadataReader, err := bucket.NewRangeReader(ctx, key, int64(header.MetadataOffset), int64(header.MetadataLength)) if err != nil { return fmt.Errorf("Failed to create range reader for %s, %w", key, err) } - var metadata_bytes []byte + var metadataBytes []byte if header.InternalCompression == Gzip { - r, _ := gzip.NewReader(metadata_reader) - metadata_bytes, err = io.ReadAll(r) + r, _ := gzip.NewReader(metadataReader) + metadataBytes, err = io.ReadAll(r) if err != nil { return fmt.Errorf("Failed to read %s, %w", key, err) } } else { - metadata_bytes, err = io.ReadAll(metadata_reader) + metadataBytes, err = io.ReadAll(metadataReader) if err != nil { return fmt.Errorf("Failed to read %s, %w", key, err) } } - metadata_reader.Close() + metadataReader.Close() - if show_metadata_only && show_tilejson { - return fmt.Errorf("Cannot use --metadata and --tilejson together.") + if showMetadataOnly && showTilejson { + return fmt.Errorf("cannot use --metadata and --tilejson together") } - if show_metadata_only { - fmt.Print(string(metadata_bytes)) - } else if show_tilejson { - if public_url == "" { + if showMetadataOnly { + fmt.Print(string(metadataBytes)) + } else if showTilejson { + if publicURL == "" { // Using Fprintf instead of logger here, as this message should be written to Stderr in case // Stdout is being redirected. - fmt.Fprintln(os.Stderr, "Warning: No --public-url specified; using placeholder tiles URL.") + fmt.Fprintln(os.Stderr, "no --public-url specified; using placeholder tiles URL") } - tilejson_bytes, err := CreateTilejson(header, metadata_bytes, public_url) + tilejsonBytes, err := CreateTilejson(header, metadataBytes, publicURL) if err != nil { return fmt.Errorf("Failed to create tilejson for %s, %w", key, err) } - fmt.Print(string(tilejson_bytes)) + fmt.Print(string(tilejsonBytes)) } else { fmt.Printf("pmtiles spec version: %d\n", header.SpecVersion) // fmt.Printf("total size: %s\n", humanize.Bytes(uint64(r.Size()))) - fmt.Printf("tile type: %s\n", tile_type) + fmt.Printf("tile type: %s\n", tileType) fmt.Printf("bounds: %f,%f %f,%f\n", float64(header.MinLonE7)/10000000, float64(header.MinLatE7)/10000000, float64(header.MaxLonE7)/10000000, float64(header.MaxLatE7)/10000000) fmt.Printf("min zoom: %d\n", header.MinZoom) fmt.Printf("max zoom: %d\n", header.MaxZoom) @@ -121,9 +121,9 @@ func Show(logger *log.Logger, bucketURL string, key string, show_metadata_only b fmt.Printf("internal compression: %d\n", header.InternalCompression) fmt.Printf("tile compression: %d\n", header.TileCompression) - var metadata_map map[string]interface{} - json.Unmarshal(metadata_bytes, &metadata_map) - for k, v := range metadata_map { + var metadataMap map[string]interface{} + json.Unmarshal(metadataBytes, &metadataMap) + for k, v := range metadataMap { switch v := v.(type) { case string: fmt.Println(k, v) @@ -135,13 +135,13 @@ func Show(logger *log.Logger, bucketURL string, key string, show_metadata_only b } else { // write the tile to stdout - tile_id := ZxyToId(uint8(z), uint32(x), uint32(y)) + tileID := ZxyToID(uint8(z), uint32(x), uint32(y)) - dir_offset := header.RootOffset - dir_length := header.RootLength + dirOffset := header.RootOffset + dirLength := header.RootLength for depth := 0; depth <= 3; depth++ { - r, err := bucket.NewRangeReader(ctx, key, int64(dir_offset), int64(dir_length)) + r, err := bucket.NewRangeReader(ctx, key, int64(dirOffset), int64(dirLength)) if err != nil { return fmt.Errorf("Network error") } @@ -150,24 +150,24 @@ func Show(logger *log.Logger, bucketURL string, key string, show_metadata_only b if err != nil { return fmt.Errorf("I/O Error") } - directory := deserialize_entries(bytes.NewBuffer(b)) - entry, ok := find_tile(directory, tile_id) + directory := deserializeEntries(bytes.NewBuffer(b)) + entry, ok := findTile(directory, tileID) if ok { if entry.RunLength > 0 { - tile_r, err := bucket.NewRangeReader(ctx, key, int64(header.TileDataOffset+entry.Offset), int64(entry.Length)) + tileReader, err := bucket.NewRangeReader(ctx, key, int64(header.TileDataOffset+entry.Offset), int64(entry.Length)) if err != nil { return fmt.Errorf("Network error") } - defer tile_r.Close() - tile_b, err := io.ReadAll(tile_r) + defer tileReader.Close() + tileBytes, err := io.ReadAll(tileReader) if err != nil { return fmt.Errorf("I/O Error") } - os.Stdout.Write(tile_b) + os.Stdout.Write(tileBytes) break } else { - dir_offset = header.LeafDirectoryOffset + entry.Offset - dir_length = uint64(entry.Length) + dirOffset = header.LeafDirectoryOffset + entry.Offset + dirLength = uint64(entry.Length) } } else { fmt.Println("Tile not found in archive.") diff --git a/pmtiles/tile_id.go b/pmtiles/tile_id.go index c818ccc..66e259a 100644 --- a/pmtiles/tile_id.go +++ b/pmtiles/tile_id.go @@ -10,7 +10,7 @@ func rotate(n uint64, x *uint64, y *uint64, rx uint64, ry uint64) { } } -func t_on_level(z uint8, pos uint64) (uint8, uint32, uint32) { +func tOnLevel(z uint8, pos uint64) (uint8, uint32, uint32) { var n uint64 = 1 << z rx, ry, t := pos, pos, pos var tx uint64 @@ -27,7 +27,7 @@ func t_on_level(z uint8, pos uint64) (uint8, uint32, uint32) { return uint8(z), uint32(tx), uint32(ty) } -func ZxyToId(z uint8, x uint32, y uint32) uint64 { +func ZxyToID(z uint8, x uint32, y uint32) uint64 { var acc uint64 var tz uint8 for ; tz < z; tz++ { @@ -56,33 +56,33 @@ func ZxyToId(z uint8, x uint32, y uint32) uint64 { return acc + d } -func IdToZxy(i uint64) (uint8, uint32, uint32) { +func IDToZxy(i uint64) (uint8, uint32, uint32) { var acc uint64 var z uint8 for { - var num_tiles uint64 - num_tiles = (1 << z) * (1 << z) - if acc+num_tiles > i { - return t_on_level(z, i-acc) + var numTiles uint64 + numTiles = (1 << z) * (1 << z) + if acc+numTiles > i { + return tOnLevel(z, i-acc) } - acc += num_tiles + acc += numTiles z++ } } // fast parent ID calculation without converting to ZXY. -func ParentId(i uint64) uint64 { +func ParentID(i uint64) uint64 { var acc uint64 - var last_acc uint64 + var lastAcc uint64 var z uint8 for { - var num_tiles uint64 - num_tiles = (1 << z) * (1 << z) - if acc+num_tiles > i { - return last_acc + (i-acc)/4 + var numTiles uint64 + numTiles = (1 << z) * (1 << z) + if acc+numTiles > i { + return lastAcc + (i-acc)/4 } - last_acc = acc - acc += num_tiles + lastAcc = acc + acc += numTiles z++ } diff --git a/pmtiles/tile_id_test.go b/pmtiles/tile_id_test.go index 8669914..c68ed13 100644 --- a/pmtiles/tile_id_test.go +++ b/pmtiles/tile_id_test.go @@ -6,24 +6,24 @@ import ( ) func TestZxyToId(t *testing.T) { - assert.Equal(t, uint64(0), ZxyToId(0, 0, 0)) - assert.Equal(t, uint64(1), ZxyToId(1, 0, 0)) - assert.Equal(t, uint64(2), ZxyToId(1, 0, 1)) - assert.Equal(t, uint64(3), ZxyToId(1, 1, 1)) - assert.Equal(t, uint64(4), ZxyToId(1, 1, 0)) - assert.Equal(t, uint64(5), ZxyToId(2, 0, 0)) + assert.Equal(t, uint64(0), ZxyToID(0, 0, 0)) + assert.Equal(t, uint64(1), ZxyToID(1, 0, 0)) + assert.Equal(t, uint64(2), ZxyToID(1, 0, 1)) + assert.Equal(t, uint64(3), ZxyToID(1, 1, 1)) + assert.Equal(t, uint64(4), ZxyToID(1, 1, 0)) + assert.Equal(t, uint64(5), ZxyToID(2, 0, 0)) } func TestIdToZxy(t *testing.T) { - z, x, y := IdToZxy(0) + z, x, y := IDToZxy(0) assert.Equal(t, uint8(0), z) assert.Equal(t, uint32(0), x) assert.Equal(t, uint32(0), y) - z, x, y = IdToZxy(1) + z, x, y = IDToZxy(1) assert.Equal(t, uint8(1), z) assert.Equal(t, uint32(0), x) assert.Equal(t, uint32(0), y) - z, x, y = IdToZxy(19078479) + z, x, y = IDToZxy(19078479) assert.Equal(t, uint8(12), z) assert.Equal(t, uint32(3423), x) assert.Equal(t, uint32(1763), y) @@ -36,8 +36,8 @@ func TestManyTileIds(t *testing.T) { for z = 0; z < 10; z++ { for x = 0; x < (1 << z); x++ { for y = 0; y < (1 << z); y++ { - id := ZxyToId(z, x, y) - rz, rx, ry := IdToZxy(id) + id := ZxyToID(z, x, y) + rz, rx, ry := IDToZxy(id) if !(z == rz && x == rx && y == ry) { t.Fatalf(`fail on %d %d %d`, z, x, y) } @@ -51,19 +51,19 @@ func TestExtremes(t *testing.T) { for tz = 0; tz < 32; tz++ { var dim uint32 dim = (1 << tz) - 1 - z, x, y := IdToZxy(ZxyToId(tz, 0, 0)) + z, x, y := IDToZxy(ZxyToID(tz, 0, 0)) assert.Equal(t, tz, z) assert.Equal(t, uint32(0), x) assert.Equal(t, uint32(0), y) - z, x, y = IdToZxy(ZxyToId(z, dim, 0)) + z, x, y = IDToZxy(ZxyToID(z, dim, 0)) assert.Equal(t, tz, z) assert.Equal(t, dim, x) assert.Equal(t, uint32(0), y) - z, x, y = IdToZxy(ZxyToId(z, 0, dim)) + z, x, y = IDToZxy(ZxyToID(z, 0, dim)) assert.Equal(t, tz, z) assert.Equal(t, uint32(0), x) assert.Equal(t, dim, y) - z, x, y = IdToZxy(ZxyToId(z, dim, dim)) + z, x, y = IDToZxy(ZxyToID(z, dim, dim)) assert.Equal(t, tz, z) assert.Equal(t, dim, x) assert.Equal(t, dim, y) @@ -71,25 +71,25 @@ func TestExtremes(t *testing.T) { } func TestParent(t *testing.T) { - assert.Equal(t, ZxyToId(0, 0, 0), ParentId(ZxyToId(1, 0, 0))) + assert.Equal(t, ZxyToID(0, 0, 0), ParentID(ZxyToID(1, 0, 0))) - assert.Equal(t, ZxyToId(1, 0, 0), ParentId(ZxyToId(2, 0, 0))) - assert.Equal(t, ZxyToId(1, 0, 0), ParentId(ZxyToId(2, 0, 1))) - assert.Equal(t, ZxyToId(1, 0, 0), ParentId(ZxyToId(2, 1, 0))) - assert.Equal(t, ZxyToId(1, 0, 0), ParentId(ZxyToId(2, 1, 1))) + assert.Equal(t, ZxyToID(1, 0, 0), ParentID(ZxyToID(2, 0, 0))) + assert.Equal(t, ZxyToID(1, 0, 0), ParentID(ZxyToID(2, 0, 1))) + assert.Equal(t, ZxyToID(1, 0, 0), ParentID(ZxyToID(2, 1, 0))) + assert.Equal(t, ZxyToID(1, 0, 0), ParentID(ZxyToID(2, 1, 1))) - assert.Equal(t, ZxyToId(1, 0, 1), ParentId(ZxyToId(2, 0, 2))) - assert.Equal(t, ZxyToId(1, 0, 1), ParentId(ZxyToId(2, 0, 3))) - assert.Equal(t, ZxyToId(1, 0, 1), ParentId(ZxyToId(2, 1, 2))) - assert.Equal(t, ZxyToId(1, 0, 1), ParentId(ZxyToId(2, 1, 3))) + assert.Equal(t, ZxyToID(1, 0, 1), ParentID(ZxyToID(2, 0, 2))) + assert.Equal(t, ZxyToID(1, 0, 1), ParentID(ZxyToID(2, 0, 3))) + assert.Equal(t, ZxyToID(1, 0, 1), ParentID(ZxyToID(2, 1, 2))) + assert.Equal(t, ZxyToID(1, 0, 1), ParentID(ZxyToID(2, 1, 3))) - assert.Equal(t, ZxyToId(1, 1, 0), ParentId(ZxyToId(2, 2, 0))) - assert.Equal(t, ZxyToId(1, 1, 0), ParentId(ZxyToId(2, 2, 1))) - assert.Equal(t, ZxyToId(1, 1, 0), ParentId(ZxyToId(2, 3, 0))) - assert.Equal(t, ZxyToId(1, 1, 0), ParentId(ZxyToId(2, 3, 1))) + assert.Equal(t, ZxyToID(1, 1, 0), ParentID(ZxyToID(2, 2, 0))) + assert.Equal(t, ZxyToID(1, 1, 0), ParentID(ZxyToID(2, 2, 1))) + assert.Equal(t, ZxyToID(1, 1, 0), ParentID(ZxyToID(2, 3, 0))) + assert.Equal(t, ZxyToID(1, 1, 0), ParentID(ZxyToID(2, 3, 1))) - assert.Equal(t, ZxyToId(1, 1, 1), ParentId(ZxyToId(2, 2, 2))) - assert.Equal(t, ZxyToId(1, 1, 1), ParentId(ZxyToId(2, 2, 3))) - assert.Equal(t, ZxyToId(1, 1, 1), ParentId(ZxyToId(2, 3, 2))) - assert.Equal(t, ZxyToId(1, 1, 1), ParentId(ZxyToId(2, 3, 3))) + assert.Equal(t, ZxyToID(1, 1, 1), ParentID(ZxyToID(2, 2, 2))) + assert.Equal(t, ZxyToID(1, 1, 1), ParentID(ZxyToID(2, 2, 3))) + assert.Equal(t, ZxyToID(1, 1, 1), ParentID(ZxyToID(2, 3, 2))) + assert.Equal(t, ZxyToID(1, 1, 1), ParentID(ZxyToID(2, 3, 3))) } diff --git a/pmtiles/tilejson.go b/pmtiles/tilejson.go index 207a979..68ed25f 100644 --- a/pmtiles/tilejson.go +++ b/pmtiles/tilejson.go @@ -4,35 +4,35 @@ import ( "encoding/json" ) -func CreateTilejson(header HeaderV3, metadata_bytes []byte, tileUrl string) ([]byte, error) { - var metadata_map map[string]interface{} - json.Unmarshal(metadata_bytes, &metadata_map) +func CreateTilejson(header HeaderV3, metadataBytes []byte, tileURL string) ([]byte, error) { + var metadataMap map[string]interface{} + json.Unmarshal(metadataBytes, &metadataMap) tilejson := make(map[string]interface{}) tilejson["tilejson"] = "3.0.0" tilejson["scheme"] = "xyz" - if tileUrl == "" { - tileUrl = "https://example.com" + if tileURL == "" { + tileURL = "https://example.com" } - tilejson["tiles"] = []string{tileUrl + "/{z}/{x}/{y}" + headerExt(header)} - tilejson["vector_layers"] = metadata_map["vector_layers"] + tilejson["tiles"] = []string{tileURL + "/{z}/{x}/{y}" + headerExt(header)} + tilejson["vector_layers"] = metadataMap["vector_layers"] - if val, ok := metadata_map["attribution"]; ok { + if val, ok := metadataMap["attribution"]; ok { tilejson["attribution"] = val } - if val, ok := metadata_map["description"]; ok { + if val, ok := metadataMap["description"]; ok { tilejson["description"] = val } - if val, ok := metadata_map["name"]; ok { + if val, ok := metadataMap["name"]; ok { tilejson["name"] = val } - if val, ok := metadata_map["version"]; ok { + if val, ok := metadataMap["version"]; ok { tilejson["version"] = val } diff --git a/pmtiles/upload.go b/pmtiles/upload.go index 4d4d7f4..dfacea3 100644 --- a/pmtiles/upload.go +++ b/pmtiles/upload.go @@ -10,7 +10,7 @@ import ( "os" ) -func Upload(logger *log.Logger, input string, bucket string, key string, max_concurrency int) error { +func Upload(logger *log.Logger, input string, bucket string, key string, maxConcurrency int) error { ctx := context.Background() b, err := blob.OpenBucket(ctx, bucket) if err != nil { @@ -34,7 +34,7 @@ func Upload(logger *log.Logger, input string, bucket string, key string, max_con opts := &blob.WriterOptions{ BufferSize: 256 * 1024 * 1024, - MaxConcurrency: max_concurrency, + MaxConcurrency: maxConcurrency, } w, err := b.NewWriter(ctx, key, opts) diff --git a/pmtiles/verify.go b/pmtiles/verify.go index 34c508b..76e28ed 100644 --- a/pmtiles/verify.go +++ b/pmtiles/verify.go @@ -39,7 +39,7 @@ func Verify(logger *log.Logger, file string) error { } r.Close() - header, err := deserialize_header(b[0:HEADERV3_LEN_BYTES]) + header, err := deserializeHeader(b[0:HeaderV3LenBytes]) var CollectEntries func(uint64, uint64, func(EntryV3)) @@ -54,7 +54,7 @@ func Verify(logger *log.Logger, file string) error { panic(fmt.Errorf("I/O Error")) } - directory := deserialize_entries(bytes.NewBuffer(b)) + directory := deserializeEntries(bytes.NewBuffer(b)) for _, entry := range directory { if entry.RunLength > 0 { f(entry) @@ -64,25 +64,25 @@ func Verify(logger *log.Logger, file string) error { } } - var min_tile_id uint64 - var max_tile_id uint64 - min_tile_id = math.MaxUint64 - max_tile_id = 0 + var minTileID uint64 + var maxTileID uint64 + minTileID = math.MaxUint64 + maxTileID = 0 - addressed_tiles := 0 - tile_entries := 0 + addressedTiles := 0 + tileEntries := 0 offsets := roaring64.New() - var current_offset uint64 + var currentOffset uint64 CollectEntries(header.RootOffset, header.RootLength, func(e EntryV3) { offsets.Add(e.Offset) - addressed_tiles += int(e.RunLength) - tile_entries += 1 + addressedTiles += int(e.RunLength) + tileEntries++ - if e.TileId < min_tile_id { - min_tile_id = e.TileId + if e.TileID < minTileID { + minTileID = e.TileID } - if e.TileId > max_tile_id { - max_tile_id = e.TileId + if e.TileID > maxTileID { + maxTileID = e.TileID } if e.Offset+uint64(e.Length) > header.TileDataLength { @@ -91,31 +91,31 @@ func Verify(logger *log.Logger, file string) error { if header.Clustered { if !offsets.Contains(e.Offset) { - if e.Offset != current_offset { + if e.Offset != currentOffset { fmt.Printf("Invalid: out-of-order entry %v in clustered archive.", e) } - current_offset += uint64(e.Length) + currentOffset += uint64(e.Length) } } }) - if uint64(addressed_tiles) != header.AddressedTilesCount { - fmt.Printf("Invalid: header AddressedTilesCount=%v but %v tiles addressed.", header.AddressedTilesCount, addressed_tiles) + if uint64(addressedTiles) != header.AddressedTilesCount { + fmt.Printf("Invalid: header AddressedTilesCount=%v but %v tiles addressed.", header.AddressedTilesCount, addressedTiles) } - if uint64(tile_entries) != header.TileEntriesCount { - fmt.Printf("Invalid: header TileEntriesCount=%v but %v tile entries.", header.TileEntriesCount, tile_entries) + if uint64(tileEntries) != header.TileEntriesCount { + fmt.Printf("Invalid: header TileEntriesCount=%v but %v tile entries.", header.TileEntriesCount, tileEntries) } if offsets.GetCardinality() != header.TileContentsCount { fmt.Printf("Invalid: header TileContentsCount=%v but %v tile contents.", header.TileContentsCount, offsets.GetCardinality()) } - if z, _, _ := IdToZxy(min_tile_id); z != header.MinZoom { + if z, _, _ := IDToZxy(minTileID); z != header.MinZoom { fmt.Printf("Invalid: header MinZoom=%v does not match min tile z %v", header.MinZoom, z) } - if z, _, _ := IdToZxy(max_tile_id); z != header.MaxZoom { + if z, _, _ := IDToZxy(maxTileID); z != header.MaxZoom { fmt.Printf("Invalid: header MaxZoom=%v does not match max tile z %v", header.MaxZoom, z) }