-
Notifications
You must be signed in to change notification settings - Fork 19
/
config.yaml.sample
447 lines (429 loc) · 16.3 KB
/
config.yaml.sample
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
queue:
# type can be `sqs`, `file`, `mem`, or `redis`
# can also be a list of queues to configure multiple
type: mem
name: <sqs queue/list of queue names/queue file name/redis key/>
# if using sqs configuration
sqs:
region: <aws-region>
queue-mapping:
type: <single/multiple>
# for multiple configuration
multiple:
- start-zoom: 10
# end_zoom is exclusive
end-zoom: 16
queue-name: <name of configured queue>
# if set, the group-by-zoom tells the mapper to group
# coordinates by common tile parents
group-by-zoom: 10
# NOTE: start_zoom and end_zoom can be None if queue is only used for
# priority purposes
# the order here in queue-mapping is important, and is in
# priority order that the queues will be checking when reading
# messages during processing
message-marshall:
# controls how queue messages get marshalled/unmarshalled
# this should correspond to the appropriate queue-mapping implementation
type: <single/multiple>
# can omit the in-inflight configuration completely
in-flight:
type: <redis>
message-tracker:
# should correspond with the marshall and queue implementations
type: <single/multiple>
# NOTE: message-visibility is sqs specific in utility
# config is shared across queues
message-visibility:
# on partial completion, how long to extend the visibility
extend-seconds: 1800
# don't exceed max, which is 12 hours for sqs
max-seconds: 43200
# read visibility timeout for receive message calls
timeout-seconds: 3600
# Uncomment this section to activate statsd
#statsd:
# host: 127.0.0.1
# port: 8125
# prefix: dev.tilequeue
store:
type: s3 # Can also be `directory`, which would dump the tiles to disk.
name: <s3 bucket/tile directory name>
# The following store properties are s3 specific.
reduced-redundancy: true
date-prefix: 19851026
# control the s3 tile key for the object
# can either be hash-prefix or prefix-hash
# hash-prefix -> hash comes before the prefix
# prefix-hash -> prefix comes before the hash
key-format-type: <hash-prefix|prefix-hash>
# optional tags that get saved onto the s3 objects
tags:
prefix: 19851026
run_id: 19851026-1
aws:
# credentials are optional, and better to use an iam role assigned
# to the instance if possible
credentials:
aws_access_key_id: <aws_access_key_id>
aws_secret_access_key: <aws_secret_access_key>
tiles:
seed:
# can use any combination of the following options for the
# tilequeue seed command
# all tiles in a particular zoom range
all:
zoom-start: 0
zoom-until: 10
# parse the json from Mapzen's metro extracts for a particular
# zoom range
metro-extract:
url: https://raw.githubusercontent.com/mapzen/metroextractor-cities/master/cities.json
zoom-start: 11
zoom-until: 15
# Can also be set to an array of metro-extract city names, like
# 'new-york_new-york', to only seed tiles from those areas
# rather than all of them.
cities:
- new-york_new-york
# parse the csv file from Mapbox's post on the top 50k most
# requested tiles
top-tiles:
url: https://gist.github.com/brunosan/b7ce3df8b48229a61b5b/raw/37e42e77f253bc204076111c92acc4d5e653edd2/top_50k_tiles.csv
zoom-start: 11
zoom-until: 20
# Specify any number of custom bounding boxes to seed tiles for.
custom:
zoom-start: 10
zoom-until: 10
# Must be set to an array of bounding boxes in `[left, bottom, right,
# top]` (or `[min lon, min lat, max lon, max lat]`) format
bboxes:
# whether the tiles that are seeded should also be added to the
# tiles of interest, or just enqueued
should-add-to-tiles-of-interest: true
# how many threads to use when enqueueing
n-threads: 20
# whether the tiles should be unique in memory
# when seeding many tiles that are known to be unique, this should
# be set to false to save memory
unique: true
# zoom where tile content stops changing
max-zoom-with-changes: 16
process:
# number of simultaneous "querysets" to issue to the database. The
# query for each layer will be issued in parallel to the same
# database. This can be 0 or unspecified, in which case the number
# will be inferred from the number of database names configured
# below.
n-simultaneous-query-sets: 1
# whether to print out the internal python queue sizes
log-queue-sizes: true
# and at what interval
log-queue-sizes-interval-seconds: 30
query-config: <path/to/vector-datasource/queries.yaml>
template-path: <path/to/vector-datasource/queries>
# whether to reload jinja query templates on each request. This
# should be off in production.
reload-templates: false
# extensions of formats to generate
# buffered Mapbox Vector Tiles are also possible by specifying mvtb
formats: [json, topojson, mvt]
# additionally, the data included for some formats expects to be
# buffered. This is where buffers per layer or per geometry type can
# be specified, with layers trumping geometry types
buffer:
mvtb:
layer:
earth: {point: 256}
water: {point: 256}
places: {point: 128}
geometry:
point: 64
line: 8
polygon: 8
# control how python code from yaml is used
yaml:
# dotted name or runtime
type: parse
parse:
# this will parse the yaml files and compile them on startup
# useful when iterating on updating the yaml files themselves
path: <path/to/vector-datasource/yaml-directory>
callable:
# this will call a function that should return a mapping from
# layer name to function that generates the output properties
# mapping returned should look like:
# layer-name -> function(shape, properties, feature_id)
# useful when the output functions have already been generated
# and it's simply a matter of pointing to them
dotted-name: path.to.function.returning.mapping
args:
- 'any-args-to-pass'
logging:
# logging.conf on this page:
# https://docs.python.org/2/howto/logging.html#logging-basic-tutorial
config: logging.conf.sample
redis:
host: localhost
port: 6379
db: 0
# expected data source is postgresql
postgresql:
host: localhost
port: 5432
# multiple databases can be specified, and these are iterated
# through to balance query loads. This is useful when connecting to
# pgbouncer, which can dispatch to different back end databases
# based on the name.
dbnames: [osm]
user: osm
password:
wof:
# url path to neighbourhoods, microhoods, and macrohoods meta csv files
neighbourhoods-meta-url: https://github.com/whosonfirst/whosonfirst-data/raw/master/meta/wof-neighbourhood-latest.csv
microhoods-meta-url: https://github.com/whosonfirst/whosonfirst-data/raw/master/meta/wof-microhood-latest.csv
macrohoods-meta-url: https://github.com/whosonfirst/whosonfirst-data/raw/master/meta/wof-macrohood-latest.csv
boroughs-meta-url: https://github.com/whosonfirst/whosonfirst-data/raw/master/meta/wof-borough-latest.csv
# url path prefix for wof raw data
data-prefix-url: https://data.whosonfirst.org
# filesystem path to wof data checkout
data-path: /tmp/whosonfirst-data
postgresql:
host: localhost
port: 5432
dbname: osm
user: osm
password:
# A metatile is a .zip package of multiple tiles. Storing multiple, related
# tiles together can be beneficial by reducing the number of writes to disk when
# storing and loading them.
metatile:
# The metatile size setting has the following effects by itself. But please
# note that the `tile-sizes` setting can affect this too!
#
# - null or omitted entry: Individual tiles, one for each format, will be
# saved on disk. This can be a good choice for local development use, or to
# save files to be served directly from storage (i.e: without tapalcatl).
#
# - 1: Tiles with the same coordinate will be saved together in a .zip file,
# which will require using tapalcatl to serve the individual files to users.
# This is a good choice to reduce the number of writes to disk.
#
# - 2: A "512px" tile and the 4 "256px" tiles covering the same geographical
# area will be saved together, with all their formats. The filename of the
# tile will be based on the coordinate of the "512px" tile, but the zoom
# level used for styling will be one higher; that of the "256px" tiles.
#
# - 4: A "1024px" tile, 4 "512px" tiles and 16 "256px" tiles covering the
# same area.
size: null
# if specified, the zoom levels to leave off the top of the metatile pyramid.
# for example, if the metatile size is specified at 4 and the start-zoom as 1,
# then the "1024px" tile will be omitted. if start-zoom is 2, then the
# "1024px" tile and all four "512px" tiles will be omitted.
#
# optional: defaults to zero.
start-zoom: 0
# if specificed, the tile sizes to render in each metatile. by default, this
# is all the sizes for the metatile, i.e: for `size=4`, then `tile-sizes`
# will default to [1024, 512, 256].
#
# if you don't need some of the sizes, then setting this to a reduced set
# can make your metatiles smaller. however, with a reduced set of sizes, the
# following special behaviour applies:
#
# - at the maximum nominal zoom (see `tile.max-zoom-with-changes`), the
# metatile will additionally contain all the smaller tiles at the same
# nominal zoom. for example, if `tile-sizes` is [1024], `metatile.size`
# is 4 and the `max-zoom-with-changes` is 16, then the highest zoom
# metatile will be 14/x/y and will contain one 1024px tile, 2x2 512px
# tiles and 4x4 256px tiles all at the `max-zoom-with-changes` nominal
# zoom. this allows programs such as tapalcatl to serve smaller tiles
# for a few zoom levels of "overzoom".
#
# - at zoom zero, the metatile will additionally contain the same size of
# tile at lower nominal and coordinate zooms. for example, if the
# `tile-sizes` is [512, 256] and the metatile size is 8, then in addition
# to the 512px and 256px tiles at nominal zoom 3, the tile will also
# contain 512px tiles at nominal zooms 2 and 1. (because the 512px tile
# at nominal zoom 1 covers the whole world, there is no tile output at
# nominal zoom 0).
#
# optional: default behaviour is all the tile sizes, as described in the
# `size` parameter description.
#
#tile-sizes: [512, 256]
# Configuration for where to store the tiles of interest set
toi-store:
# We support storing the TOI in S3 or as a file
type: <file or s3>
file:
# The name of the file to store the TOI
name: toi.txt.gz
s3:
# The bucket and key for the s3 object that will contain the tiles of interest
bucket: test_bucket
key: toi/toi.txt.gz
# Configuration for the tiles of interest prune/garden command.
toi-prune:
# location of tileserver logs
tile-traffic-log-path: ../tileserver/nohup.out
# Connection and query configuration for a database containing
# request information for tiles.
tile-history:
database-uri: postgresql://user:password@localhost:5439/database
# The number of days of history to query for.
days: 30
# The maximum zoom level to query redshift.
max-zoom: 16
cutoff:
# The minimum number of requests per tile for a tile to be included in
# the tiles of interest
min-requests: 16
# The maximum number of tiles to include from the RedShift-based calculation
max-tiles: 100000
s3:
# Connection information for deleting tiles from S3 when the tile is no longer
# in the tiles of interest.
bucket: mapzen-tiles-dev-us-east
date-prefix: 20170322
path: osm
layer: all
format: zip
# layer and format of tiles to be deleted
store:
layer: all
format: zip
always-include:
# Sets of tiles to always include in the tiles of interest.
# For more information about options here, see the code:
# https://github.com/tilezen/tilequeue/blob/1ca908e/tilequeue/command.py#L1046-L1061
world:
bbox: -180.0,-85.06,180.0,85.06
min_zoom: 0
max_zoom: 10
# sf_metro:
# bbox: -122.524338,37.571794,-122.345123,37.821175
# min_zoom: 0
# max_zoom: 15
# tests:
# tiles: 0/0/0,1/1/0
# tests-file:
# file: list-of-tile-coordinates.txt
# NOTE: the coordinates should be of the same size as the toi, currently 512
# integration-test-coordinates:
# bucket: mapzen-tiles-assets
# key: test/integration-test-coords.txt
rawr:
# this is the parent zoom level that coordinates are expected to be grouped under
group-zoom: 10
queue:
type: sqs
name: <sqs-queue-name>
wait-seconds: 20
region: us-east-1
# alternatively, can use a file-backed queue
#type: file
#input-file: <file containing z/x/y per line>
postgresql:
host: localhost
port: 5432
dbname: dbname
user: dbuser
password: dbpassword
sink:
# type can also be "none" which ignores all writes
type: s3
s3:
bucket: s3-bucket-name
region: us-east-1
prefix: s3-bucket-prefix
extension: zip
# control the s3 tile key for the object
# can either be hash-prefix or prefix-hash
# hash-prefix -> hash comes before the prefix
# prefix-hash -> prefix comes before the hash
key-format-type: <hash-prefix|prefix-hash>
# optional tags that get saved onto the s3 objects
tags:
prefix: 19851026
run_id: 19851026-1
# alternatively, provide a "store" config - same as elsewhere, can be s3 or directory
#store:
# type: directory
# path: rawr_tiles
source:
type: store # type can also be "generate" or "s3"
store:
type: directory
name: rawr_tiles
# add a postgresql section if you're using "generate"
#postgresql:
# host: null
# port: null
# dbname: osm
# user: osm
table-sources:
planet_osm_line: &osm { name: osm, value: openstreetmap.org }
planet_osm_point: *osm
planet_osm_polygon: *osm
planet_osm_ways: *osm
planet_osm_rels: *osm
wof_neighbourhood: { name: wof, value: whosonfirst.org }
water_polygons: &osmdata { name: shp, value: osmdata.openstreetmap.de }
land_polygons: *osmdata
ne_10m_urban_areas: { name: ne, value: naturalearthdata.com }
# when a feature's shape is of the type given in the key and the feature
# appears in the listed layers, then generate a label centroid. multi*
# geometries are considered the same as single ones for the purposes of key
# lookup.
label-placement-layers:
point: ['earth', 'water']
polygon: ['buildings', 'earth', 'landuse', 'water']
linestring: ['earth', 'landuse', 'water']
# indexes to generate on the RAWR tile. features are looked up only if they
# appear in an index, so this list must be exhaustive. all OSM-based features
# are handled by a single index of {type: osm}. other indexes are provided on
# single table/layer combinations with the {type: simple}, providing the
# names of both the table and layer. it's also possible to provide start_zoom
# and end_zoom parameters which control when the data is visible. note that
# the start_zoom is inclusive (and optional; defaults to 0) and the end_zoom
# is exclusive (also optional; defaults to infinity).
indexes:
- type: osm
- type: simple
table: wof_neighbourhood
layer: places
- type: simple
table: water_polygons
layer: water
- type: simple
table: land_polygons
layer: earth
- type: simple
table: ne_10m_urban_areas
layer: landuse
end_zoom: 10
intersect:
# type can be toi, none, all, all-parents
type: toi
# uncomment this to use RAWR tiles rather than go direct to the database.
#use-rawr-tiles: true
batch:
# zoom level to enqueue jobs
queue-zoom: 7
# name of batch job definition
job-definition: <job-definition>
# name of batch queue to use
job-queue: <job-queue>
# optional number of retry attempts, overrides job definition value
retry-attempts: 5
# check store if metatile already exists, and if so skip processing
check-metatile-exists: true
# optional override memory reserved for container when submiting job
# NOTE: in megabytes
# https://docs.aws.amazon.com/batch/latest/APIReference/API_ContainerOverrides.html
memory:
vcpus:
run_id: optional-string-which-gets-logged