Support Account Snapshot Flushes (#154)

* feat: mock snapshots code

* refactor: remove dead code

* feat: getting snapshot stream in the works

* feat: adds snapshot stream

* feat: adds config-ingester settings

* Remove dead config

* Add snapshot config in ingester

---------

Co-authored-by: Kevin Rodriguez <_@kevinrodriguez.io>

grpc-ingest/config-ingester.yml

@@ -1,29 +1,15 @@
 prometheus: 127.0.0.1:8875
-redis:
- url: redis://localhost:6379
- group: ingester
- consumer: consumer # every new ingester instance should have uniq name
-
- streams:
- - type: account
- stream: ACCOUNTS
- - type: transaction
- stream: TRANSACTIONS
- xack_batch_max_size: 100
- xack_batch_max_idle_ms: 10
- xack_max_in_process: 100
- - type: metadatajson
- stream: METADATA_JSON
- prefetch_queue_size: 5_000 # max number of messages available in the read queue for processing
- xpending_max: 1_000 # used for reading pending messages
- xpending_only: false # exit once all pending messages consumed (should be applied if you want downscale number of ingesters)
- xreadgroup_max: 1_000 # applied per each stream in one request
+redis: redis://localhost:6379
 postgres:
  url: postgres://solana:solana@localhost/solana
  min_connections: 10
  max_connections: 50 # `max_connection` should be bigger than `program_transformer.max_tasks_in_process` otherwise unresolved lock is possible
-program_transformer:
- max_tasks_in_process: 100
+snapshots:
+ name: SNAPSHOTS
+ max_concurrency: 10
+ batch_size: 100
+ xack_batch_max_idle_ms: 1_000
+ xack_buffer_size: 10_000
 accounts:
  name: ACCOUNTS
  max_concurrency: 10
@@ -34,5 +20,5 @@ transactions:
  name: TRANSACTIONS
 download_metadata:
  max_attempts: 3
- stream_config:
+ stream:
  name: METADATA_JSON