V1.0.1 (#98)

* Command line flags * Refactored sync to import buffer * New app based node arch Resolved transaction GQL error * Remove stale tx and add quickstart guide
ArweaveTeam · Sep 13, 2021 · 5cab9dd · 5cab9dd
1 parent cd8384f
commit 5cab9dd
Show file tree

Hide file tree

Showing 32 changed files with 639 additions and 282 deletions.
diff --git a/.env.dev b/.env.dev
@@ -1,4 +1,4 @@
-ARWEAVE_NODES=["https://arweave.net"]
+ARWEAVE_NODES=["http://lon-2.eu-west-1.arweave.net:1984","http://lon-4.eu-west-1.arweave.net:1984","http://lon-6.eu-west-1.arweave.net:1984"]
 DATABASE_HOST=0.0.0.0
 DATABASE_PORT=5432
 DATABASE_USER=arweave
@@ -8,15 +8,19 @@ DATABASE_NAME=arweave
 ENVIRONMENT=public
 PORT=3000
 
-PARALLEL=4
+PARALLEL=1
+ANS102=1
 
 DEFAULT_PAGE_SIZE=10
 MAX_PAGE_SIZE=100
 
-INDICES=["App-Name", "app", "domain", "namespace"]
-
 CACHING=1
 CACHE_FOLDER=/gateway/cache
 CACHE_OFFSET=0
 
-MANIFEST_PREFIX=amp-gw.online
+MANIFEST=1
+MANIFEST_PREFIX=amp-gw.online
+
+TYPE=APP
+FILTER=app.filter.json
+START_HEIGHT=764180
diff --git a/.env.docker b/.env.docker
@@ -1,4 +1,4 @@
-ARWEAVE_NODES=["https://arweave.net"]
+ARWEAVE_NODES=["http://lon-2.eu-west-1.arweave.net:1984","http://lon-4.eu-west-1.arweave.net:1984","http://lon-6.eu-west-1.arweave.net:1984"]
 DATABASE_HOST=postgres
 DATABASE_PORT=5432
 DATABASE_USER=arweave
@@ -8,15 +8,19 @@ DATABASE_NAME=arweave
 ENVIRONMENT=public
 PORT=3000
 
-PARALLEL=4
+PARALLEL=1
+ANS102=1
 
 DEFAULT_PAGE_SIZE=10
 MAX_PAGE_SIZE=100
 
-INDICES=["App-Name", "app", "domain", "namespace"]
-
 CACHING=1
 CACHE_FOLDER=/gateway/cache
 CACHE_OFFSET=0
 
-MANIFEST_PREFIX=amp-gw.online
+MANIFEST=1
+MANIFEST_PREFIX=amp-gw.online
+
+TYPE=APP
+FILTER=app.filter.json
+START_HEIGHT=764180
diff --git a/.gitignore b/.gitignore
@@ -16,4 +16,7 @@ snapshot
 **/Thumbs.db
 
 # Webstorm
-.idea
+.idea
+
+# App Filters
+app.filter.json
diff --git a/QUICKSTART.md b/QUICKSTART.md
@@ -0,0 +1,68 @@
+# Quickstart with App Nodes
+
+## Configuring your environment
+
+In your `.env` file make sure to have the following configured:
+
+```conf
+TYPE=APP # This makes sure it's configured to be an app node
+```
+
+```conf
+FILTER=app.filter.json # The path to the app filter json file
+```
+
+```conf
+START_HEIGHT=764189 # The block (-1) from where your contract was deployed at
+```
+
+Make sure `START_HEIGHT` is atleast one block before where the contract was deployed.
+
+## Creating a filter file
+
+Your `app.filter.json` file needs to filter for both the smart contract deployment and the smart contract source. You can filter for specific ids by using the `id` key.
+
+```json
+{
+    "filter": [
+        {
+            "id": "boJ3Fa1OU9W1NY5g1dkmgqYk5Lg_mndcdC9q64CmDPU"
+        },
+        {
+            "id": "aZrQ9fNp1fdKqBsdZKVHFF4NZRx-icDGfAncRw4zGpY"
+        }
+    ]
+}
+```
+
+You will also need to filter for the Contract tag with that ID. You can do so by adding a tag filter.
+
+```json
+{
+    "filter": [
+        {
+            "name": "Contract",
+            "value": "boJ3Fa1OU9W1NY5g1dkmgqYk5Lg_mndcdC9q64CmDPU"
+        }
+    ]
+}
+```
+
+See `app.filter.dev.json` for reference of how to setup a filter file. Update the values with your smart contract info.
+
+## Filtering additional tags
+
+You can also filter for other tags too. Just add a new `filter` object to the filter array.
+
+```json
+{
+    "filter": [
+        {
+            "name": "AR-Tag-Key",
+            "value": "Any-Value"
+        }
+    ]
+}
+```
+
+Adding multiple `name, value` objects to the filter acts as an `AND` operator. Creating new `filter` objects act as an `OR` operator when filtering for transactions.
diff --git a/README.md b/README.md
@@ -12,22 +12,16 @@ Review the [documentation](https://arweaveteam.github.io/gateway/#/) to learn mo
 
 2. Docker and Docker Compose LTS
 
-### Suggested Hardware
+## Quickstart with App Nodes
 
-There are several million transactions on the Arweave chain. In order to effectively serve content on the gateway you'll need a decent sized computer. The ideal specs for a Gateway should have the following:
-
-1. 16GB RAM (ideally 32GB RAM)
-
-2. ~1TB of SSD storage available
-
-3. Intel i5 / AMD FX or greater, +4 vCPUs should be more than enough, these are typically Intel Xeon CPUs.
+To get started with the new app nodes. Please read the [Quick Start Guide](./QUICKSTART.md). It goes over how to configure the environment and write filters for application specific needs.
 
 ## Environment
 
 By default, there is a default environment you can use located at `.env.docker` in the repository.
 
 ```env
-ARWEAVE_NODES=["https://arweave.net"]
+ARWEAVE_NODES=["http://lon-2.eu-west-1.arweave.net:1984","http://lon-4.eu-west-1.arweave.net:1984","http://lon-6.eu-west-1.arweave.net:1984"]
 
 DATABASE_HOST=postgres
 DATABASE_PORT=5432
@@ -38,24 +32,30 @@ DATABASE_NAME=arweave
 ENVIRONMENT=public
 PORT=3000
 
-PARALLEL=4
-
-INDICES=["App-Name", "app", "domain", "namespace"]
+PARALLEL=1
+ANS102=1
 
 CACHING=1
 CACHE_FOLDER=/gateway/cache
 CACHE_OFFSET=0
 
+MANIFEST=1
 MANIFEST_PREFIX=amp-gw.online
+
+TYPE=APP
+FILTER=app.filter.json
+START_HEIGHT=764180
 ```
 
 Make sure you copy this configuration to `.env`.
 
 ```bash
-cp .env.docker .env
+cp .env.dev .env
 ```
 
-## Compilation
+You should also update the `ARWEAVE_NODES` to valid
+
+## Running the server
 
 You can start the server with `docker-compose`.
 

diff --git a/app.filter.dev.json b/app.filter.dev.json
@@ -0,0 +1,20 @@
+[
+    {
+        "filter": [
+            {
+                "name": "Contract",
+                "value": "boJ3Fa1OU9W1NY5g1dkmgqYk5Lg_mndcdC9q64CmDPU"
+            }
+        ]
+    },
+    {
+        "filter": [
+            {
+                "id": "boJ3Fa1OU9W1NY5g1dkmgqYk5Lg_mndcdC9q64CmDPU"
+            },
+            {
+                "id": "aZrQ9fNp1fdKqBsdZKVHFF4NZRx-icDGfAncRw4zGpY"
+            }
+        ]
+    }
+]
diff --git a/bin/index.create.sh b/bin/index.create.sh
@@ -26,14 +26,6 @@ CREATE INDEX "transactions_height" ON transactions USING BTREE ("height");
 CREATE INDEX "transactions_target" ON transactions USING BTREE ("target");
 --- Transaction Owner Address Index
 CREATE INDEX "transactions_owner_address" ON transactions USING BTREE ("owner_address");
---- Transaction Namespace Index
-CREATE INDEX "index_namespace_transactions" ON transactions USING BTREE ("namespace");
---- Transaction Domain Index
-CREATE INDEX "index_domain_transactions" ON transactions USING BTREE ("domain");
---- Transaction App Index
-CREATE INDEX "index_app_transactions" ON transactions USING BTREE ("app");
---- Transaction App-Name Index
-CREATE INDEX "index_App-Name_transactions" ON transactions USING BTREE ("App-Name");
 --- Transactions created_at index
 CREATE INDEX "transactions_created_at" ON transactions USING BTREE ("created_at");
 
@@ -58,8 +50,6 @@ CREATE INDEX "tags_name_value_128" ON tags USING BTREE ("name", "value") WHERE L
 CREATE INDEX "tags_tx_id_name_128" ON tags USING BTREE ("tx_id", "name") WHERE LENGTH("name") > 64 AND LENGTH("name") < 128;
 --- Tag created_at
 CREATE INDEX "tags_created_at" ON tags USING BTREE ("created_at");
-CREATE INDEX "tags_created_at_64" ON tags USING BTREE ("created_at") WHERE LENGTH("value") < 64;
-CREATE INDEX "tags_created_at_128" ON tags USING BTREE ("created_at") WHERE LENGTH("value") < 128;
 
 
 EOF

diff --git a/docs/DEV.md b/docs/DEV.md
@@ -44,7 +44,7 @@ exit
 By default, there is a development environment you can use located at `.env.dev` in the repository. This `.dev` environment is different to the `.env.docker` environment which is designed for `docker` usage.
 
 ```env
-ARWEAVE_NODES=["https://arweave.net"]
+ARWEAVE_NODES=["http://lon-2.eu-west-1.arweave.net:1984","http://lon-4.eu-west-1.arweave.net:1984","http://lon-6.eu-west-1.arweave.net:1984"]
 
 DATABASE_HOST=0.0.0.0
 DATABASE_PORT=5432
@@ -55,18 +55,22 @@ DATABASE_NAME=arweave
 ENVIRONMENT=public
 PORT=3000
 
-PARALLEL=4
+PARALLEL=1
+ANS102=1
 
 DEFAULT_PAGE_SIZE=10
 MAX_PAGE_SIZE=100
 
-INDICES=["App-Name", "app", "domain", "namespace"]
-
 CACHING=1
 CACHE_FOLDER=/gateway/cache
 CACHE_OFFSET=0
 
+MANIFEST=1
 MANIFEST_PREFIX=amp-gw.online
+
+TYPE=APP
+FILTER=app.filter.json
+START_HEIGHT=764180
 ```
 
 Make sure you copy this configuration to `.env`.

diff --git a/docs/MANIFEST.md b/docs/MANIFEST.md
@@ -4,6 +4,20 @@ Transaction manifests are only usable with a valid domain. Please make sure to h
 
 In order to effectively use manifests. You need to have domain wildcards pointing to your domain.
 
+### Enabling Manifests
+
+Make sure that the environment variable `MANIFEST` is set to `1`.
+
+```conf
+MANIFEST=1
+```
+
+To disable manifests (useful for non TLD gateways), run:
+
+```conf
+MANIFEST=0
+```
+
 ### Configuring Manifests
 
 In order to configure manifests, you need to change the `MANIFEST_PREFIX` environment variable. It should just be your domain name. Simply change it from:

diff --git a/docs/README.md b/docs/README.md
@@ -27,7 +27,7 @@ There are several million transactions on the Arweave chain. In order to effecti
 By default, there is a default environment you can use located at `.env.docker` in the repository.
 
 ```env
-ARWEAVE_NODES=["https://arweave.net"]
+ARWEAVE_NODES=["http://lon-2.eu-west-1.arweave.net:1984","http://lon-4.eu-west-1.arweave.net:1984","http://lon-6.eu-west-1.arweave.net:1984"]
 
 DATABASE_HOST=postgres
 DATABASE_PORT=5432
@@ -40,18 +40,22 @@ MANIFESTS=0
 BIP39=0
 PORT=3000
 
-PARALLEL=4
+PARALLEL=1
+ANS102=1
 
 DEFAULT_PAGE_SIZE=10
 MAX_PAGE_SIZE=100
 
-INDICES=["App-Name", "app", "domain", "namespace"]
-
 CACHING=1
 CACHE_FOLDER=/gateway/cache
 CACHE_OFFSET=0
 
+MANIFEST=1
 MANIFEST_PREFIX=amp-gw.online
+
+TYPE=APP
+FILTER=app.filter.json
+START_HEIGHT=764180
 ```
 
 Make sure you copy this configuration to `.env`.

diff --git a/migrations/20200404025828_initialize.ts b/migrations/20200404025828_initialize.ts
@@ -4,9 +4,6 @@ import {config} from 'dotenv';
 config();
 
 export async function up(knex: Knex) {
-  const indices = JSON.parse(process.env.INDICES || '[]');
-
-
   return knex.schema
       .withSchema(process.env.ENVIRONMENT || 'public')
       .createTable('transactions', (table) => {
@@ -22,16 +19,12 @@ export async function up(knex: Knex) {
         table.string('content_type');
         table.integer('format', 2);
         table.integer('height', 4);
-        table.string('owner_address');
-        table.string('data_root', 64);
-        table.string('parent', 64);
+        table.integer('precache_height', 4);
+        table.text('owner_address');
+        table.text('data_root');
+        table.text('parent');
         table.timestamp('created_at').defaultTo(knex.fn.now());
 
-        for (let i = 0; i < indices.length; i++) {
-          const index = indices[i];
-          table.string(index, 64);
-        }
-
         table.primary(['id'], 'pkey_transactions');
       })
       .createTable('blocks', (table) => {

diff --git a/package.json b/package.json
@@ -39,6 +39,7 @@
     "detect-mocha": "^0.1.0",
     "dotenv": "^8.2.0",
     "event-stream": "^4.0.1",
+    "execa": "^5.1.1",
     "express": "^4.17.1",
     "express-pg-session": "^1.1.0",
     "express-session": "^1.17.1",