Merge pull request #246 from ardriveapp/PE-480_gateway_flag

PE-480: Custom gateway flag

README.md

@@ -126,6 +126,7 @@ ardrive upload-file --wallet-file /path/to/my/wallet.json --parent-folder-id "f0
         3. [Check for network congestion before uploading](#check-congestion)
         4. [Front-run Congestion By Boosting Miner Rewards](#boost)
         5. [Send AR Transactions From a Cold Wallet](#cold-tx)
+        6. [Using a Custom Arweave Gateway](#using-a-custom-arweave-gateway)
 4. [All ArDrive CLI Commands](#all-ardrive-cli-commands)
 5. [Getting Help](#getting-help)
 
@@ -1177,6 +1178,41 @@ Transport your signed transaction to the Internet-connected machine and run the
 ardrive send-tx -x /path/to/sendme.json
 ```
 
+### Using a Custom Arweave Gateway
+
+On each command that uses a gateway, it is possible to supply your own custom Arweave gateway using the flag `--gateway` or by setting an environment variable named `ARWEAVE_GATEWAY`.
+
+For example, you could test out that your ArFS transactions are working as expected on a local test network such as [ArLocal] with this flow:
+
+```shell
+# Setup ArLocal instance on port 1984
+npx arlocal
+
+# In another terminal, fund your wallet with AR
+curl http://localhost:1984/mint/{ your public wallet address }/99999999999999
+
+# Create drive and root folder on ArLocal using `--gateway` flag
+ardrive create-drive --gateway http://localhost:1984 -w /path/to/wallet -n 'my-test-drive'
+
+# Setup ARWEAVE_GATEWAY as ENV variable
+export ARWEAVE_GATEWAY="http://localhost:1984"
+
+# Mine block with drive + root folder transactions
+curl "$ARWEAVE_GATEWAY/mine"
+
+# Upload file to ArLocal with ENV var
+ardrive upload-file -F { root folder id from create drive } -l /path/to/file -w /path/to/wallet
+
+# Mine block with file transaction
+curl "$ARWEAVE_GATEWAY/mine"
+
+# Inspect meta data of created entities
+ardrive list-drive -d { drive id from create drive }
+
+# Download file to verify integrity
+ardrive download-file -f { file id from upload file }
+```
+
 # All ArDrive CLI Commands
 
 ```shell
@@ -1269,4 +1305,5 @@ ardrive <command> --help
 [kb-wallets]: https://ardrive.atlassian.net/l/c/FpK8FuoQ
 [arweave-manifests]: https://github.com/ArweaveTeam/arweave/wiki/Path-Manifests
 [example-manifest-webpage]: https://arweave.net/qozq9YIUPEHfZhoTp9DkBpJuA_KNULBnfLiMroj5pZI
+[arlocal]: https://github.com/textury/arlocal
 [mozilla-mime-types]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types

bats_test/base-reward/base-reward.bats

@@ -9,3 +9,8 @@ load '/home/node/packages/node_modules/bats-assert/load.bash'
     run -0 yarn ardrive base-reward
     assert_output --regexp '^[0-9]+$'
 }
+
+@test "'ardrive base-reward --gateway' will error with an invalid Arweave gateway" {
+    run -1 yarn ardrive base-reward --gateway http://localhost:1337
+    assert_line -n 0 'Error: connect ECONNREFUSED 127.0.0.1:1337'
+}

bats_test/get-mempool/get-mempool.bats

@@ -11,6 +11,11 @@ load '/home/node/packages/node_modules/bats-assert/load.bash'
     assert_output --regexp '^[0-9]{1,4}'
 }
 
+@test "'ardrive get-mempool --gateway' will error with an invalid Arweave gateway" {
+    run -1 yarn ardrive get-mempool --gateway http://localhost:1337
+    assert_line -n 0 'Error: connect ECONNREFUSED 127.0.0.1:1337'
+}
+
 @test "'ardrive get-mempool' first line contains a valid TX" {
     # Needs wrapping for piping
     run yarn ardrive get-mempool

bats_test/help/base-reward.bats

@@ -9,20 +9,26 @@ load '/home/node/packages/node_modules/bats-assert/load.bash'
     run -0 yarn ardrive base-reward -h
     assert_line --index 0 'Usage: ardrive base-reward [options]'
     assert_line --index 1 'Options:'
-    assert_line -n 2 '  --boost <boost>  (OPTIONAL) a multiple of the base transaction mining reward'
-    assert_line --index 3 '                   that can be used to accelerate transaction mining. A'
-    assert_line --index 4 '                   multiple of 2.5 would boost a 100 Winston transaction reward'
-    assert_line --index 5 '                   to 250 Winston.'
-    assert_line -n 6 '  -h, --help       display help for command'
+    assert_line -n 2 '  --boost <boost>         (OPTIONAL) a multiple of the base transaction mining'
+    assert_line --index 3 '                          reward that can be used to accelerate transaction'
+    assert_line --index 4 '                          mining. A multiple of 2.5 would boost a 100 Winston'
+    assert_line --index 5 '                          transaction reward to 250 Winston.'
+    assert_line -n 6 "  -g --gateway <gateway>  (OPTIONAL) a 'protocol://host:port' formatted string"
+    assert_line -n 7 "                          specifying the connection info for the Arweave"
+    assert_line -n 8 "                          gateway server to use"
+    assert_line -n 9 '  -h, --help              display help for command'
 }
 
 @test "'ardrive base-reward --help' prints help with all expected fields" {
     run -0 yarn ardrive base-reward --help
     assert_line --index 0 'Usage: ardrive base-reward [options]'
     assert_line --index 1 'Options:'
-    assert_line -n 2 '  --boost <boost>  (OPTIONAL) a multiple of the base transaction mining reward'
-    assert_line --index 3 '                   that can be used to accelerate transaction mining. A'
-    assert_line --index 4 '                   multiple of 2.5 would boost a 100 Winston transaction reward'
-    assert_line --index 5 '                   to 250 Winston.'
-    assert_line -n 6 '  -h, --help       display help for command'
+    assert_line -n 2 '  --boost <boost>         (OPTIONAL) a multiple of the base transaction mining'
+    assert_line --index 3 '                          reward that can be used to accelerate transaction'
+    assert_line --index 4 '                          mining. A multiple of 2.5 would boost a 100 Winston'
+    assert_line --index 5 '                          transaction reward to 250 Winston.'
+    assert_line -n 6 "  -g --gateway <gateway>  (OPTIONAL) a 'protocol://host:port' formatted string"
+    assert_line -n 7 "                          specifying the connection info for the Arweave"
+    assert_line -n 8 "                          gateway server to use"
+    assert_line -n 9 '  -h, --help              display help for command'
 }

bats_test/help/get-mempool.bats

@@ -9,12 +9,18 @@ load '/home/node/packages/node_modules/bats-assert/load.bash'
     run -0 yarn ardrive get-mempool -h
     assert_line --index 0 'Usage: ardrive get-mempool [options]'
     assert_line --index 1 'Options:'
-    assert_line -n 2 '  -h, --help  display help for command'
+    assert_line -n 2 "  -g --gateway <gateway>  (OPTIONAL) a 'protocol://host:port' formatted string"
+    assert_line -n 3 "                          specifying the connection info for the Arweave"
+    assert_line -n 4 "                          gateway server to use"
+    assert_line -n 5 '  -h, --help              display help for command'
 }
 
 @test "'ardrive get-mempool --help' prints help with all expected fields" {
     run -0 yarn ardrive get-mempool --help
     assert_line --index 0 'Usage: ardrive get-mempool [options]'
     assert_line --index 1 'Options:'
-    assert_line -n 2 '  -h, --help  display help for command'
+    assert_line -n 2 "  -g --gateway <gateway>  (OPTIONAL) a 'protocol://host:port' formatted string"
+    assert_line -n 3 "                          specifying the connection info for the Arweave"
+    assert_line -n 4 "                          gateway server to use"
+    assert_line -n 5 '  -h, --help              display help for command'
 }

bats_test/help/no_command.bats

@@ -23,7 +23,7 @@ Commands:
   get-balance [options]
   get-drive-key [options]
   get-file-key [options]
-  get-mempool
+  get-mempool [options]
   last-tx [options]
   list-all-drives [options]
   list-drive [options]

folder.json

@@ -0,0 +1 @@
+Error: Request failed with status code 504

package.json

@@ -8,7 +8,7 @@
 	},
 	"types": "./lib/index.d.ts",
 	"dependencies": {
-		"ardrive-core-js": "1.8.0",
+		"ardrive-core-js": "1.9.0",
 		"arweave": "1.10.18",
 		"axios": "^0.21.1",
 		"commander": "^8.2.0",

src/CLICommand/parameters_helper.test.ts

@@ -24,6 +24,7 @@ import {
 	WalletFileParameter,
 	MaxDepthParameter,
 	AllParameter,
+	GatewayParameter,
 	DryRunParameter
 } from '../parameter_declarations';
 import '../parameter_declarations';
@@ -549,6 +550,114 @@ describe('ParametersHelper class', () => {
 		});
 	});
 
+	describe('getGateway method', () => {
+		it('returns the expected URLs', async () => {
+			const cmd = declareCommandWithParams(program, [GatewayParameter]);
+
+			const validUrlTests: [string, { protocol: string; hostName: string; port: string }][] = [
+				['http://arweave.net', { hostName: 'arweave.net', protocol: 'http:', port: '' }],
+				['http://arweave.net:80', { hostName: 'arweave.net', protocol: 'http:', port: '' }], // http uses default port 80
+				['http://arweave.net:443', { hostName: 'arweave.net', protocol: 'http:', port: '443' }],
+				['weirdProtocol://arweave.net:21', { hostName: 'arweave.net', protocol: 'weirdprotocol:', port: '21' }],
+				['https://arweave.net:443', { hostName: 'arweave.net', protocol: 'https:', port: '' }], // https uses default port 443
+				['https://arweave.net:101', { hostName: 'arweave.net', protocol: 'https:', port: '101' }],
+				['https://arweave.net:80', { hostName: 'arweave.net', protocol: 'https:', port: '80' }],
+				[
+					'https://port-zero-is-fine.net:0',
+					{ hostName: 'port-zero-is-fine.net', protocol: 'https:', port: '0' }
+				],
+				['https://max-port-limit:65535', { hostName: 'max-port-limit', protocol: 'https:', port: '65535' }],
+				['folder1://testIt.Com:9', { hostName: 'testIt.Com', protocol: 'folder1:', port: '9' }],
+				[
+					'http://removes-leading-zeroes-on-port:009',
+					{ hostName: 'removes-leading-zeroes-on-port', protocol: 'http:', port: '9' }
+				],
+				[
+					'file://file-protocol-no-port.ok',
+					{ hostName: 'file-protocol-no-port.ok', protocol: 'file:', port: '' }
+				],
+				['http://no-dot-com-is-fine', { hostName: 'no-dot-com-is-fine', protocol: 'http:', port: '' }],
+				[
+					'ar://gr8.emoji.🚀.com:1337',
+					{ hostName: 'gr8.emoji.%F0%9F%9A%80.com', protocol: 'ar:', port: '1337' }
+				]
+			];
+
+			for (const [
+				testUrlString,
+				{ hostName: expectedHostName, port: expectedPort, protocol: expectedProtocol }
+			] of validUrlTests) {
+				CLICommand.parse(program, [...baseArgv, testCommandName, '--gateway', testUrlString]);
+				await cmd.action.then((options) => {
+					const parameters = new ParametersHelper(options);
+					const gateway = parameters.getGateway();
+
+					expect(gateway.hostname).to.equal(expectedHostName);
+					expect(gateway.port).to.equal(expectedPort);
+					expect(gateway.protocol).to.equal(expectedProtocol);
+				});
+			}
+		});
+
+		it('throws on invalid URLs during the URL class constructor', async () => {
+			const cmd = declareCommandWithParams(program, [GatewayParameter]);
+
+			const invalidUrlTests: string[] = [
+				// Throws on hosts without a protocol
+				'arweave.net',
+				'no-protocol.com',
+				// Throws on ports using invalid characters; ports must use integers
+				'https://bad-port.hello:INVALID',
+				'https://arweave.net:443B',
+				'http://arweave.net:Nope',
+				// Throws above the max port number limit, which is an 16-bit integer -- so 65535 is the max limit
+				'https://arweave.net:1000000000',
+				'https://arweave.net:65536',
+				// Throws on file protocols with a port (Special case)
+				'file://testit.com:92',
+				'file://testit.com:101',
+				// Throws on (`:`) in hostName
+				'http://ar:weave.net',
+				// Throws on (` `) in hostName
+				'http://ar weave.net'
+			];
+
+			for (const testUrlString of invalidUrlTests) {
+				CLICommand.parse(program, [...baseArgv, testCommandName, '--gateway', testUrlString]);
+
+				await cmd.action.then((options) => {
+					const parameters = new ParametersHelper(options);
+
+					expect(() => parameters.getGateway()).to.throw(TypeError, `Invalid URL: ${testUrlString}`);
+				});
+			}
+		});
+
+		it('throws on user provided URLs where the hostName property cannot be determined', async () => {
+			const cmd = declareCommandWithParams(program, [GatewayParameter]);
+
+			const invalidHostNameTests: string[] = [
+				'arweave.net:443',
+				'arweave.net:80',
+				'strange:words:??',
+				'there:are:so:many:colons:here:for:this:test'
+			];
+
+			for (const testUrlString of invalidHostNameTests) {
+				CLICommand.parse(program, [...baseArgv, testCommandName, '--gateway', testUrlString]);
+
+				await cmd.action.then((options) => {
+					const parameters = new ParametersHelper(options);
+
+					expect(() => parameters.getGateway()).to.throw(
+						TypeError,
+						`Host name could not be determined from provided URL: ${testUrlString}`
+					);
+				});
+			}
+		});
+	});
+
 	describe('isDryRun', () => {
 		it('returns true when the parameter is specified', () => {
 			const cmd = declareCommandWithParams(program, [DryRunParameter]);

src/CLICommand/parameters_helper.ts

@@ -13,6 +13,7 @@ import {
 	AskParameter,
 	SkipParameter,
 	BoostParameter,
+	GatewayParameter,
 	DryRunParameter
 } from '../parameter_declarations';
 import { cliWalletDao } from '..';
@@ -40,6 +41,9 @@ import { JWKInterface } from 'arweave/node/lib/wallet';
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 type ParameterOptions = any;
 
+const DEFAULT_GATEWAY = 'https://arweave.net:443';
+const ARWEAVE_GATEWAY_ENV_VAR = 'ARWEAVE_GATEWAY';
+
 interface GetDriveKeyParams {
 	driveId: DriveID;
 	drivePassword?: string;
@@ -277,6 +281,42 @@ export class ParametersHelper {
 		return mapFunc(value);
 	}
 
+	/**
+	 * Gathers a valid gateway URL from user provided gateway parameter,
+	 * an environment variable, or returns the default arweave gateway
+	 *
+	 * @throws on user provided gateways that are incompatible with URL class constructor
+	 * @throws when hostName cannot be derived from a user provided gateway
+	 */
+	public getGateway(): URL {
+		const userProvidedURL = (() => {
+			// Use optional --gateway supplied parameter as first choice
+			const gatewayFromParam = this.getParameterValue(GatewayParameter);
+			if (gatewayFromParam) {
+				return new URL(gatewayFromParam);
+			}
+
+			// Then check for an ENV provided gateway
+			const envGateway = process.env[ARWEAVE_GATEWAY_ENV_VAR];
+			if (envGateway) {
+				return new URL(envGateway);
+			}
+
+			return undefined;
+		})();
+
+		if (!userProvidedURL) {
+			// Return default CLI Arweave if no gateway can be derived from the user
+			return new URL(DEFAULT_GATEWAY);
+		}
+
+		if (userProvidedURL.hostname === '') {
+			// Ensure a valid host name was provided to be used in Arweave.init()
+			throw new TypeError(`Host name could not be determined from provided URL: ${userProvidedURL.href}`);
+		}
+		return userProvidedURL;
+	}
+
 	public isDryRun(): boolean {
 		const dryRun = this.getParameterValue(DryRunParameter);
 		return !!dryRun;

src/commands/base_reward.ts

@@ -2,20 +2,21 @@ import { ByteCount } from 'ardrive-core-js';
 import { CLICommand, ParametersHelper } from '../CLICommand';
 import { CLIAction } from '../CLICommand/action';
 import { SUCCESS_EXIT_CODE } from '../CLICommand/error_codes';
-import { BoostParameter } from '../parameter_declarations';
+import { BoostParameter, GatewayParameter } from '../parameter_declarations';
 import axios, { AxiosResponse } from 'axios';
 
-async function getBaseReward(byteCount?: ByteCount): Promise<string> {
-	const response: AxiosResponse = await axios.get(`https://arweave.net/price/${byteCount ?? 0}`);
+async function getBaseReward(gateway: URL, byteCount?: ByteCount): Promise<string> {
+	const response: AxiosResponse = await axios.get(`${gateway.href}price/${byteCount ?? 0}`);
 	return `${response.data}`;
 }
 
 new CLICommand({
 	name: 'base-reward',
-	parameters: [BoostParameter],
+	parameters: [BoostParameter, GatewayParameter],
 	action: new CLIAction(async function action(options) {
 		const parameters = new ParametersHelper(options);
-		let baseRewardStr = await getBaseReward();
+		const gateway = parameters.getGateway();
+		let baseRewardStr = await getBaseReward(gateway);
 		const multiple = parameters.getOptionalBoostSetting();
 		if (multiple) {
 			baseRewardStr = multiple.boostReward(baseRewardStr);