Applies multinodessuite in twonodessuite (#1031)

* Sets timeout for codexclient httpClient. Adds reliable transfer test.

* disable new test to check timeout setting in CI

* restores new test

* adds heartbeat log and logfile to ci output

* fixes suite

* fixes blocked output stream by switching to multinode fixture

* new twonodessuite based on multinodesuite

* Applies updated twonodessuite

* removes heartbeat log

* applies multinodesuite in testsales

* applies multinodesuite in testmarketplace

* fixes account fetch for host and client in testmarketplace

* adds waitTillNextPeriod at end of marketplace test

* Uses marketplacesuite in testmarketplace

tests/integration/codexclient.nim

@@ -20,9 +20,11 @@ type CodexClient* = ref object
 
 type CodexClientError* = object of CatchableError
 
+const HttpClientTimeoutMs = 60 * 1000
+
 proc new*(_: type CodexClient, baseurl: string): CodexClient =
   CodexClient(
-    http: newHttpClient(),
+    http: newHttpClient(timeout=HttpClientTimeoutMs),
     baseurl: baseurl,
     session: HttpSessionRef.new({HttpClientFlag.Http11Pipeline})
   )
@@ -247,7 +249,7 @@ proc close*(client: CodexClient) =
 
 proc restart*(client: CodexClient) =
   client.http.close()
-  client.http = newHttpClient()
+  client.http = newHttpClient(timeout=HttpClientTimeoutMs)
 
 proc purchaseStateIs*(client: CodexClient, id: PurchaseId, state: string): bool =
   client.getPurchase(id).option.?state == some state

tests/integration/multinodes.nim

@@ -162,6 +162,8 @@ template multinodesuite*(name: string, body: untyped) =
           let updatedLogFile = getLogFile(role, some roleIdx)
           config.withLogFile(updatedLogFile)
 
+        if bootstrap.len > 0:
+          config.addCliOption("--bootstrap-node", bootstrap)
         config.addCliOption("--api-port", $ await nextFreePort(8080 + nodeIdx))
         config.addCliOption("--data-dir", datadir)
         config.addCliOption("--nat", "127.0.0.1")
@@ -223,7 +225,6 @@ template multinodesuite*(name: string, body: untyped) =
     proc startProviderNode(conf: CodexConfig): Future[NodeProcess] {.async.} =
       let providerIdx = providers().len
       var config = conf
-      config.addCliOption("--bootstrap-node", bootstrap)
       config.addCliOption(StartUpCmd.persistence, "--eth-provider", jsonRpcProviderUrl)
       config.addCliOption(StartUpCmd.persistence, "--eth-account", $accounts[running.len])
       config.addCliOption(PersistenceCmd.prover, "--circom-r1cs",
@@ -238,7 +239,6 @@ template multinodesuite*(name: string, body: untyped) =
     proc startValidatorNode(conf: CodexConfig): Future[NodeProcess] {.async.} =
       let validatorIdx = validators().len
       var config = conf
-      config.addCliOption("--bootstrap-node", bootstrap)
       config.addCliOption(StartUpCmd.persistence, "--eth-provider", jsonRpcProviderUrl)
       config.addCliOption(StartUpCmd.persistence, "--eth-account", $accounts[running.len])
       config.addCliOption(StartUpCmd.persistence, "--validator")
@@ -311,7 +311,7 @@ template multinodesuite*(name: string, body: untyped) =
                           role: Role.Client,
                           node: node
                         )
-            if clients().len == 1:
+            if running.len == 1:
               without ninfo =? CodexProcess(node).client.info():
                 # raise CatchableError instead of Defect (with .get or !) so we
                 # can gracefully shutdown and prevent zombies

tests/integration/nodeprocess.nim

@@ -150,17 +150,20 @@ method stop*(node: NodeProcess) {.base, async.} =
 
     trace "node stopped"
 
-proc waitUntilStarted*(node: NodeProcess) {.async.} =
+proc waitUntilOutput*(node: NodeProcess, output: string) {.async.} =
   logScope:
     nodeName = node.name
 
-  trace "waiting until node started"
+  trace "waiting until", output
 
   let started = newFuture[void]()
+  let fut = node.captureOutput(output, started).track(node)
+  asyncSpawn fut
+  await started.wait(60.seconds) # allow enough time for proof generation
+
+proc waitUntilStarted*(node: NodeProcess) {.async.} =
   try:
-    let fut = node.captureOutput(node.startedOutput, started).track(node)
-    asyncSpawn fut
-    await started.wait(60.seconds) # allow enough time for proof generation
+    await node.waitUntilOutput(node.startedOutput)
     trace "node started"
   except AsyncTimeoutError:
     # attempt graceful shutdown in case node was partially started, prevent

tests/integration/testblockexpiration.nim

@@ -5,10 +5,11 @@ from std/net import TimeoutError
 
 import pkg/chronos
 import ../ethertest
-import ./nodes
+import ./codexprocess
+import ./nodeprocess
 
 ethersuite "Node block expiration tests":
-  var node: NodeProcess
+  var node: CodexProcess
   var baseurl: string
 
   let dataDir = getTempDir() / "Codex1"
@@ -18,12 +19,12 @@ ethersuite "Node block expiration tests":
     baseurl = "http://localhost:8080/api/codex/v1"
 
   teardown:
-    node.stop()
+    await node.stop()
 
     dataDir.removeDir()
 
-  proc startTestNode(blockTtlSeconds: int) =
-    node = startNode([
+  proc startTestNode(blockTtlSeconds: int) {.async.} =
+    node = await CodexProcess.startNode(@[
       "--api-port=8080",
       "--data-dir=" & dataDir,
       "--nat=127.0.0.1",
@@ -32,9 +33,11 @@ ethersuite "Node block expiration tests":
       "--disc-port=8090",
       "--block-ttl=" & $blockTtlSeconds,
       "--block-mi=1",
-      "--block-mn=10"
-    ], debug = false)
-    node.waitUntilStarted()
+      "--block-mn=10"],
+      false,
+      "cli-test-node"
+    )
+    await node.waitUntilStarted()
 
   proc uploadTestFile(): string =
     let client = newHttpClient()
@@ -61,7 +64,7 @@ ethersuite "Node block expiration tests":
     content.code == Http200
 
   test "node retains not-expired file":
-    startTestNode(blockTtlSeconds = 10)
+    await startTestNode(blockTtlSeconds = 10)
 
     let contentId = uploadTestFile()
 
@@ -74,7 +77,7 @@ ethersuite "Node block expiration tests":
       response.body == content
 
   test "node deletes expired file":
-    startTestNode(blockTtlSeconds = 1)
+    await startTestNode(blockTtlSeconds = 1)
 
     let contentId = uploadTestFile()
 

tests/integration/testcli.nim

@@ -1,57 +1,66 @@
-import std/unittest
 import std/tempfiles
 import codex/conf
 import codex/utils/fileutils
-import ./nodes
+import ../asynctest
+import ../checktest
+import ./codexprocess
+import ./nodeprocess
 import ../examples
 
-suite "Command line interface":
+asyncchecksuite "Command line interface":
 
   let key = "4242424242424242424242424242424242424242424242424242424242424242"
 
+  proc startCodex(args: seq[string]): Future[CodexProcess] {.async.} =
+    return await CodexProcess.startNode(
+      args,
+      false,
+      "cli-test-node"
+    )
+
   test "complains when persistence is enabled without ethereum account":
-    let node = startNode(@[
+    let node = await startCodex(@[
       "persistence"
     ])
-    node.waitUntilOutput("Persistence enabled, but no Ethereum account was set")
-    node.stop()
+    await node.waitUntilOutput("Persistence enabled, but no Ethereum account was set")
+    await node.stop()
 
   test "complains when ethereum private key file has wrong permissions":
     let unsafeKeyFile = genTempPath("", "")
     discard unsafeKeyFile.writeFile(key, 0o666)
-    let node = startNode(@[
+    let node = await startCodex(@[
       "persistence",
       "--eth-private-key=" & unsafeKeyFile])
-    node.waitUntilOutput("Ethereum private key file does not have safe file permissions")
-    node.stop()
+    await node.waitUntilOutput("Ethereum private key file does not have safe file permissions")
+    await node.stop()
     discard removeFile(unsafeKeyFile)
 
   let
     marketplaceArg = "--marketplace-address=" & $EthAddress.example
     expectedDownloadInstruction = "Proving circuit files are not found. Please run the following to download them:"
 
   test "suggests downloading of circuit files when persistence is enabled without accessible r1cs file":
-    let node = startNode(@["persistence", "prover", marketplaceArg])
-    node.waitUntilOutput(expectedDownloadInstruction)
-    node.stop()
+    let node = await startCodex(@["persistence", "prover", marketplaceArg])
+    await node.waitUntilOutput(expectedDownloadInstruction)
+    await node.stop()
 
   test "suggests downloading of circuit files when persistence is enabled without accessible wasm file":
-    let node = startNode(@[
+    let node = await startCodex(@[
       "persistence",
       "prover",
       marketplaceArg,
       "--circom-r1cs=tests/circuits/fixtures/proof_main.r1cs"
     ])
-    node.waitUntilOutput(expectedDownloadInstruction)
-    node.stop()
+    await node.waitUntilOutput(expectedDownloadInstruction)
+    await node.stop()
 
   test "suggests downloading of circuit files when persistence is enabled without accessible zkey file":
-    let node = startNode(@[
+    let node = await startCodex(@[
       "persistence",
       "prover",
       marketplaceArg,
       "--circom-r1cs=tests/circuits/fixtures/proof_main.r1cs",
       "--circom-wasm=tests/circuits/fixtures/proof_main.wasm"
     ])
-    node.waitUntilOutput(expectedDownloadInstruction)
-    node.stop()
+    await node.waitUntilOutput(expectedDownloadInstruction)
+    await node.stop()