wasm bug fixes, and added smoke tests and ci workflow (#389)

.github/workflows/ci.yaml

@@ -30,7 +30,7 @@ jobs:
         run: cargo install -q worker-build && worker-build
 
       - name: test
-        run: cargo test -- --nocapture
+        run: cargo test --workspace --exclude smoke_tests -- --nocapture
         env:
           RUST_BACKTRACE: full
 
@@ -47,6 +47,74 @@ jobs:
         working-directory: libs/napi
         run: yarn build
 
+  smoke-test:
+    name: smoke test
+    needs:
+      - build
+    runs-on: ubuntu-22.04
+    steps:
+      - name: checkout
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
+
+      - name: setup environment
+        uses: ./.github/actions/setup
+
+      - name: run containers
+        working-directory: libs/smoke_tests/
+        run: docker compose -f docker-compose.yaml up -d
+
+      - uses: JarvusInnovations/background-action@v1
+        name: run test server in background
+        with:
+          run: cargo run
+          working-directory: tests/test-server
+          wait-on: http-get://127.0.0.1:4000
+          tail: true
+          wait-for: 10m
+          log-output-if: failure
+          log-output: true
+
+      - uses: JarvusInnovations/background-action@v1
+        name: run gateway in background (binary)
+        with:
+          run: cargo run --bin conductor -- ./libs/smoke_tests/test_gw.yaml
+          wait-on: http-get://127.0.0.1:9000/graphql
+          tail: true
+          wait-for: 15m
+          log-output-if: failure
+          log-output: true
+
+      - name: run tests (binary)
+        working-directory: libs/smoke_tests/
+        run: cargo test --features binary -- --nocapture
+        env:
+          CONDUCTOR_URL: http://127.0.0.1:9000
+
+      - uses: the-guild-org/shared-config/setup@main
+        name: setup env (wasm)
+        with:
+          nodeVersion: 20
+          packageManager: pnpm
+          workingDirectory: bin/cloudflare_worker
+          packageManagerVersion: 8
+
+      - uses: JarvusInnovations/background-action@v1
+        name: run gateway in background (wasm)
+        with:
+          working-directory: bin/cloudflare_worker
+          run: pnpm start:smoke
+          wait-on: http-get://127.0.0.1:8787/graphql
+          tail: true
+          wait-for: 15m
+          log-output-if: failure
+          log-output: true
+
+      - name: run tests (wasm)
+        working-directory: libs/smoke_tests/
+        run: cargo test --features wasm -- --nocapture
+        env:
+          CONDUCTOR_URL: http://127.0.0.1:8787
+
   graphql-over-http:
     runs-on: ubuntu-22.04
     steps:
@@ -151,7 +219,7 @@ jobs:
       - name: run panic free analyzer
         run: cargo panic-analyzer > ./panic-audit.md
         env:
-          IGNORED_CRATES: e2e_tests,benches,server
+          IGNORED_CRATES: smoke_tests,e2e_tests,benches,server
           IGNORED_FILES: ./plugins/jwt_auth/src/test.rs
 
       - name: comment on pull request

Cargo.toml

@@ -1,7 +1,7 @@
 [workspace]
 resolver = "2"
 members = ["bin/*", "libs/*", "plugins/*"]
-exclude = ["bin/npm", "tests/test-server", "tools/panic_free_analyzer"]
+exclude = ["bin/npm", "tests/test-server"]
 
 [workspace.dependencies]
 tokio = "1.35.1"

README.md

@@ -7,7 +7,7 @@
 > [!IMPORTANT]
 > Conductor gateway is still under development, and currently available as alpha.
 >
-> Please use it with caution. Feedback and Contributions are always welcome! 
+> Please use it with caution. Feedback and Contributions are always welcome!
 
 # Conductor: MIT open-source GraphQL Gateway
 
@@ -53,6 +53,12 @@ Conductor's configuration can be defined in both YAML and JSON formats. The conf
 ### Configuration File Example
 
 ```yaml
+server:
+  port: 9000
+
+logger:
+  filter: error
+
 sources:
   - type: graphql
     id: my-source
@@ -63,12 +69,10 @@ endpoints:
   - path: /graphql
     from: my-source
     plugins:
+      - type: cors
+        config:
+          allowed_origin: "*"
       - type: graphiql
-
-plugins:
-  - type: cors
-    config:
-      allowed_origin: "*"
 ```
 
 ## Running Conductor

bin/cloudflare_worker/.cargo/config

@@ -0,0 +1,4 @@
+[target.wasm32-unknown-unknown]
+rustflags = [
+  "-C", "link-args=-z stack-size=3000000",
+]

bin/cloudflare_worker/package.json

@@ -4,7 +4,8 @@
   "private": true,
   "scripts": {
     "deploy": "wrangler deploy",
-    "dev": "wrangler dev --var \"CONDUCTOR_CONFIG:$(cat ../../test_config/worker.yaml)\""
+    "dev": "wrangler dev --var \"CONDUCTOR_CONFIG:$(cat ../../test_config/worker.yaml)\"",
+    "start:smoke": "wrangler dev --var \"ENABLE_WASM_FEATURES:false\" --var \"CONDUCTOR_CONFIG:$(cat ../../libs/smoke_tests/test_gw.yaml)\""
   },
   "devDependencies": {
     "wrangler": "3.25.0"

bin/conductor/docker/Dockerfile

@@ -9,6 +9,7 @@ COPY plugins plugins
 COPY bin bin
 RUN rm -rf lib/benches
 RUN rm -rf lib/e2e_tests
+RUN rm -rf lib/smoke_tests
 RUN echo 'fn main() { println!(""); }' > ./bin/conductor/src/main.rs
 RUN echo 'fn main() { println!(""); }' > ./bin/conductor/src/lib.rs
 # We are only building the dependencies here, with a dummy file, this compiles all dependencies code only.

libs/common/src/graphql.rs

@@ -64,6 +64,8 @@ impl Display for GraphQLRequest {
 
 #[derive(thiserror::Error, Debug)]
 pub enum ExtractGraphQLOperationError {
+  #[error("invalid url query parameter")]
+  InvalidQueryParameterEncoding,
   #[error("missing query parameter")]
   MissingQueryParameter,
   #[error("invalid content-type header")]

libs/config/conductor.schema.json

@@ -162,7 +162,7 @@
             "type": {
               "type": "string",
               "enum": [
-                "mocl"
+                "mock"
               ]
             },
             "id": {
@@ -1706,14 +1706,7 @@
             "cache_duration": {
               "description": "Duration after which the cached JWKS should be expired. If not specified, the default value will be used.",
               "default": "10m",
-              "anyOf": [
-                {
-                  "$ref": "#/definitions/Duration"
-                },
-                {
-                  "type": "null"
-                }
-              ]
+              "type": "string"
             },
             "prefetch": {
               "description": "If set to `true`, the JWKS will be fetched on startup and cached. In case of invalid JWKS, the error will be ignored and the plugin will try to fetch again when server receives the first request. If set to `false`, the JWKS will be fetched on-demand, when the first request comes in.",

libs/config/src/lib.rs

@@ -403,7 +403,7 @@ pub enum SourceDefinition {
     /// The configuration for the GraphQL source.
     config: GraphQLSourceConfig,
   },
-  #[serde(rename = "mocl")]
+  #[serde(rename = "mock")]
   /// A simple, single GraphQL endpoint
   Mock {
     /// The identifier of the source. This is used to reference the source in the `from` field of an endpoint definition.
@@ -601,7 +601,7 @@ pub fn parse_config_contents(
     }
     Err(errors) => {
       for error in errors {
-        println!("error: {}", error);
+        println!("error: {:?}", error);
       }
 
       // @expected: 👇

libs/engine/src/gateway.rs

@@ -57,6 +57,13 @@ pub enum GatewayError {
 
 impl ConductorGateway {
   pub fn match_route(&self, route: &Url) -> Result<&ConductorGatewayRouteData, GatewayError> {
+    // TODO: This function should probably use a more sophisticated matching algorithm.
+    for conductor_route in &self.routes {
+      if route.path() == conductor_route.base_path {
+        return Ok(&conductor_route.route_data);
+      }
+    }
+
     for conductor_route in &self.routes {
       if route.path().starts_with(&conductor_route.base_path) {
         return Ok(&conductor_route.route_data);

libs/smoke_tests/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "smoke_tests"
+version = "0.0.0"
+edition = "2021"
+
+[lib]
+path = "src/lib.rs"
+
+[dependencies]
+tokio = { workspace = true, features = ["full"] }
+reqwest = { workspace = true, features = ["json"] }
+conductor_common = { path = "../common", features = ["test_utils"] }
+serde = { workspace = true }
+serde_json = { workspace = true }
+insta = "1.34.0"
+lazy_static = "1.4.0"
+
+[features]
+binary = []
+wasm = []

libs/smoke_tests/README.md

@@ -0,0 +1,22 @@
+## Smoke Tests
+
+The purpose of these tests is to execute different flows that are connected to real data sources and other third-party dependencies.
+
+Failure in one of these tests during a PR, might indicate that one of the crucial flows is broken.
+
+## CI Setup
+
+Check the `.github/workflows/ci.yaml` (job: `smoke-test`). We are running the same tests against WASM and binary builds.
+
+## Running Locally
+
+To run locally, following these instructions:
+
+1. Make sure you have Docker engine installed and running.
+2. Start the third-pary dependencies by running: `docker compose -f docker-compose.yaml up -d --remove-orphans --wait --force-recreate` inside the `libs/smoke_tests` directory.
+3. Start a real GraphQL server by running: `cargo run` inside `tests/test-server` directory.
+
+Now, run Conductor with one of the configurations:
+
+- For binary runtime, use: `cargo run --bin conductor -- ./libs/smoke_tests/test_gw.yaml` in the root workspace dir.
+- For WASM runtime, use: `pnpm start:smoke` inside `bin/cloudflare_worker`.

libs/smoke_tests/docker-compose.yaml

@@ -0,0 +1,26 @@
+version: "3.8"
+name: "conductor-smoke-test"
+services:
+  # Keycloak (JWT/JWKS Provider)
+  keycloak:
+    image: quay.io/keycloak/keycloak:23.0.4
+    environment:
+      KEYCLOAK_ADMIN: admin
+      KEYCLOAK_ADMIN_PASSWORD: admin
+    command: ["start-dev", "--import-realm"]
+    volumes:
+      - ./volumes/keycloak:/opt/keycloak/data/import
+    ports:
+      - 4001:8080
+
+  # Jaeger (Telmetry)
+  jaeger:
+    image: jaegertracing/all-in-one:1.53
+    environment:
+      COLLECTOR_OTLP_ENABLED: true
+    ports:
+      - 4317:4317 # OTLP over gRPC
+      - 4318:4318 # OTLP over HTTP
+      - 6831:6831/udp # Jaeger Thrift over compact thrift protocol, UDP
+      - 6832:6832/udp # Jaeger Thrift over binary thrift protocol, UDP
+      - 16686:16686 # Jaeger UI / API

libs/smoke_tests/src/http_get.rs

@@ -0,0 +1,39 @@
+#[cfg(test)]
+mod smoke_http_get {
+  use insta::assert_debug_snapshot;
+  use lazy_static::lazy_static;
+  use reqwest::header::{HeaderMap, CONTENT_TYPE};
+  use serde_json::Value;
+  use std::env::var;
+
+  lazy_static! {
+    static ref CONDUCTOR_URL: String = var("CONDUCTOR_URL").expect("CONDUCTOR_URL env var not set");
+  }
+
+  #[tokio::test]
+  async fn http_get() {
+    let mut headers = HeaderMap::default();
+    headers.append(
+      CONTENT_TYPE,
+      "application/x-www-form-urlencoded".parse().unwrap(),
+    );
+
+    let req = reqwest::Client::new()
+      .request(
+        reqwest::Method::GET,
+        format!(
+          "{}/http-get?query=query%20%7B%20__typename%20%7D",
+          CONDUCTOR_URL.as_str()
+        ),
+      )
+      .headers(headers);
+
+    let gql_response = req
+      .send()
+      .await
+      .expect("failed to run http req to conductor");
+    assert_eq!(gql_response.status(), 200);
+    let json_body = gql_response.json::<Value>().await.unwrap();
+    assert_debug_snapshot!(json_body);
+  }
+}