21585: Refactored client and types, MAJOR (#19)

- Types are now auto generated via the Howso Engine instead of using
openapi
- Trainee methods are now generated via the Howso Engine
- Refactored type and client packages structure
- The WasmClient is now named HowsoWorkerClient

---------

Co-authored-by: Lance Gliser <lance.gliser@howso.com>

.gitignore

@@ -121,3 +121,6 @@ Desktop.ini
 # Node
 **/node_modules
 *.tgz
+
+# Rollup
+.rollup.cache

.vscode/extensions.json

@@ -0,0 +1,9 @@
+{
+  // See https://go.microsoft.com/fwlink/?LinkId=827846 to learn about workspace recommendations.
+  // Extension identifier format: ${publisher}.${name}. Example: vscode.csharp
+
+  // List of extensions which should be recommended for users of this workspace.
+  "recommendations": ["ronnidc.nunjucks"],
+  // List of extensions recommended by VS Code that should not be recommended for users of this workspace.
+  "unwantedRecommendations": []
+}

.vscode/settings.json

@@ -1,5 +1,5 @@
 {
-  "cSpell.words": ["caml", "coeff", "rmse"],
+  "cSpell.words": ["absdev", "caml", "coeff", "datetime", "rmse", "Subtrainee", "targetless", "timedelta"],
   "editor.codeActionsOnSave": {
     "source.organizeImports": "explicit"
   },

MIGRATION.md

@@ -1,5 +1,37 @@
 # Migration notes
 
+## 6.x
+
+This major change refactors the client and types for all Howso Engine operations.
+
+### Typing Changes
+
+- Most of all the types are now autogenerated from the Engine API, and will have a different naming schema across
+  the board. However, most of the type's properties should remain the same.
+- `types/shims` have been added as a stop gap for types lost from this migration. These will be transitioned out as
+  types improve in the Howso Engine API specification.
+- The existing `Trainee` type has been renamed to `BaseTrainee` and no longer has a `features` property. Request
+  features via the method `getFeatureAttributes` instead.
+
+### Client Changes
+
+- `BaseClient` has been renamed to `AbstractBaseClient`.
+- `WasmClient` has been renamed to `HowsoWorkerClient`.
+- `ProblemError` has been renamed to `HowsoError`.
+- `ValidationError` has been renamed to `HowsoValidationError`.
+- `ApiError`, `TimeoutError`, and `RetriableError` have been removed.
+- The client's `ClientOptions.libDir` property has been removed.
+- The client constructor now expects an instance of an `AbstractFileSystem` as its second parameter.
+  A `BrowserFileSystem` and `NodeFileSystem` are provided for use in their respective environments.
+- The `train` method no longer batches requests to the Amalgam worker service automatically. Use `batchTrain` instead.
+- The `createTrainee` method no longer sets feature attributes. Call `setFeatureAttributes` manually instead
+  (this also returns the updated object back).
+- The `client/capabilities/*` interfaces have been removed.
+- The return value of all Trainee operations has been changed to be an object with properties:
+  `payload` (this matches the previous return value format) and `warnings` (a list of warning messages, if applicable).
+- The `react` method now uses `context_values` instead of `context` and `action_values` instead of `actions`.
+- `local_*` react features have been removed. Use their unqualified versions instead.
+
 ## 5.x
 
 The `inferFeatureAttributes` function now requires a `sourceFormat` argument, and is strongly typed through a union.

README.md

@@ -49,6 +49,7 @@ This process can be CPU intensive, you are encouraged to use a web `Worker` if r
 #### Through a web Worker
 
 ```ts
+// @/workers/AmalgamWorker
 import { AmalgamWasmService, initRuntime } from "@howso/amalgam-lang";
 import wasmDataUri from "@howso/amalgam-lang/lib/amalgam-st.data?url";
 import wasmUri from "@howso/amalgam-lang/lib/amalgam-st.wasm?url";
@@ -77,15 +78,16 @@ import wasmUri from "@howso/amalgam-lang/lib/amalgam-st.wasm?url";
 You can then create the worker client using a url import:
 
 ```ts
-import howsoUrl from "@/data/engine/howso.caml?url";
-import migrationsUrl from "@/data/engine/migrations.caml?url";
-import { type ClientOptions, Trainee, WasmClient } from "@howso/engine";
+import howsoUrl from "@howso/engine/lib/howso.caml?url";
+import migrationsUrl from "@howso/engine/lib/migrations.caml?url";
+import { type ClientOptions, HowsoWorkerClient, BrowserFileSystem } from "@howso/engine";
 
-const getClient = async (): Promise<WasmClient> => {
+const getClient = async (options?: ClientOptions): Promise<HowsoWorkerClient> => {
   const worker = new Worker(new URL("@/workers/AmalgamWorker", import.meta.url), { type: "module" });
-  const client = new WasmClient(worker, {
+  const fs = new BrowserFileSystem(worker);
+  const client = new HowsoWorkerClient(worker, fs, {
     howsoUrl,
-    migrationsUrl,
+    migrationsUrl, // Optional, used for upgrading Trainees saved to disk.
     ...options,
   });
   return client.setup();

codegen/.gitignore

@@ -0,0 +1 @@
+build

codegen/README.md

@@ -0,0 +1,25 @@
+# Introduction
+
+This sub project is a utility used for introspecting the current Howso Engine
+API documentation and generating types and client code automatically for each
+exposed label (method).
+
+## Getting Started
+
+Note that by running this utility, files will be added/updated/deleted in the
+`src` directory. Once generated, files should then be committed back to the
+repository.
+
+Ensure that the Howso Engine caml file located at `src/assets/howso.caml` is
+up to date before generation.
+
+### Generating the code
+
+```bash
+npm run generate
+```
+
+### Development Guide
+
+This sub project uses [nunjucks](https://mozilla.github.io/nunjucks/) for
+templating, which is very similar to jinja2.

codegen/engine.ts

@@ -0,0 +1,115 @@
+import { initRuntime } from "@howso/amalgam-lang";
+import fs from "node:fs";
+import { createRequire } from "node:module";
+const require = createRequire(import.meta.url);
+
+export type SchemaTypeOption = "any" | "assoc" | "boolean" | "list" | "number" | "string" | "null";
+export type SchemaType = SchemaTypeOption | SchemaTypeOption[];
+
+export interface BaseSchema {
+  description?: string;
+  required?: boolean;
+  default?: any;
+}
+
+export interface Ref extends BaseSchema {
+  ref: string;
+}
+
+export interface Schema extends BaseSchema {
+  type: SchemaType;
+  enum?: (number | string)[];
+  min?: number;
+  max?: number;
+  exclusive_min?: number;
+  exclusive_max?: number;
+  min_size?: number;
+  max_size?: number;
+  values?: SchemaType | Schema | Ref;
+  indices?: Record<string, SchemaType | Schema | Ref>;
+  dynamic_indices?: Record<string, SchemaType | Schema | Ref>;
+  additional_indices?: SchemaType | Schema | Ref | false;
+}
+
+export interface LabelDefinition {
+  parameters: Record<string, Schema | Ref> | null;
+  returns?: SchemaType | Schema | Ref | null;
+  description?: string | null;
+  attribute?: boolean;
+  long_running?: boolean;
+  read_only?: boolean;
+  idempotent?: boolean;
+  statistically_idempotent?: boolean;
+}
+
+export interface EngineApi {
+  readonly labels: Record<string, LabelDefinition>;
+  readonly schemas: Record<string, Schema>;
+  readonly description: string;
+  readonly version: string;
+}
+
+export async function getEngineApi(): Promise<EngineApi> {
+  const handle = "default";
+  const enginePath = require.resolve("../../src/assets/howso.caml");
+  const dataPath = require.resolve("@howso/amalgam-lang/lib/amalgam-st.data");
+  const wasmPath = require.resolve("@howso/amalgam-lang/lib/amalgam-st.wasm");
+
+  const amalgam = await initRuntime(
+    {},
+    {
+      locateFile: (path) => {
+        // Override file paths to resolved locations
+        if (path.endsWith("amalgam-st.data")) {
+          return dataPath;
+        } else if (path.endsWith("amalgam-st.wasm")) {
+          return wasmPath;
+        }
+        return path;
+      },
+    },
+  );
+
+  try {
+    // Load the Howso Engine into Amalgam
+    amalgam.runtime.FS.writeFile("howso.caml", fs.readFileSync(enginePath));
+    amalgam.loadEntity(handle, "howso.caml");
+    console.log(`Amalgam Version: ${amalgam.getVersion()}`);
+
+    // Initialize the Engine
+    const initialized = amalgam.executeEntityJson(handle, "initialize", { trainee_id: handle });
+    if (!initialized) {
+      throw new Error("Failed to initialize the Howso Engine.");
+    }
+
+    // Get the api documentation from the Engine
+    const response = amalgam.executeEntityJson(handle, "get_api", "");
+    if (!Array.isArray(response) || response[0] != 1) {
+      throw new Error("Failed to retrieve API documentation from the Howso Engine.");
+    }
+    const doc: EngineApi = response[1].payload;
+    console.log(`Howso Engine Version: ${doc.version}`);
+    return doc;
+  } finally {
+    amalgam.destroyEntity(handle);
+  }
+}
+
+export function isRef(value: SchemaType | Schema | Ref | null | undefined): value is Ref {
+  if (value == null || Array.isArray(value) || typeof value === "string") {
+    return false;
+  }
+  return "ref" in value && value.ref != null;
+}
+
+export function isSchema(value: SchemaType | Schema | Ref | null | undefined): value is Schema {
+  if (value == null || Array.isArray(value) || typeof value === "string") {
+    return false;
+  }
+  return !isRef(value) && "type" in value && (typeof value.type === "string" || Array.isArray(value.type));
+}
+
+export function isSchemaOrRef(value: SchemaType | Schema | Ref | boolean | null | undefined): value is Schema | Ref {
+  if (typeof value === "boolean") return false;
+  return isRef(value) || isSchema(value);
+}

codegen/filters/comments.ts

@@ -0,0 +1,37 @@
+/**
+ * Block commenter filter.
+ *
+ * This filter will ensure each line starts with * in a block comment. Assumes
+ * the template includes * on first line.
+ */
+export function blockComment(content: any) {
+  // Ensure the content is a string
+  const str = typeof content === "string" ? content : String(content);
+
+  return str
+    .split("\n")
+    .reduceRight<string[]>((accumulator, value) => {
+      // Removing tailing empty lines
+      if (accumulator.length === 0 && value.trim() === "") {
+        return accumulator;
+      }
+      accumulator.unshift(value);
+      return accumulator;
+    }, [])
+    .map((line, index) => {
+      let value: string;
+      if (line.replace(/^\s+$/gm, "")) {
+        value = " " + line.replace(/\t/g, "  ");
+      } else {
+        // Render empty lines
+        value = "";
+      }
+      if (index > 0) {
+        // Add '*' to the beginning of subsequent lines
+        return " *" + value;
+      }
+      return value.trimStart(); // First line should have no leading spaces
+    })
+    .join("\n")
+    .trimEnd();
+}

codegen/filters/index.ts

@@ -0,0 +1,23 @@
+import { Environment } from "nunjucks";
+import * as engine from "../engine";
+import * as comments from "./comments";
+import * as strings from "./strings";
+import * as types from "./types";
+
+/**
+ * Add all filters to an Environment.
+ * @param env The environment to register filters in.
+ */
+export function registerFilters(env: Environment) {
+  env.addFilter(comments.blockComment.name, comments.blockComment);
+  env.addFilter(strings.pascalCase.name, strings.pascalCase);
+  env.addFilter(strings.camelCase.name, strings.camelCase);
+  env.addFilter(strings.toJson.name, strings.toJson);
+  env.addFilter(strings.autoQuote.name, strings.autoQuote);
+  env.addFilter(types.isString.name, types.isString);
+  env.addFilter(types.isArray.name, types.isArray);
+  env.addFilter(types.convertType.name, types.convertType);
+  env.addFilter(engine.isRef.name, engine.isRef);
+  env.addFilter(engine.isSchema.name, engine.isSchema);
+  env.addFilter(engine.isSchemaOrRef.name, engine.isSchemaOrRef);
+}

codegen/filters/strings.ts

@@ -0,0 +1,24 @@
+import { toCamelCase, toPascalCase } from "../utils";
+
+export function pascalCase(value: any) {
+  const str = typeof value === "string" ? value : String(value);
+  return toPascalCase(str);
+}
+
+export function camelCase(value: any) {
+  const str = typeof value === "string" ? value : String(value);
+  return toCamelCase(str);
+}
+
+export function toJson(value: any) {
+  return JSON.stringify(value);
+}
+
+export function autoQuote(value: any) {
+  // Quote values for safe use as object keys
+  const str = typeof value === "string" ? value : String(value);
+  if (/^[a-zA-Z_$][a-zA-Z0-9_$]*$/.test(str)) {
+    return str;
+  }
+  return this.env.filters.safe(JSON.stringify(str));
+}

codegen/filters/types.ts

@@ -0,0 +1,41 @@
+import { SchemaType } from "../engine";
+
+export function isString(value: any) {
+  return typeof value === "string";
+}
+
+export function isArray(value: any) {
+  return Array.isArray(value);
+}
+
+export function convertType(schema: SchemaType) {
+  let value: string;
+  switch (schema) {
+    case "assoc":
+      value = "Record<string, any>";
+      break;
+    case "list":
+      value = "any[]";
+      break;
+    case "string":
+      value = "string";
+      break;
+    case "boolean":
+      value = "boolean";
+      break;
+    case "number":
+      value = "number";
+      break;
+    case "null":
+      value = "null";
+      break;
+    case "any":
+      value = "any";
+      break;
+    default:
+      console.warn(`Unexpected type received: ${schema}`);
+      value = "any";
+  }
+
+  return this.env.filters.safe(value);
+}