From 44aeeebd8dd0d7467fdc0c8ac493fc69cc96f9a6 Mon Sep 17 00:00:00 2001 From: Tomi Jaga Date: Wed, 8 May 2024 21:57:20 -0400 Subject: [PATCH] Add support for converting to and from ICRC3 value type --- .github/workflows/makefile.yml | 93 +++++++++++++++++------------ makefile | 16 ++--- mops.toml | 12 ++-- readme.md | 5 +- src/CBOR/lib.mo | 38 ++++++------ src/Candid/Blob/Decoder.mo | 46 +++++++------- src/Candid/Blob/Encoder.mo | 9 +-- src/Candid/Text/Parser/Array.mo | 1 - src/Candid/Text/Parser/Bool.mo | 4 +- src/Candid/Text/Parser/Int.mo | 2 - src/Candid/Text/Parser/IntX.mo | 5 +- src/Candid/Text/Parser/Nat.mo | 1 - src/Candid/Text/Parser/NatX.mo | 5 +- src/Candid/Text/Parser/Option.mo | 1 - src/Candid/Text/Parser/Principal.mo | 2 - src/Candid/Text/Parser/Record.mo | 2 +- src/Candid/Text/Parser/Variant.mo | 1 - src/Candid/Text/Parser/lib.mo | 2 +- src/Candid/Text/ToText.mo | 2 +- src/Candid/Types.mo | 11 ++++ src/Candid/lib.mo | 12 +--- src/JSON/FromText.mo | 7 --- src/JSON/ToText.mo | 10 +--- src/JSON/lib.mo | 1 + src/UrlEncoded/FromText.mo | 27 ++++----- src/UrlEncoded/Parser.mo | 3 - src/UrlEncoded/ToText.mo | 13 +--- src/UrlEncoded/lib.mo | 1 + src/Utils.mo | 2 - src/lib.mo | 4 +- tests/Candid.ICRC3.Test.mo | 44 ++++++++++++++ tests/Candid.Test.mo | 4 +- tests/JSON.Test.mo | 5 +- usage.md | 73 +++++++++++++++++++++- 34 files changed, 281 insertions(+), 183 deletions(-) create mode 100644 tests/Candid.ICRC3.Test.mo diff --git a/.github/workflows/makefile.yml b/.github/workflows/makefile.yml index d701864..88da08e 100644 --- a/.github/workflows/makefile.yml +++ b/.github/workflows/makefile.yml @@ -1,42 +1,59 @@ name: Makefile CI -on: - push: - branches: ["main"] - pull_request: - branches: ["main"] +on: + push: + branches: + - main + pull_request: + branches: + - "*" jobs: - build: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: 18 - - uses: aviate-labs/setup-dfx@v0.2.5 - with: - dfx-version: 0.14.1 - - - name: install wasmtime - run: | - curl https://wasmtime.dev/install.sh -sSf | bash - echo "$HOME/.wasmtime/bin" >> $GITHUB_PATH - - npm --yes -g i mocv - - - name: Select mocv version - run: mocv use 0.10.3 - - - name: install mops - run: | - npm --yes -g i ic-mops@0.34.3 - mops i - mops sources - - # - name: Detect Warnings - # run: make no-warn - - - name: Run Tests - run: mops test + build: + runs-on: ubuntu-latest + + name: Build and test + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: 18 + + - name: Cache Node modules + uses: actions/cache@v2 + with: + path: ~/.npm + key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node- + + - name: Cache mops packages + uses: actions/cache@v3 + with: + key: mops-packages-${{ hashFiles('mops.toml') }} + path: | + ~/.cache/mops + ~/mops + + - name: Install dfx + uses: dfinity/setup-dfx@main + - name: Confirm successful installation + run: dfx --version + + - name: Install dfx cache + run: dfx cache install + + - name: Install mops & mocv + run: | + npm --yes -g i ic-mops + mops i + mops toolchain init + + # set moc path for dfx to use + echo "DFX_MOC_PATH=$(mops toolchain bin moc)" >> $GITHUB_ENV + + # - name: Detect warnings + # run: make check + + - name: Run Tests + run: mops test \ No newline at end of file diff --git a/makefile b/makefile index 078a134..5e4991e 100644 --- a/makefile +++ b/makefile @@ -1,11 +1,11 @@ -.PHONY: compile-tests no-warn docs +.PHONY: test compile-tests docs no-warn -compile-tests: - bash compile-tests.sh $(file) +test: + mops test -no-warn: - find src -type f -name '*.mo' -print0 | xargs -0 $(shell mocv bin)/moc -r $(shell mops sources) -Werror -wasi-system-api +check: + find src -type f -name '*.mo' -print0 | \ + xargs -0 $(shell mops toolchain bin moc) -r $(shell mops sources) -Werror -wasi-system-api -docs: - $(shell mocv bin)/mo-doc - $(shell mocv bin)/mo-doc --format plain +bench: + mops bench --gc incremental \ No newline at end of file diff --git a/mops.toml b/mops.toml index 11d97c6..019da8b 100644 --- a/mops.toml +++ b/mops.toml @@ -1,13 +1,14 @@ [package] name = "serde" -version = "2.2.0" +version = "2.3.0" description = "A serialisation and deserialisation library for Motoko." repository = "https://github.com/NatLabs/serde" keywords = [ "json", "candid", "cbor", "urlencoded", "serialization" ] +license = "MIT" [dependencies] -base = "0.10.3" -itertools = "0.1.2" +base = "0.11.1" +itertools = "0.2.1" candid = "1.0.2" xtended-numbers = "0.2.1" json = "https://github.com/NatLabs/json.mo#float" @@ -15,7 +16,8 @@ parser-combinators = "https://github.com/aviate-labs/parser-combinators.mo#v0.1. cbor = "0.1.3" [dev-dependencies] -test = "1.2.0" +test = "2.0.0" [toolchain] -wasmtime = "14.0.4" \ No newline at end of file +wasmtime = "14.0.4" +moc = "0.11.1" \ No newline at end of file diff --git a/readme.md b/readme.md index 8209cdb..4936212 100644 --- a/readme.md +++ b/readme.md @@ -8,7 +8,7 @@ The library contains four modules: - `toText()` - Converts serialized candid to its [textual representation](https://internetcomputer.org/docs/current/tutorials/developer-journey/level-2/2.4-intro-candid/#candid-textual-values). - `encode()` - Converts the [Candid variant](./src/Candid/Types.mo#L6) to a blob. - `decode()` - Converts a blob to the [Candid variant](./src/Candid/Types.mo#L6). - + > encoding and decoding functions also support conversion between the [`ICRC3` value type](https://github.com/dfinity/ICRC-1/tree/main/standards/ICRC-3#value) and candid. Checkout the example in the [usage guide](./usage.md#icrc3-value) - **CBOR** - `encode()` - Converts serialized candid to CBOR. - `decode()` - Converts CBOR to a serialized candid. @@ -159,13 +159,14 @@ assert renamedKeys == ?{ item_type = "bar"; item_label = "foo"; id = 112 }; ``` Checkout the [usage guide](https://github.com/NatLabs/serde/blob/main/usage.md) for additional examples: -- [Candid Text](https://github.com/NatLabs/serde/blob/main/usage.md#candid-text) +- [Candid](https://github.com/NatLabs/serde/blob/main/usage.md#candid-text) - [URL-Encoded Pairs](https://github.com/NatLabs/serde/blob/main/usage.md#url-encoded-pairs) ## Limitations - Users must provide a list of record keys and variant names during conversions from Motoko to other data formats due to constraints in the candid format. - Lack of specific syntax for conversion between `Blob`, `Principal`, and bounded `Nat`/`Int` types. +- Floats are only recognised if they have a decimal point, e.g., `1.0` is a Float, but `1` is an `Int` / `Nat`. ## Running Tests diff --git a/src/CBOR/lib.mo b/src/CBOR/lib.mo index 39f68d7..9d72a30 100644 --- a/src/CBOR/lib.mo +++ b/src/CBOR/lib.mo @@ -4,6 +4,7 @@ import Int8 "mo:base/Int8"; import Int16 "mo:base/Int16"; import Int32 "mo:base/Int32"; import Int64 "mo:base/Int64"; +import Option "mo:base/Option"; import Nat64 "mo:base/Nat64"; import Result "mo:base/Result"; import Principal "mo:base/Principal"; @@ -12,7 +13,6 @@ import CBOR_Value "mo:cbor/Value"; import CBOR_Encoder "mo:cbor/Encoder"; import CBOR_Decoder "mo:cbor/Decoder"; import NatX "mo:xtended-numbers/NatX"; -import IntX "mo:xtended-numbers/IntX"; import FloatX "mo:xtended-numbers/FloatX"; import Candid "../Candid"; @@ -32,14 +32,14 @@ module { let decoded_res = Candid.decode(blob, keys, options); let #ok(candid) = decoded_res else return Utils.send_error(decoded_res); - let json_res = fromCandid(candid[0]); + let json_res = fromCandid(candid[0], Option.get(options, CandidTypes.defaultOptions)); let #ok(json) = json_res else return Utils.send_error(json_res); #ok(json); }; /// Convert a Candid value to CBOR blob - public func fromCandid(candid : Candid) : Result { - let res = transpile_candid_to_cbor(candid); + public func fromCandid(candid : Candid, options: CandidTypes.Options) : Result { + let res = transpile_candid_to_cbor(candid, options); let #ok(transpiled_cbor) = res else return Utils.send_error(res); let cbor_with_self_describe_tag = #majorType6({ tag = 55799 : Nat64; value = transpiled_cbor; }); @@ -50,7 +50,7 @@ module { }; }; - func transpile_candid_to_cbor(candid : Candid) : Result { + func transpile_candid_to_cbor(candid : Candid, options: CandidTypes.Options) : Result { let transpiled_cbor : CBOR = switch(candid){ case (#Empty) #majorType7(#_undefined); case (#Null) #majorType7(#_null); @@ -75,18 +75,18 @@ module { let buffer = Buffer.Buffer(arr.size()); for (item in arr.vals()){ - let res = transpile_candid_to_cbor(item); + let res = transpile_candid_to_cbor(item, options); let #ok(cbor_val) = res else return Utils.send_error(res); buffer.add(cbor_val); }; #majorType4(Buffer.toArray(buffer)); }; - case (#Record(records)) { + case (#Record(records) or #Map(records)) { let newRecords = Buffer.Buffer<(CBOR, CBOR)>(records.size()); for ((key, val) in records.vals()){ - let res = transpile_candid_to_cbor(val); + let res = transpile_candid_to_cbor(val, options); let #ok(cbor_val) = res else return Utils.send_error(res); newRecords.add((#majorType3(key), cbor_val)); }; @@ -100,7 +100,7 @@ module { // // check out "CBOR Tests.options" in the tests folder to see how this in action case (#Option(option)) { - let res = transpile_candid_to_cbor(option); + let res = transpile_candid_to_cbor(option, options); let #ok(cbor_val) = res else return Utils.send_error(res); cbor_val }; @@ -116,18 +116,18 @@ module { }; public func decode(blob: Blob, options: ?Options): Result { - let candid_res = toCandid(blob); + let candid_res = toCandid(blob, Option.get(options, CandidTypes.defaultOptions)); let #ok(candid) = candid_res else return Utils.send_error(candid_res); Candid.encodeOne(candid, options); }; - public func toCandid(blob: Blob): Result { + public func toCandid(blob: Blob, options: CandidTypes.Options): Result { let cbor_res = CBOR_Decoder.decode(blob); let candid_res = switch (cbor_res) { case (#ok(cbor)) { - let #majorType6({ tag = 55799; value }) = cbor else return transpile_cbor_to_candid(cbor); - transpile_cbor_to_candid(value); + let #majorType6({ tag = 55799; value }) = cbor else return transpile_cbor_to_candid(cbor, options); + transpile_cbor_to_candid(value, options); }; case (#err(cbor_error)) { switch(cbor_error){ @@ -142,7 +142,7 @@ module { #ok(candid); }; - public func transpile_cbor_to_candid(cbor: CBOR) : Result{ + public func transpile_cbor_to_candid(cbor: CBOR, options: CandidTypes.Options) : Result{ let transpiled_candid = switch(cbor){ case (#majorType0(n)) #Nat(Nat64.toNat(n)); case (#majorType1(n)) #Int(n); @@ -151,7 +151,7 @@ module { case (#majorType4(arr)) { let buffer = Buffer.Buffer(arr.size()); for (item in arr.vals()){ - let res = transpile_cbor_to_candid(item); + let res = transpile_cbor_to_candid(item, options); let #ok(candid_val) = res else return Utils.send_error(res); buffer.add(candid_val); }; @@ -162,12 +162,16 @@ module { for ((cbor_text, val) in records.vals()){ let #majorType3(key) = cbor_text else return #err("Error decoding CBOR: Unexpected key type"); - let res = transpile_cbor_to_candid(val); + let res = transpile_cbor_to_candid(val, options); let #ok(candid_val) = res else return Utils.send_error(res); buffer.add((key, candid_val)); }; - #Record(Buffer.toArray(buffer)); + if (options.use_icrc_3_value_type){ + #Map(Buffer.toArray(buffer)); + } else { + #Record(Buffer.toArray(buffer)); + }; }; case (#majorType7(#_undefined)) #Empty; case (#majorType7(#_null)) #Null; diff --git a/src/Candid/Blob/Decoder.mo b/src/Candid/Blob/Decoder.mo index 94a56c7..43e3a9e 100644 --- a/src/Candid/Blob/Decoder.mo +++ b/src/Candid/Blob/Decoder.mo @@ -1,17 +1,13 @@ -import Array "mo:base/Array"; import Blob "mo:base/Blob"; import Buffer "mo:base/Buffer"; -import Debug "mo:base/Debug"; import Result "mo:base/Result"; import TrieMap "mo:base/TrieMap"; import Nat32 "mo:base/Nat32"; import Text "mo:base/Text"; import Iter "mo:base/Iter"; -import Order "mo:base/Order"; +import Option "mo:base/Option"; import Hash "mo:base/Hash"; -import Prelude "mo:base/Prelude"; -import Encoder "mo:candid/Encoder"; import Decoder "mo:candid/Decoder"; import Arg "mo:candid/Arg"; import Value "mo:candid/Value"; @@ -81,7 +77,7 @@ module { let ?(args) = decoded else return #err("Candid Error: Failed to decode candid blob"); - fromArgs(args, recordKeyMap); + fromArgs(args, recordKeyMap, Option.get(options, T.defaultOptions)); }; func formatVariantKey(key : Text) : Text { @@ -99,11 +95,11 @@ module { ); }; - public func fromArgs(args : [Arg], recordKeyMap : TrieMap.TrieMap) : Result<[Candid], Text> { + public func fromArgs(args : [Arg], recordKeyMap : TrieMap.TrieMap, options: T.Options) : Result<[Candid], Text> { let buffer = Buffer.Buffer(args.size()); for (arg in args.vals()) { - let res = fromArg(arg.type_, arg.value, recordKeyMap); + let res = fromArg(arg.type_, arg.value, recordKeyMap, options); let #ok(val) = res else return Utils.send_error(res); buffer.add(val); }; @@ -111,7 +107,7 @@ module { #ok(Buffer.toArray(buffer)); }; - func fromArg(type_ : Type, val : Value, recordKeyMap : TrieMap.TrieMap) : Result { + func fromArg(type_ : Type, val : Value, recordKeyMap : TrieMap.TrieMap, options: T.Options) : Result { let result : Candid = switch (type_, val) { case ((#recursiveReference(_) or #nat), #nat(n)) #Nat(n); case ((#recursiveReference(_) or #nat8), #nat8(n)) #Nat8(n); @@ -140,13 +136,13 @@ module { case (_, #opt(#null_)) { #Option(#Null) }; case (#opt(innerType), #opt(optVal)) { - let res = fromArg(innerType, optVal, recordKeyMap); + let res = fromArg(innerType, optVal, recordKeyMap, options); let #ok(val) = res else return Utils.send_error(res); #Option(val); }; case (#recursiveReference(ref_id), #opt(optVal)){ - let res = fromArg(#recursiveReference(ref_id), optVal, recordKeyMap); + let res = fromArg(#recursiveReference(ref_id), optVal, recordKeyMap, options); let #ok(val) = res else return Utils.send_error(res); #Option(val); }; @@ -168,7 +164,7 @@ module { let buffer = Buffer.Buffer(arr.size()); for (elem in arr.vals()){ - let res = fromArg(innerType, elem, recordKeyMap); + let res = fromArg(innerType, elem, recordKeyMap, options); let #ok(val) = res else return Utils.send_error(res); buffer.add(val); }; @@ -182,7 +178,7 @@ module { let buffer = Buffer.Buffer(arr.size()); for (elem in arr.vals()){ - let res = fromArg(#recursiveReference(ref_id), elem, recordKeyMap); + let res = fromArg(#recursiveReference(ref_id), elem, recordKeyMap, options); let #ok(val) = res else return Utils.send_error(res); buffer.add(val); }; @@ -203,15 +199,19 @@ module { let { tag; value } = record_val; let key = getKey(tag, recordKeyMap); - let res = fromArg(innerType, value, recordKeyMap); + let res = fromArg(innerType, value, recordKeyMap, options); let #ok(val) = res else return Utils.send_error(res); newRecords.add((key, val)); }; newRecords.sort(U.cmpRecords); - - #Record(Buffer.toArray(newRecords)); + + if (options.use_icrc_3_value_type){ + #Map(Buffer.toArray(newRecords)); + } else { + #Record(Buffer.toArray(newRecords)); + }; }; case (#recursiveReference(ref_id), #record(records)) { @@ -222,7 +222,7 @@ module { let { tag; value } = record; let key = getKey(tag, recordKeyMap); - let res = fromArg(#recursiveReference(ref_id), value, recordKeyMap); + let res = fromArg(#recursiveReference(ref_id), value, recordKeyMap, options); let #ok(val) = res else return Utils.send_error(res); newRecords.add((key, val)); @@ -230,7 +230,11 @@ module { newRecords.sort(U.cmpRecords); - #Record(Buffer.toArray(newRecords)); + if (options.use_icrc_3_value_type){ + #Map(Buffer.toArray(newRecords)); + } else { + #Record(Buffer.toArray(newRecords)); + }; }; case ( #variant(variantTypes), #variant(v)) { @@ -238,7 +242,7 @@ module { for ({ tag; type_ = innerType } in variantTypes.vals()) { if (tag == v.tag) { let key = getKey(tag, recordKeyMap); - let res = fromArg(innerType, v.value, recordKeyMap); + let res = fromArg(innerType, v.value, recordKeyMap, options); let #ok(val) = res else return Utils.send_error(res); @@ -251,7 +255,7 @@ module { case (#recursiveReference(ref_id), #variant(v)) { let key = getKey(v.tag, recordKeyMap); - let res = fromArg(#recursiveReference(ref_id), v.value, recordKeyMap); + let res = fromArg(#recursiveReference(ref_id), v.value, recordKeyMap, options); let #ok(val) = res else return Utils.send_error(res); @@ -259,7 +263,7 @@ module { }; case (#recursiveType({ type_ }), value_) { - let res = fromArg(type_, value_, recordKeyMap); + let res = fromArg(type_, value_, recordKeyMap, options); let #ok(val) = res else return Utils.send_error(res); val; }; diff --git a/src/Candid/Blob/Encoder.mo b/src/Candid/Blob/Encoder.mo index a16b980..2b56fda 100644 --- a/src/Candid/Blob/Encoder.mo +++ b/src/Candid/Blob/Encoder.mo @@ -1,7 +1,6 @@ import Array "mo:base/Array"; import Blob "mo:base/Blob"; import Buffer "mo:base/Buffer"; -import Debug "mo:base/Debug"; import Result "mo:base/Result"; import Nat "mo:base/Nat"; import Iter "mo:base/Iter"; @@ -9,7 +8,6 @@ import Prelude "mo:base/Prelude"; import Text "mo:base/Text"; import Encoder "mo:candid/Encoder"; -import Decoder "mo:candid/Decoder"; import Arg "mo:candid/Arg"; import Value "mo:candid/Value"; import Type "mo:candid/Type"; @@ -18,12 +16,9 @@ import Itertools "mo:itertools/Iter"; import PeekableIter "mo:itertools/PeekableIter"; import T "../Types"; -import U "../../Utils"; import TrieMap "mo:base/TrieMap"; import Utils "../../Utils"; -import Order "mo:base/Order"; import Func "mo:base/Func"; -import Char "mo:base/Char"; module { type Arg = Arg.Arg; @@ -171,7 +166,7 @@ module { (#vector(types), #vector(values)); }; - case (#Record(records)) { + case (#Record(records) or #Map(records)) { let types_buffer = Buffer.Buffer(records.size()); let values_buffer = Buffer.Buffer(records.size()); @@ -269,7 +264,6 @@ module { func merge_variants_and_array_types(rows : Buffer<[InternalTypeNode]>) : Result { let buffer = Buffer.Buffer(8); - let total_rows = rows.size(); func calc_height(parent : Nat, child : Nat) : Nat = parent + child; @@ -431,7 +425,6 @@ module { }; func order_types_by_height_bfs(rows : Buffer<[InternalTypeNode]>) { - var merged_type : ?InternalType = null; label while_loop while (rows.size() > 0) { let candid_values = Buffer.last(rows) else return Prelude.unreachable(); diff --git a/src/Candid/Text/Parser/Array.mo b/src/Candid/Text/Parser/Array.mo index 207e909..fb75a1e 100644 --- a/src/Candid/Text/Parser/Array.mo +++ b/src/Candid/Text/Parser/Array.mo @@ -1,4 +1,3 @@ -import Iter "mo:base/Iter"; import List "mo:base/List"; import C "mo:parser-combinators/Combinators"; diff --git a/src/Candid/Text/Parser/Bool.mo b/src/Candid/Text/Parser/Bool.mo index e6da61a..5e563cc 100644 --- a/src/Candid/Text/Parser/Bool.mo +++ b/src/Candid/Text/Parser/Bool.mo @@ -1,12 +1,10 @@ -import Blob "mo:base/Blob"; -import Iter "mo:base/Iter"; import List "mo:base/List"; import C "mo:parser-combinators/Combinators"; import P "mo:parser-combinators/Parser"; import Candid "../../Types"; -import { ignoreSpace; hexChar; fromHex } "Common"; +import { ignoreSpace; } "Common"; module { type Candid = Candid.Candid; diff --git a/src/Candid/Text/Parser/Int.mo b/src/Candid/Text/Parser/Int.mo index 4618698..3655afc 100644 --- a/src/Candid/Text/Parser/Int.mo +++ b/src/Candid/Text/Parser/Int.mo @@ -3,11 +3,9 @@ import List "mo:base/List"; import C "mo:parser-combinators/Combinators"; import P "mo:parser-combinators/Parser"; -import NatX "mo:xtended-numbers/NatX"; import Candid "../../Types"; -import { listToNat } "Common"; import { parseNat } "Nat"; module { diff --git a/src/Candid/Text/Parser/IntX.mo b/src/Candid/Text/Parser/IntX.mo index 4246214..40ca70d 100644 --- a/src/Candid/Text/Parser/IntX.mo +++ b/src/Candid/Text/Parser/IntX.mo @@ -1,5 +1,4 @@ import Debug "mo:base/Debug"; -import Iter "mo:base/Iter"; import List "mo:base/List"; import Int8 "mo:base/Int8"; import Int16 "mo:base/Int16"; @@ -10,8 +9,8 @@ import C "mo:parser-combinators/Combinators"; import P "mo:parser-combinators/Parser"; import Candid "../../Types"; -import { ignoreSpace; toText } "Common"; -import { parseInt; intParser } "Int"; +import { ignoreSpace } "Common"; +import { parseInt } "Int"; module { type Candid = Candid.Candid; diff --git a/src/Candid/Text/Parser/Nat.mo b/src/Candid/Text/Parser/Nat.mo index 5980150..ffa1e4c 100644 --- a/src/Candid/Text/Parser/Nat.mo +++ b/src/Candid/Text/Parser/Nat.mo @@ -1,4 +1,3 @@ -import Debug "mo:base/Debug"; import Iter "mo:base/Iter"; import List "mo:base/List"; import Nat64 "mo:base/Nat64"; diff --git a/src/Candid/Text/Parser/NatX.mo b/src/Candid/Text/Parser/NatX.mo index f916c6e..00c4f3c 100644 --- a/src/Candid/Text/Parser/NatX.mo +++ b/src/Candid/Text/Parser/NatX.mo @@ -1,5 +1,4 @@ import Debug "mo:base/Debug"; -import Iter "mo:base/Iter"; import List "mo:base/List"; import Nat8 "mo:base/Nat8"; import Nat16 "mo:base/Nat16"; @@ -10,8 +9,8 @@ import C "mo:parser-combinators/Combinators"; import P "mo:parser-combinators/Parser"; import Candid "../../Types"; -import { ignoreSpace; toText } "Common"; -import { parseNat; natParser } "Nat"; +import { ignoreSpace } "Common"; +import { parseNat } "Nat"; module { type Candid = Candid.Candid; diff --git a/src/Candid/Text/Parser/Option.mo b/src/Candid/Text/Parser/Option.mo index 338ff3c..2edc30b 100644 --- a/src/Candid/Text/Parser/Option.mo +++ b/src/Candid/Text/Parser/Option.mo @@ -1,4 +1,3 @@ -import Iter "mo:base/Iter"; import List "mo:base/List"; import C "mo:parser-combinators/Combinators"; diff --git a/src/Candid/Text/Parser/Principal.mo b/src/Candid/Text/Parser/Principal.mo index be12212..9830edb 100644 --- a/src/Candid/Text/Parser/Principal.mo +++ b/src/Candid/Text/Parser/Principal.mo @@ -1,5 +1,3 @@ -import Blob "mo:base/Blob"; -import Iter "mo:base/Iter"; import List "mo:base/List"; import Principal "mo:base/Principal"; diff --git a/src/Candid/Text/Parser/Record.mo b/src/Candid/Text/Parser/Record.mo index 84658f0..f0c0529 100644 --- a/src/Candid/Text/Parser/Record.mo +++ b/src/Candid/Text/Parser/Record.mo @@ -5,7 +5,7 @@ import C "mo:parser-combinators/Combinators"; import P "mo:parser-combinators/Parser"; import Candid "../../Types"; -import { ignoreSpace; hexChar; fromHex; toText } "Common"; +import { ignoreSpace; toText } "Common"; import { parseText } "Text"; module { diff --git a/src/Candid/Text/Parser/Variant.mo b/src/Candid/Text/Parser/Variant.mo index 442ee6d..73a4f9f 100644 --- a/src/Candid/Text/Parser/Variant.mo +++ b/src/Candid/Text/Parser/Variant.mo @@ -1,4 +1,3 @@ -import Iter "mo:base/Iter"; import List "mo:base/List"; import C "mo:parser-combinators/Combinators"; diff --git a/src/Candid/Text/Parser/lib.mo b/src/Candid/Text/Parser/lib.mo index 3686b3e..0e0fe23 100644 --- a/src/Candid/Text/Parser/lib.mo +++ b/src/Candid/Text/Parser/lib.mo @@ -9,7 +9,7 @@ import P "mo:parser-combinators/Parser"; import Candid "../../Types"; -import { ignoreSpace; any } "Common"; +import { ignoreSpace } "Common"; import { arrayParser } "Array"; import { blobParser } "Blob"; diff --git a/src/Candid/Text/ToText.mo b/src/Candid/Text/ToText.mo index 062ebbb..1bd06f5 100644 --- a/src/Candid/Text/ToText.mo +++ b/src/Candid/Text/ToText.mo @@ -70,7 +70,7 @@ module { text # "}"; }; - case (#Record(fields)) { + case (#Record(fields) or #Map(fields)) { var text = "record { "; for ((key, val) in fields.vals()) { diff --git a/src/Candid/Types.mo b/src/Candid/Types.mo index 0b95131..a84143b 100644 --- a/src/Candid/Types.mo +++ b/src/Candid/Types.mo @@ -28,6 +28,7 @@ module { #Option : Candid; #Array : [Candid]; #Record : [KeyValuePair]; + #Map : [KeyValuePair]; #Variant : KeyValuePair; }; @@ -36,6 +37,16 @@ module { public type Options = { /// Contains an array of tuples of the form (old_name, new_name) to rename the record keys. renameKeys : [(Text, Text)]; + + // convertAllNumbersToFloats : Bool; + + use_icrc_3_value_type : Bool; }; + public let defaultOptions = { + renameKeys = []; + // convertAllNumbersToFloats = false; + use_icrc_3_value_type = false; + }; + }; diff --git a/src/Candid/lib.mo b/src/Candid/lib.mo index 16282f4..beb2add 100644 --- a/src/Candid/lib.mo +++ b/src/Candid/lib.mo @@ -1,14 +1,6 @@ -import Array "mo:base/Array"; -import Debug "mo:base/Debug"; -import Text "mo:base/Text"; -import Order "mo:base/Order"; - /// A representation of the Candid format with variants for all possible types. -import Result "mo:base/Result"; -import Prelude "mo:base/Prelude"; - -import Itertools "mo:itertools/Iter"; +import Text "mo:base/Text"; import Encoder "Blob/Encoder"; @@ -23,6 +15,8 @@ import Utils "../Utils"; module { /// A representation of the Candid format with variants for all possible types. public type Candid = T.Candid; + public type Options = T.Options; + public let defaultOptions = T.defaultOptions; /// Converts a motoko value to a [Candid](#Candid) value public let { encode; encodeOne } = Encoder; diff --git a/src/JSON/FromText.mo b/src/JSON/FromText.mo index 62a74e8..fba2516 100644 --- a/src/JSON/FromText.mo +++ b/src/JSON/FromText.mo @@ -1,14 +1,7 @@ import Array "mo:base/Array"; -import Debug "mo:base/Debug"; import Result "mo:base/Result"; -import TrieMap "mo:base/TrieMap"; -import Nat32 "mo:base/Nat32"; import Text "mo:base/Text"; -import Iter "mo:base/Iter"; import Int "mo:base/Int"; -import Hash "mo:base/Hash"; -import Float "mo:base/Float"; -import Prelude "mo:base/Prelude"; import JSON "mo:json/JSON"; diff --git a/src/JSON/ToText.mo b/src/JSON/ToText.mo index d6ea6a9..f516e76 100644 --- a/src/JSON/ToText.mo +++ b/src/JSON/ToText.mo @@ -1,14 +1,6 @@ -import Array "mo:base/Array"; import Buffer "mo:base/Buffer"; -import Debug "mo:base/Debug"; import Result "mo:base/Result"; -import TrieMap "mo:base/TrieMap"; -import Nat32 "mo:base/Nat32"; import Text "mo:base/Text"; -import Iter "mo:base/Iter"; -import Hash "mo:base/Hash"; -import Float "mo:base/Float"; -import Prelude "mo:base/Prelude"; import JSON "mo:json/JSON"; import NatX "mo:xtended-numbers/NatX"; @@ -82,7 +74,7 @@ module { #Array(Buffer.toArray(newArr)); }; - case (#Record(records)) { + case (#Record(records) or #Map(records)) { let newRecords = Buffer.Buffer<(Text, JSON)>(records.size()); for ((key, val) in records.vals()){ diff --git a/src/JSON/lib.mo b/src/JSON/lib.mo index 934413f..76623e6 100644 --- a/src/JSON/lib.mo +++ b/src/JSON/lib.mo @@ -9,6 +9,7 @@ import Utils "../Utils"; module { public type JSON = JSON.JSON; + public let defaultOptions = Candid.defaultOptions; public let { fromText; toCandid } = FromText; diff --git a/src/UrlEncoded/FromText.mo b/src/UrlEncoded/FromText.mo index 134e1f2..2bbd0ba 100644 --- a/src/UrlEncoded/FromText.mo +++ b/src/UrlEncoded/FromText.mo @@ -6,18 +6,14 @@ import Debug "mo:base/Debug"; import Result "mo:base/Result"; import TrieMap "mo:base/TrieMap"; import Nat "mo:base/Nat"; -import Nat32 "mo:base/Nat32"; import Text "mo:base/Text"; import Iter "mo:base/Iter"; -import Hash "mo:base/Hash"; -import Float "mo:base/Float"; import Option "mo:base/Option"; -import Prelude "mo:base/Prelude"; import Itertools "mo:itertools/Iter"; import Candid "../Candid"; -import CandidTypes "../Candid/Types"; +import T "../Candid/Types"; import { parseValue } "./Parser"; import U "../Utils"; import Utils "../Utils"; @@ -42,20 +38,20 @@ module { func newMap() : NestedTrieMap = TrieMap.TrieMap(Text.equal, Text.hash); /// Converts a Url-Encoded Text to a serialized Candid Record - public func fromText(text : Text, options: ?CandidTypes.Options) : Result { - let res = toCandid(text); + public func fromText(text : Text, options: ?T.Options) : Result { + let res = toCandid(text, Option.get(options, T.defaultOptions)); let #ok(candid) = res else return Utils.send_error(res); Candid.encodeOne(candid, options); }; /// Converts a Url-Encoded Text to a Candid Record - public func toCandid(text : Text) : Result { - let triemap_res = entriesToTrieMap(text); + public func toCandid(text : Text, options: T.Options) : Result { + let triemap_res = entriesToTrieMap(text, options); let #ok(triemap) = triemap_res else return Utils.send_error(triemap_res); - trieMapToCandid(triemap); + trieMapToCandid(triemap, options); }; // Converting entries from UrlSearchParams @@ -88,7 +84,7 @@ module { // }, // } // -------------------------------------------------- - func entriesToTrieMap(text : Text) : Result { + func entriesToTrieMap(text : Text, options: T.Options) : Result { let entries : [Text] = Array.sort( Iter.toArray(Text.split(text, #char '&')), Text.compare, @@ -183,7 +179,7 @@ module { // } // -------------------------------------------------- - func trieMapToCandid(triemap : NestedTrieMap) : Result { + func trieMapToCandid(triemap : NestedTrieMap, options: T.Options) : Result { var i = 0; let isArray = Itertools.all( Iter.sort(triemap.keys(), Text.compare), @@ -205,7 +201,7 @@ module { buffer.add(candid); }; case (?(#triemap(map))) { - let res = trieMapToCandid(map); + let res = trieMapToCandid(map, options); let #ok(candid) = res else return Utils.send_error(res); buffer.add(candid); }; @@ -233,7 +229,7 @@ module { let value_res = switch (value) { case (#text(text)) #ok(parseValue(text)); - case (#triemap(map)) trieMapToCandid(map); + case (#triemap(map)) trieMapToCandid(map, options); }; let #ok(val) = value_res else return Utils.send_error(value_res); @@ -251,7 +247,7 @@ module { buffer.add((key, candid)); }; case (#triemap(map)) { - let res = trieMapToCandid(map); + let res = trieMapToCandid(map, options); let #ok(candid) = res else return Utils.send_error(res); buffer.add((key, candid)); }; @@ -260,6 +256,7 @@ module { let records = Buffer.toArray(buffer); + // let map_or_record = if () #ok(#Record(records)); }; diff --git a/src/UrlEncoded/Parser.mo b/src/UrlEncoded/Parser.mo index 622c994..a78567b 100644 --- a/src/UrlEncoded/Parser.mo +++ b/src/UrlEncoded/Parser.mo @@ -1,5 +1,4 @@ import Char "mo:base/Char"; -import Debug "mo:base/Debug"; import Iter "mo:base/Iter"; import Float "mo:base/Float"; import List "mo:base/List"; @@ -7,10 +6,8 @@ import Nat32 "mo:base/Nat32"; import C "mo:parser-combinators/Combinators"; import P "mo:parser-combinators/Parser"; -import Itertools "mo:itertools/Iter"; import Candid "../Candid"; -import U "../Utils"; module { type Candid = Candid.Candid; diff --git a/src/UrlEncoded/ToText.mo b/src/UrlEncoded/ToText.mo index 8ee9b4b..57119d5 100644 --- a/src/UrlEncoded/ToText.mo +++ b/src/UrlEncoded/ToText.mo @@ -1,20 +1,11 @@ -import Array "mo:base/Array"; -import Blob "mo:base/Blob"; -import Buffer "mo:base/Buffer"; -import Debug "mo:base/Debug"; import Result "mo:base/Result"; import Nat "mo:base/Nat"; -import Int "mo:base/Int"; import Nat32 "mo:base/Nat32"; import Text "mo:base/Text"; import TrieMap "mo:base/TrieMap"; import Iter "mo:base/Iter"; -import Hash "mo:base/Hash"; import Float "mo:base/Float"; -import Order "mo:base/Order"; -import Option "mo:base/Option"; import Principal "mo:base/Principal"; -import Prelude "mo:base/Prelude"; import itertools "mo:itertools/Iter"; @@ -39,7 +30,7 @@ module { public func fromCandid(candid : Candid) : Result { let records = switch (candid) { - case (#Record(records)) records; + case (#Record(records) or #Map(records)) records; case (_) return #err("invalid type: the value must be a record"); }; @@ -82,7 +73,7 @@ module { }; }; - case (#Record(records)) { + case (#Record(records) or #Map(records)) { for ((key, value) in records.vals()) { let record_key = storedKey # "[" # key # "]"; toKeyValuePairs(pairs, record_key, value); diff --git a/src/UrlEncoded/lib.mo b/src/UrlEncoded/lib.mo index 645a0cd..3439b1e 100644 --- a/src/UrlEncoded/lib.mo +++ b/src/UrlEncoded/lib.mo @@ -11,5 +11,6 @@ module { public let { toText; fromCandid } = ToText; public let concatKeys = Utils.concatKeys; + public let defaultOptions = Candid.defaultOptions; }; diff --git a/src/Utils.mo b/src/Utils.mo index cdeb0e9..d92e078 100644 --- a/src/Utils.mo +++ b/src/Utils.mo @@ -84,8 +84,6 @@ module { }; public func text_to_nat32(text : Text) : Nat32 { - var n : Nat32 = 0; - Itertools.fold( text.chars(), 0 : Nat32, diff --git a/src/lib.mo b/src/lib.mo index 9842d3b..6328a34 100644 --- a/src/lib.mo +++ b/src/lib.mo @@ -1,6 +1,3 @@ -import Iter "mo:base/Iter"; - -import Itertools "mo:itertools/Iter"; import CandidTypes "Candid/Types"; import UrlEncodedModule "UrlEncoded"; @@ -22,4 +19,5 @@ module { public let CBOR = CborModule; public let concatKeys = Utils.concatKeys; + public let defaultOptions = CandidTypes.defaultOptions; } \ No newline at end of file diff --git a/tests/Candid.ICRC3.Test.mo b/tests/Candid.ICRC3.Test.mo new file mode 100644 index 0000000..f81a569 --- /dev/null +++ b/tests/Candid.ICRC3.Test.mo @@ -0,0 +1,44 @@ +// @testmode wasi +import Array "mo:base/Array"; +import Blob "mo:base/Blob"; +import Debug "mo:base/Debug"; +import Iter "mo:base/Iter"; +import Principal "mo:base/Principal"; +import Text "mo:base/Text"; + +import { test; suite } "mo:test"; + +import { Candid } "../src"; + +suite( + "Candid ICRC3 compatability Test", + func() { + test( + "record", + func() { + let record = { a = 1; b = 2; }; + + let record_candid_blob = to_candid(record); + + let options = { Candid.defaultOptions with use_icrc_3_value_type = true; }; + let #ok(record_candid) = Candid.decode(record_candid_blob, ["a", "b"], ?options); + + assert record_candid[0] == #Map([ + ("a", #Nat(1)), + ("b", #Nat(2)), + ]); + + let #ok(record_candid_blob2) = Candid.encode(record_candid, ?options); + + assert record_candid_blob == record_candid_blob2; + + let ?record2 : ?{a: Nat; b: Nat;} = from_candid(record_candid_blob2); + + assert record2 == record; + + }, + ); + + + }, +); diff --git a/tests/Candid.Test.mo b/tests/Candid.Test.mo index f18977d..c691006 100644 --- a/tests/Candid.Test.mo +++ b/tests/Candid.Test.mo @@ -246,7 +246,7 @@ let success = run([ do { let motoko = [{ name = "candid"; arr = [1, 2, 3, 4] }, { name = "motoko"; arr = [5, 6, 7, 8] }, { name = "rust"; arr = [9, 10, 11, 12] }]; let blob = to_candid (motoko); - let options = { + let options = { Candid.defaultOptions with renameKeys = [("arr", "array"), ("name", "username")]; }; let candid = Candid.decode(blob, ["name", "arr"], ?options); @@ -613,7 +613,7 @@ let success = run([ daily_downloads : [Nat]; }; - let options = { + let options = { Candid.defaultOptions with renameKeys = [("array", "daily_downloads"), ("name", "language")]; }; let #ok(blob) = Candid.encodeOne(candid, ?options); diff --git a/tests/JSON.Test.mo b/tests/JSON.Test.mo index 626b0b5..44794a9 100644 --- a/tests/JSON.Test.mo +++ b/tests/JSON.Test.mo @@ -155,7 +155,8 @@ let success = run([ }; let text = "{\"label\": 123, \"query\": \"?user_id=12&address=2014%20Forest%20Hill%20Drive\"}"; - let options = { + + let options = { Candid.defaultOptions with renameKeys = [("label", "account_label"), ("query", "user_query")]; }; @@ -260,7 +261,7 @@ let success = run([ }; let UserDataKeys = ["account_label", "user_query"]; - let options = { + let options = { Candid.defaultOptions with renameKeys = [("account_label", "label"), ("user_query", "query")]; }; diff --git a/usage.md b/usage.md index 9ca0d73..c4a2c2d 100644 --- a/usage.md +++ b/usage.md @@ -3,7 +3,8 @@ ### CBOR -```mokoto +```motoko + import { CBOR } "mo:serde"; type User = { @@ -18,6 +19,75 @@ let #ok(cbor) = cbor_res; ``` +#### Candid Variant +```motoko + + import { Candid } "mo:serde"; + + type User = { + name: Text; + id: Nat; + }; + + let candid_variant = #Record([ + ("name", #Text("bar")), + ("id", #Nat(112)) + ]); + + let #ok(blob) = Candid.encode(candid_variant, null); + let user : ?User = from_candid(blob); + + assert user == ?{ name = "bar"; id = 112 }; + +``` + +#### ICRC3 Value +- The [`ICRC3` value type](https://github.com/dfinity/ICRC-1/tree/main/standards/ICRC-3#value) is a representation of candid types in motoko used for sending information without breaking compatibility between canisters that might change their api/data types over time. + +- **Converting from ICRC3 to motoko** +```motoko + import { Candid } "mo:serde"; + + type User = { + name: Text; + id: Nat; + }; + + let candid_variant = #Map([ + ("name", #Text("bar")), + ("id", #Nat(112)) + ]); + + let options = { Candid.defaultOptions with use_icrc_3_value_type = true }; + let #ok(blob) = Candid.encode(candid_variant, ?Options); + let user : ?User = from_candid(blob); + + assert user == ?{ name = "bar"; id = 112 }; + +``` + +- **Converting from motoko to ICRC3** +```motoko + import { Candid } "mo:serde"; + + type User = { + name: Text; + id: Nat; + }; + + let user : User = { name = "bar"; id = 112 }; + + let blob = to_candid(user); + let options = { Candid.defaultOptions with use_icrc_3_value_type = true }; + let icrc3 = Candid.encode(blob, ?options); + + assert icrc3 == #Map([ + ("name", #Text("bar")), + ("id", #Nat(112)) + ]); + +``` + ### Candid Text ```motoko import { Candid } "mo:serde"; @@ -34,6 +104,7 @@ ``` + ### URL-Encoded Pairs Serialization and deserialization for `application/x-www-form-urlencoded`.