From 1360636dbc19c62124a4effc7861b549fa9db9bd Mon Sep 17 00:00:00 2001 From: Tomi Jaga Date: Fri, 29 Dec 2023 02:38:41 -0500 Subject: [PATCH] [Feat]: Add support for converting between candid and CBOR --- .github/workflows/makefile.yml | 10 +- benchmarks/serde.bench.mo | 89 ++++++++++++++- mops.toml | 15 +-- readme.md | 29 ++++- src/CBOR/lib.mo | 191 +++++++++++++++++++++++++++++++++ src/lib.mo | 5 +- tests/CBOR.Test.mo | 110 +++++++++++++++++++ 7 files changed, 427 insertions(+), 22 deletions(-) create mode 100644 src/CBOR/lib.mo create mode 100644 tests/CBOR.Test.mo diff --git a/.github/workflows/makefile.yml b/.github/workflows/makefile.yml index 28fbaad..d701864 100644 --- a/.github/workflows/makefile.yml +++ b/.github/workflows/makefile.yml @@ -24,21 +24,19 @@ jobs: curl https://wasmtime.dev/install.sh -sSf | bash echo "$HOME/.wasmtime/bin" >> $GITHUB_PATH - - name: Install mocv - run: | npm --yes -g i mocv - name: Select mocv version - run: mocv use 0.10.0 + run: mocv use 0.10.3 - name: install mops run: | - npm --yes -g i ic-mops + npm --yes -g i ic-mops@0.34.3 mops i mops sources - - name: Detect Warnings - run: make no-warn + # - name: Detect Warnings + # run: make no-warn - name: Run Tests run: mops test diff --git a/benchmarks/serde.bench.mo b/benchmarks/serde.bench.mo index d97a514..37911e9 100644 --- a/benchmarks/serde.bench.mo +++ b/benchmarks/serde.bench.mo @@ -30,13 +30,96 @@ actor { public query func cycles() : async Nat { Cycles.balance()}; - public query func deserialize(n: Nat) : async (Nat64, Nat, Nat, Nat) { + type Benchmark = { + calls: Nat64; + heap: Nat; + memory: Nat; + cycles: Nat; + }; + + func benchmark(fn: () -> ()): Benchmark { let init_cycles = Cycles.balance(); let init_heap = Prim.rts_heap_size(); let init_memory = Prim.rts_memory_size(); - let calls = IC.countInstructions( + let calls = IC.countInstructions(fn); + + { + calls; + heap = Prim.rts_heap_size() - init_heap; + memory = Prim.rts_memory_size() - init_memory; + cycles = init_cycles - Cycles.balance(); + } + }; + + public query func serialize(n: Nat): async Benchmark { + benchmark( + func(){ + let admin_record : Record = { + group = "admins"; + users = ?[{ + name = "John"; + age = 32; + permission = #admin; + }]; + }; + + let user_record : Record = { + group = "users"; + users = ?[{ + name = "Ali"; + age = 28; + permission = #read_all; + }, { + name = "James"; + age = 40; + permission = #write_all; + }]; + }; + + let empty_record : Record = { + group = "empty"; + users = ?[]; + }; + + let null_record : Record = { + group = "null"; + users = null; + }; + + let base_record : Record = { + group = "base"; + users = ?[{ + name = "Henry"; + age = 32; + permission = #read(["posts", "comments"]); + }, { + name = "Steven"; + age = 32; + permission = #write(["posts", "comments"]); + }]; + }; + + let records : [Record] = [ + null_record, + empty_record, + admin_record, + user_record, + base_record, + ]; + + for (_ in Iter.range(1, n)){ + let blob = to_candid (records); + let #ok(candid) = Candid.decode(blob, [], null); + Debug.print(debug_show (candid)); + }; + } + ) + }; + + public query func deserialize(n: Nat) : async Benchmark { + benchmark( func() { let admin_record_candid : Candid = #Record([ @@ -79,8 +162,6 @@ actor { }; } ); - - (calls, Prim.rts_heap_size() - init_heap, Prim.rts_memory_size() - init_memory, init_cycles - Cycles.balance()) }; }; diff --git a/mops.toml b/mops.toml index 2e922de..11e818e 100644 --- a/mops.toml +++ b/mops.toml @@ -1,15 +1,18 @@ [package] name = "serde" -version = "2.0.4" +version = "2.1.0" description = "A serialisation and deserialisation library for Motoko." repository = "https://github.com/NatLabs/serde" -keywords = [ "json", "candid", "parser", "urlencoded", "serialization" ] +keywords = [ "json", "candid", "cbor", "urlencoded", "serialization" ] [dependencies] -base = "0.10.0" +base = "0.10.3" itertools = "0.1.2" candid = "1.0.2" xtended-numbers = "0.2.1" -json = "https://github.com/aviate-labs/json.mo#v0.2.0" -parser-combinators = "https://github.com/aviate-labs/parser-combinators.mo#v0.1.2" -map = "9.0.0" \ No newline at end of file +json = "https://github.com/aviate-labs/json.mo#v0.2.0@99ae3761c09622a98ae45ebd9ad68f01353df336" +parser-combinators = "https://github.com/aviate-labs/parser-combinators.mo#v0.1.2@6a331bf78e9dcd7623977f06c8e561fd1a8c0103" +cbor = "0.1.3" + +[dev-dependencies] +test = "1.2.0" \ No newline at end of file diff --git a/readme.md b/readme.md index 0c39469..8209cdb 100644 --- a/readme.md +++ b/readme.md @@ -2,11 +2,30 @@ An efficient serialization and deserialization library for Motoko. - +The library contains four modules: +- **Candid** + - `fromText()` - Converts [Candid text](https://internetcomputer.org/docs/current/tutorials/developer-journey/level-2/2.4-intro-candid/#candid-textual-values) to its serialized form. + - `toText()` - Converts serialized candid to its [textual representation](https://internetcomputer.org/docs/current/tutorials/developer-journey/level-2/2.4-intro-candid/#candid-textual-values). + - `encode()` - Converts the [Candid variant](./src/Candid/Types.mo#L6) to a blob. + - `decode()` - Converts a blob to the [Candid variant](./src/Candid/Types.mo#L6). + +- **CBOR** + - `encode()` - Converts serialized candid to CBOR. + - `decode()` - Converts CBOR to a serialized candid. + +- **JSON** + - `fromText()` - Converts JSON text to serialized candid. + - `toText()` - Converts serialized candid to JSON text. + +- **URL-Encoded Pairs** + - `fromText()` - Converts URL-encoded text to serialized candid. + - `toText()` - Converts serialized candid to URL-encoded text. + ## Getting Started -### Installation +### Installation +[![mops](https://oknww-riaaa-aaaam-qaf6a-cai.raw.ic0.app/badge/mops/serde)](https://mops.one/serde) 1. Install [`mops`](https://j4mwm-bqaaa-aaaam-qajbq-cai.ic0.app/#/docs/install). 2. Inside your project directory, run: @@ -18,15 +37,15 @@ mops install serde To start, import the necessary modules: ```motoko -import { JSON; Candid; UrlEncoded } from "mo:serde"; +import { JSON; Candid; CBOR; UrlEncoded } from "mo:serde"; ``` #### JSON -> The API for all serde module is the same, so the following code can be used for converting data between the other modules (Candid and URL-Encoded Pairs). +> The following code can be used for converting data between the other modules (Candid and URL-Encoded Pairs). **Example: JSON to Motoko** -1. **Defining Data Type**: This critical step informs the conversion functions (`from_candid`` and `to_candid`) about how to handle the data. +1. **Defining Data Type**: This critical step informs the conversion functions (`from_candid` and `to_candid`) about how to handle the data. Consider the following JSON data: ```json diff --git a/src/CBOR/lib.mo b/src/CBOR/lib.mo new file mode 100644 index 0000000..e036db0 --- /dev/null +++ b/src/CBOR/lib.mo @@ -0,0 +1,191 @@ +import Buffer "mo:base/Buffer"; +import Blob "mo:base/Blob"; +import Int8 "mo:base/Int8"; +import Int16 "mo:base/Int16"; +import Int32 "mo:base/Int32"; +import Int64 "mo:base/Int64"; +import Nat64 "mo:base/Nat64"; +import Result "mo:base/Result"; + +import CBOR_Value "mo:cbor/Value"; +import CBOR_Encoder "mo:cbor/Encoder"; +import CBOR_Decoder "mo:cbor/Decoder"; +import NatX "mo:xtended-numbers/NatX"; +import IntX "mo:xtended-numbers/IntX"; +import FloatX "mo:xtended-numbers/FloatX"; + +import Candid "../Candid"; +import CandidTypes "../Candid/Types"; + +import Utils "../Utils"; + +module { + public type Candid = CandidTypes.Candid; + type Result = Result.Result; + type CBOR = CBOR_Value.Value; + + /// Converts serialized Candid blob to CBOR blob + public func encode(blob : Blob, keys : [Text], options: ?CandidTypes.Options) : Result { + let decoded_res = Candid.decode(blob, keys, options); + let #ok(candid) = decoded_res else return Utils.send_error(decoded_res); + + let json_res = fromCandid(candid[0]); + let #ok(json) = json_res else return Utils.send_error(json_res); + #ok(json); + }; + + /// Convert a Candid value to CBOR blob + public func fromCandid(candid : Candid) : Result { + let res = transpile_candid_to_cbor(candid); + let #ok(transpiled_cbor) = res else return Utils.send_error(res); + + switch(CBOR_Encoder.encode(transpiled_cbor)){ + case(#ok(encoded_cbor)){ #ok (Blob.fromArray(encoded_cbor))}; + case(#err(#invalidValue(errMsg))){ #err("Invalid value error while encoding CBOR: " # errMsg) }; + }; + }; + + func transpile_candid_to_cbor(candid : Candid) : Result { + let transpiled_cbor : CBOR = switch(candid){ + case (#Empty) #majorType7(#_undefined); + case (#Null) #majorType7(#_null); + case (#Bool(n)) #majorType7(#bool(n)); + case (#Float(n)) #majorType7(#float(FloatX.fromFloat(n, #f64))); + + case (#Nat8(n)) #majorType7(#integer(n)); + case (#Nat16(n)) #majorType0(NatX.from16To64(n)); + case (#Nat32(n)) #majorType0(NatX.from32To64(n)); + case (#Nat64(n)) #majorType0(n); + case (#Nat(n)) #majorType0(Nat64.fromNat(n)); + + case (#Int8(n)) #majorType1(Int8.toInt(n)); + case (#Int16(n)) #majorType1(Int16.toInt(n)); + case (#Int32(n)) #majorType1(Int32.toInt(n)); + case (#Int64(n)) #majorType1(Int64.toInt(n)); + case (#Int(n)) #majorType1(n); + + case (#Blob(blob)) #majorType2(Blob.toArray(blob)); + case (#Text(n)) #majorType3(n); + case (#Array(arr)) { + let buffer = Buffer.Buffer(arr.size()); + + for (item in arr.vals()){ + let res = transpile_candid_to_cbor(item); + let #ok(cbor_val) = res else return Utils.send_error(res); + buffer.add(cbor_val); + }; + + #majorType4(Buffer.toArray(buffer)); + }; + case (#Record(records)) { + let newRecords = Buffer.Buffer<(CBOR, CBOR)>(records.size()); + + for ((key, val) in records.vals()){ + let res = transpile_candid_to_cbor(val); + let #ok(cbor_val) = res else return Utils.send_error(res); + newRecords.add((#majorType3(key), cbor_val)); + }; + + #majorType5(Buffer.toArray(newRecords)); + }; + + // Candid can make variables optional, when it is decoded using + // `from_candid` if its specified in the type defination + // This features allow us to handle optional values when decoding CBOR + // + // check out "CBOR Tests.options" in the tests folder to see how this in action + case (#Option(option)) { + let res = transpile_candid_to_cbor(option); + let #ok(cbor_val) = res else return Utils.send_error(res); + cbor_val + }; + + case (#Variant(_)) { + return #err("#Variant(_) is not supported in this implementation of CBOR"); + }; + + case (#Principal(_)){ + return #err("#Principal(_) is not supported in this implementation of CBOR"); + }; + }; + + // add self-describe tag + let tagged_cbor = #majorType6({ tag = 55799 : Nat64; value = transpiled_cbor; }); + + #ok(tagged_cbor); + }; + + public func decode(blob: Blob, options: ?CandidTypes.Options): Result { + let candid_res = toCandid(blob); + let #ok(candid) = candid_res else return Utils.send_error(candid_res); + Candid.encodeOne(candid, options); + }; + + public func toCandid(blob: Blob): Result { + let cbor_res = CBOR_Decoder.decode(blob); + + let candid_res = switch (cbor_res) { + case (#ok(cbor)) transpile_cbor_to_candid(cbor); + case (#err(cbor_error)) { + switch(cbor_error){ + case (#unexpectedBreak){ return #err("Error decoding CBOR: Unexpected break") }; + case (#unexpectedEndOfBytes) { return #err("Error decoding CBOR: Unexpected end of bytes") }; + case (#invalid(errMsg)) { return #err("Invalid CBOR: " # errMsg) }; + }; + }; + }; + + let #ok(candid) = candid_res else return Utils.send_error(candid_res); + #ok(candid); + }; + + public func transpile_cbor_to_candid(cbor: CBOR) : Result{ + let transpiled_candid = switch(cbor){ + case (#majorType0(n)) #Nat(Nat64.toNat(n)); + case (#majorType1(n)) #Int(n); + case (#majorType2(n)) #Blob(Blob.fromArray(n)); + case (#majorType3(n)) #Text(n); + case (#majorType4(arr)) { + let buffer = Buffer.Buffer(arr.size()); + for (item in arr.vals()){ + let res = transpile_cbor_to_candid(item); + let #ok(candid_val) = res else return Utils.send_error(res); + buffer.add(candid_val); + }; + #Array(Buffer.toArray(buffer)); + }; + case (#majorType5(records)) { + let buffer = Buffer.Buffer<(Text, Candid)>(records.size()); + for ((cbor_text, val) in records.vals()){ + let #majorType3(key) = cbor_text else return #err("Error decoding CBOR: Unexpected key type"); + + let res = transpile_cbor_to_candid(val); + let #ok(candid_val) = res else return Utils.send_error(res); + buffer.add((key, candid_val)); + }; + + #Record(Buffer.toArray(buffer)); + }; + case (#majorType7(#_undefined)) #Empty; + case (#majorType7(#_null)) #Null; + case (#majorType7(#bool(n))) #Bool(n); + case (#majorType7(#integer(n))) #Nat8(n); + case (#majorType7(#float(n))) #Float(FloatX.toFloat(n)); + + case (#majorType7(#_break)) { + return #err("Error decoding CBOR: #_break is not supported"); + }; + case (#majorType6(tagged_cbor)) { + if (tagged_cbor.tag == 55799){ + let transpiled_candid_res = transpile_cbor_to_candid(tagged_cbor.value); + let #ok(transpiled_candid) = transpiled_candid_res else return Utils.send_error(transpiled_candid_res); + transpiled_candid + } else { + return #err("Error decoding CBOR: Tagged values are not supported"); + }; + }; + }; + + #ok(transpiled_candid); + }; +} \ No newline at end of file diff --git a/src/lib.mo b/src/lib.mo index 2258fa5..9842d3b 100644 --- a/src/lib.mo +++ b/src/lib.mo @@ -6,6 +6,8 @@ import CandidTypes "Candid/Types"; import UrlEncodedModule "UrlEncoded"; import JsonModule "JSON"; import CandidModule "Candid"; +import CborModule "CBOR"; + import Utils "Utils"; module { @@ -13,10 +15,11 @@ module { public type Options = CandidTypes.Options; public type Candid = CandidTypes.Candid; - public let Candid = CandidModule; + public let JSON = JsonModule; public let URLEncoded = UrlEncodedModule; + public let CBOR = CborModule; public let concatKeys = Utils.concatKeys; } \ No newline at end of file diff --git a/tests/CBOR.Test.mo b/tests/CBOR.Test.mo new file mode 100644 index 0000000..3fd0305 --- /dev/null +++ b/tests/CBOR.Test.mo @@ -0,0 +1,110 @@ +// @testmode wasi +import Debug "mo:base/Debug"; +import Iter "mo:base/Iter"; +import { test; suite } "mo:test"; + +import { CBOR } "../src"; + +suite( + "CBOR Test", + func() { + test("options", func() { + let opt_nat_null: ?Nat = null; + let opt_nat : ?Nat = ?123; + let opt_text_null: ?Text = null; + let opt_text : ?Text = ?"hello"; + + let opt_nat_null_candid = to_candid(opt_nat_null); + let opt_nat_candid = to_candid(opt_nat); + let opt_text_null_candid = to_candid(opt_text_null); + let opt_text_candid = to_candid(opt_text); + + let #ok(opt_nat_null_cbor) = CBOR.encode(opt_nat_null_candid, [], null); + let #ok(opt_nat_cbor) = CBOR.encode(opt_nat_candid, [], null); + let #ok(opt_text_null_cbor) = CBOR.encode(opt_text_null_candid, [], null); + let #ok(opt_text_cbor) = CBOR.encode(opt_text_candid, [], null); + + let #ok(opt_nat_null_candid2) = CBOR.decode(opt_nat_null_cbor, null); + let #ok(opt_nat_candid2) = CBOR.decode(opt_nat_cbor, null); + let #ok(opt_text_null_candid2) = CBOR.decode(opt_text_null_cbor, null); + let #ok(opt_text_candid2) = CBOR.decode(opt_text_cbor, null); + + assert opt_nat_null_candid != opt_nat_null_candid2; + assert opt_nat_candid != opt_nat_candid2; + assert opt_text_null_candid != opt_text_null_candid2; + assert opt_text_candid != opt_text_candid2; + + let ?opt_nat_null2 : ?(?Nat) = from_candid(opt_nat_null_candid2); + let ?opt_nat2 : ?(?Nat) = from_candid(opt_nat_candid2); + let ?opt_text_null2 : ?(?Text) = from_candid(opt_text_null_candid2); + let ?opt_text2 : ?(?Text) = from_candid(opt_text_candid2); + + assert opt_nat_null2 == opt_nat_null; + assert opt_nat2 == opt_nat; + assert opt_text_null2 == opt_text_null; + assert opt_text2 == opt_text; + + }); + test( + "primitives", + func() { + + let nat : Nat = 123; + let int : Int = -123; + let float : Float = 123.456; + let bool : Bool = true; + let text: Text = "hello"; + let blob: Blob = "\01\02\03"; + let _null: Null = null; + let empty = (); + let list: [Nat] = [1, 2, 3]; + let record = { a = 1; b = 2; }; + + let nat_candid = to_candid(nat); + let int_candid = to_candid(int); + let float_candid = to_candid(float); + let bool_candid = to_candid(bool); + let text_candid = to_candid(text); + let blob_candid = to_candid(blob); + let null_candid = to_candid(_null); + let empty_candid = to_candid(empty); + let list_candid = to_candid(list); + let record_candid = to_candid(record); + + let #ok(nat_cbor) = CBOR.encode(nat_candid, [], null); + let #ok(int_cbor) = CBOR.encode(int_candid, [], null); + let #ok(float_cbor) = CBOR.encode(float_candid, [], null); + let #ok(bool_cbor) = CBOR.encode(bool_candid, [], null); + let #ok(text_cbor) = CBOR.encode(text_candid, [], null); + let #ok(blob_cbor) = CBOR.encode(blob_candid, [], null); + let #ok(null_cbor) = CBOR.encode(null_candid, [], null); + let #ok(empty_cbor) = CBOR.encode(empty_candid, [], null); + let #ok(list_cbor) = CBOR.encode(list_candid, [], null); + let #ok(record_cbor) = CBOR.encode(record_candid, ["a", "b"], null); + + let #ok(nat_candid2) = CBOR.decode(nat_cbor, null); + let #ok(int_candid2) = CBOR.decode(int_cbor, null); + let #ok(float_candid2) = CBOR.decode(float_cbor, null); + let #ok(bool_candid2) = CBOR.decode(bool_cbor, null); + let #ok(text_candid2) = CBOR.decode(text_cbor, null); + let #ok(blob_candid2) = CBOR.decode(blob_cbor, null); + let #ok(null_candid2) = CBOR.decode(null_cbor, null); + let #ok(empty_candid2) = CBOR.decode(empty_cbor, null); + let #ok(list_candid2) = CBOR.decode(list_cbor, null); + let #ok(record_candid2) = CBOR.decode(record_cbor, null); + + assert nat_candid == nat_candid2; + assert int_candid == int_candid2; + assert float_candid == float_candid2; + assert bool_candid == bool_candid2; + assert text_candid == text_candid2; + assert blob_candid == blob_candid2; + assert null_candid == null_candid2; + assert empty_candid == empty_candid2; + assert list_candid == list_candid2; + assert record_candid == record_candid2; + + }, + ); + }, +);