diff --git a/Manifest.toml b/Manifest.toml index 9bd0e32..feaddd3 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.9.0" manifest_format = "2.0" -project_hash = "9438c2e78a398947c9977dd003134cd71965f264" +project_hash = "87b470058e1cefd35fc06bc6f3913025a7740d82" [[deps.ADTypes]] git-tree-sha1 = "dcfdf328328f2645531c4ddebf841228aef74130" @@ -21,12 +21,6 @@ git-tree-sha1 = "6252039f98492252f9e47c312c8ffda0e3b9e78d" uuid = "ae81ac8f-d209-56e5-92de-9978fef736f9" version = "0.1.3+0" -[[deps.AWS]] -deps = ["Base64", "Compat", "Dates", "Downloads", "GitHub", "HTTP", "IniFile", "JSON", "MbedTLS", "Mocking", "OrderedCollections", "Random", "SHA", "Sockets", "URIs", "UUIDs", "XMLDict"] -git-tree-sha1 = "2b82cf89e8a028dec40fcd6f6d9c0406a25907d7" -uuid = "fbe9abb3-538b-5e4e-ba9e-bc94f4f92ebc" -version = "1.87.0" - [[deps.AbstractAlgebra]] deps = ["GroupsCore", "InteractiveUtils", "LinearAlgebra", "MacroTools", "Random", "RandomExtensions", "SparseArrays", "Test"] git-tree-sha1 = "3ee5c58774f4487a5bf2bb05e39d91ff5022b4cc" @@ -730,12 +724,6 @@ git-tree-sha1 = "c1d06d129da9f55715c6c212866f5b1bddc5fa00" uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04" version = "0.1.9" -[[deps.EzXML]] -deps = ["Printf", "XML2_jll"] -git-tree-sha1 = "0fa3b52a04a4e210aeb1626def9c90df3ae65268" -uuid = "8f5d6c58-4d21-5cfd-889c-e3ad7ee6a615" -version = "1.1.0" - [[deps.FFMPEG]] deps = ["FFMPEG_jll"] git-tree-sha1 = "b57e3acbe22f8484b4b5ff66a7499717fe1a9cc8" @@ -919,12 +907,6 @@ git-tree-sha1 = "9b02998aba7bf074d14de89f9d37ca24a1a0b046" uuid = "78b55507-aeef-58d4-861c-77aaff3498b1" version = "0.21.0+0" -[[deps.GitHub]] -deps = ["Base64", "Dates", "HTTP", "JSON", "MbedTLS", "Sockets", "SodiumSeal", "URIs"] -git-tree-sha1 = "5688002de970b9eee14b7af7bbbd1fdac10c9bbe" -uuid = "bc5e4493-9b4d-5f90-b8aa-2b2bcaad7a26" -version = "5.8.2" - [[deps.Glib_jll]] deps = ["Artifacts", "Gettext_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Libiconv_jll", "Libmount_jll", "PCRE2_jll", "Pkg", "Zlib_jll"] git-tree-sha1 = "d3b3624125c1474292d0d8ed0f65554ac37ddb23" @@ -1023,11 +1005,6 @@ git-tree-sha1 = "5cd07aab533df5170988219191dfad0519391428" uuid = "d25df0c9-e2be-5dd7-82c8-3ad0b3e990b9" version = "0.1.3" -[[deps.IniFile]] -git-tree-sha1 = "f550e6e32074c939295eb5ea6de31849ac2c9625" -uuid = "83e8ac13-25f8-5344-8a64-a9f2b223428f" -version = "0.5.1" - [[deps.InitialValues]] git-tree-sha1 = "4da0f88e9a39111c2fa3add390ab15f3a44f3ca3" uuid = "22cec73e-a1b8-11e9-2c92-598750a2cf9c" @@ -2292,12 +2269,6 @@ version = "1.5.0" [[deps.Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" -[[deps.SodiumSeal]] -deps = ["Base64", "Libdl", "libsodium_jll"] -git-tree-sha1 = "80cef67d2953e33935b41c6ab0a178b9987b1c99" -uuid = "2133526b-2bfb-4018-ac12-889fb3908a75" -version = "0.1.1" - [[deps.SortingAlgorithms]] deps = ["DataStructures"] git-tree-sha1 = "a4ada03f999bd01b3a25dcaa30b2d929fe537e00" @@ -2738,12 +2709,6 @@ git-tree-sha1 = "93c41695bc1c08c46c5899f4fe06d6ead504bb73" uuid = "02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a" version = "2.10.3+0" -[[deps.XMLDict]] -deps = ["EzXML", "IterTools", "OrderedCollections"] -git-tree-sha1 = "d9a3faf078210e477b291c79117676fca54da9dd" -uuid = "228000da-037f-5747-90a9-8195ccbf91a5" -version = "0.4.1" - [[deps.XSLT_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgcrypt_jll", "Libgpg_error_jll", "Libiconv_jll", "Pkg", "XML2_jll", "Zlib_jll"] git-tree-sha1 = "91844873c4085240b95e795f692c4cec4d805f8a" @@ -2946,12 +2911,6 @@ git-tree-sha1 = "94d180a6d2b5e55e447e2d27a29ed04fe79eb30c" uuid = "b53b4c65-9356-5827-b1ea-8c7a1a84506f" version = "1.6.38+0" -[[deps.libsodium_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "848ab3d00fe39d6fbc2a8641048f8f272af1c51e" -uuid = "a9144af2-ca23-56d9-984f-0d03f7b5ccf8" -version = "1.0.20+0" - [[deps.libvorbis_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Ogg_jll", "Pkg"] git-tree-sha1 = "b910cb81ef3fe6e78bf6acee440bda86fd6ae00c" diff --git a/Project.toml b/Project.toml index ca8a96e..77f7af5 100644 --- a/Project.toml +++ b/Project.toml @@ -5,7 +5,6 @@ version = "0.6.0" [deps] AMQPClient = "79c8b4cd-a41a-55fa-907c-fab5288e1383" -AWS = "fbe9abb3-538b-5e4e-ba9e-bc94f4f92ebc" AlgebraicPetri = "4f99eebe-17bf-4e98-b6a1-2c4f205a959b" Bijections = "e2ed5e7c-b2de-5872-ae92-c73ca462fb04" CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" diff --git a/docker/Dockerfile.api b/docker/Dockerfile.api index 3528def..446825b 100644 --- a/docker/Dockerfile.api +++ b/docker/Dockerfile.api @@ -11,6 +11,7 @@ RUN julia -e 'using Pkg; Pkg.instantiate();' COPY src/ /simulation-service/src/ RUN julia -e 'using Pkg; Pkg.resolve();' +COPY paths.yaml /simulation-service/ # Launch simulation-service EXPOSE 8080 CMD [ "julia", "--threads", "4", "-e", "using SimulationService; SimulationService.start!(); while true sleep(10000) end" ] \ No newline at end of file diff --git a/examples/dataset.csv b/examples/dataset.csv new file mode 100644 index 0000000..d8818ec --- /dev/null +++ b/examples/dataset.csv @@ -0,0 +1,92 @@ +timestep,S,E,I,R,D +0.0,0.49457800495224524,0.26745259325403603,0.4497387877393193,0.32807705995998604,0.8545934885162726 +1.0,0.4665022246157727,0.15226661609143755,0.282128632173281,0.5819467770808255,0.9115956844605425 +2.0,0.44995568708559147,0.0870753734907285,0.17176626841178605,0.7388229924583731,0.9468196129753799 +3.0,0.4402579969265885,0.049958082609506564,0.10268280170774824,0.8334700316632883,0.9680710215147275 +4.0,0.4345928780439725,0.02873264900876653,0.060681564306912315,0.8897296629609703,0.9807031801012375 +5.0,0.43129133897514976,0.016554961390151432,0.03559634907740039,0.9228561146017267,0.9881411703774309 +6.0,0.42937079614049717,0.00955084027561801,0.020781137104605475,0.9422429917414922,0.9924941691596463 +7.0,0.42825523174624236,0.005515226366193871,0.012094002808321005,0.9535438792052197,0.9950315942958823 +8.0,0.42760803965578864,0.003187203614737658,0.007024405738798128,0.9601135757214116,0.996506709691123 +9.0,0.42723286848143166,0.0018428391922203655,0.004074861364167079,0.9639265223459347,0.9973628430381053 +10.0,0.42701542110237617,0.001065625665672367,0.002361473964384682,0.966138016624911,0.997859397064515 +11.0,0.42688949053651354,0.0006163374574758091,0.0013677486708266896,0.9674192754645957,0.9981470822924473 +12.0,0.4268166164339979,0.0003566562089632792,0.0007921677242322537,0.968160893781643,0.9983136002730226 +13.0,0.426774399020502,0.00020632531989352596,0.0004585439285171233,0.9685905857420082,0.9984100804109383 +14.0,0.4267499641987114,0.00011936317441910912,0.00026537658614250037,0.9688393044325163,0.9984659260300698 +15.0,0.4267358390828414,6.911064486184507e-5,0.00015368573473018547,0.9689830886008763,0.9984982103585492 +16.0,0.42672764295323584,3.995637251879534e-5,8.886936321230337e-5,0.9690665218374893,0.9985169438954028 +17.0,0.42672291793839023,2.315252702782855e-5,5.149885891478391e-5,0.969114621277837,0.9985277438196892 +18.0,0.42672016837248206,1.3374421742842271e-5,2.975169371631378e-5,0.9691426114006284,0.9985340285332894 +19.0,0.4267185903383033,7.76324275234401e-6,1.72697551963783e-5,0.96915867560426,0.9985376354813472 +20.0,0.42671766524571453,4.473700806675521e-6,9.952514996761569e-6,0.9691680929676613,0.9985397499926799 +21.0,0.4267171396866097,2.605039389810057e-6,5.795292710617236e-6,0.9691734431227009,0.9985409512804482 +22.0,0.4267168292213564,1.5011076626621485e-6,3.339530654332321e-6,0.9691766036403886,0.9985416609217972 +23.0,0.4267166503308677,8.650367137104304e-7,1.9244997328791993e-6,0.9691784247361698,0.998542069818375 +24.0,0.4267165513975358,5.133018515249044e-7,1.1418963787668045e-6,0.9691794318721828,0.9985422959539103 +25.0,0.42671648849345345,2.8961248029392266e-7,6.443470300592253e-7,0.9691800722327397,0.9985424397361555 +26.0,0.4267164534150905,1.6488872179188643e-7,3.668730947050761e-7,0.9691804293288433,0.9985425199161089 +27.0,0.4267164363527159,1.0425224374051472e-7,2.3187776609803832e-7,0.9691806030229255,0.9985425589162079 +28.0,0.4267164237650669,5.94635822904114e-8,1.323403435661362e-7,0.969180731164592,0.9985425876882744 +29.0,0.42671641532016785,2.938997840309349e-8,6.558731960133238e-8,0.9691808171332872,0.9985426069911061 +30.0,0.4267164119320277,1.7383990307292018e-8,3.8745941399490847e-8,0.9691808516243932,0.9985426147355065 +31.0,0.42671641142993344,1.567980166379535e-8,3.4693296446806386e-8,0.9691808567356694,0.9985426158831582 +32.0,0.4267164097239014,9.397997811086657e-9,2.141426794359566e-8,0.9691808741029923,0.9985426197826998 +33.0,0.4267164077513253,1.6630451169718799e-9,6.5322673177872115e-9,0.9691808941837284,0.998542624291493 +34.0,0.4267164068327058,-1.7144298561756136e-9,-6.228678598180968e-10,0.969180903535234,0.998542626391217 +35.0,0.42671640713709813,1.995841690399913e-10,9.531889471946351e-10,0.9691809004365325,0.9985426256954555 +36.0,0.4267164080167858,4.441082888563269e-9,6.797923212261653e-9,0.9691808914813484,0.9985426236847188 +37.0,0.42671640835547514,3.2138485814499743e-9,1.190846777114169e-8,0.9691808880335046,0.9985426229105632 +38.0,0.42671641412844163,-2.6272280610911653e-8,1.0758565137248983e-7,0.9691808292649736,0.9985426097150731 +39.0,0.42671642043471775,-5.7124386784960156e-8,2.107435751597396e-7,0.9691807650673759,0.9985425953005771 +40.0,0.42671642069465143,-6.101555879931584e-8,2.1761506634755097e-7,0.9691807624212634,0.9985425947064368 +41.0,0.42671641308224717,-2.98748098503364e-8,9.919283597417899e-8,0.9691808399151859,0.9985426121064 +42.0,0.42671640052516574,2.41129225278762e-8,-9.87705210985136e-8,0.9691809677456914,0.9985426408086006 +43.0,0.4267163907047244,6.850677646347266e-8,-2.557625261850519e-7,0.9691810677173265,0.998542663255558 +44.0,0.42671639824152524,4.377942441187322e-8,-1.4462071904623285e-7,0.9691809909931911,0.9985426460284376 +45.0,0.4267165177761834,-3.5045943998565953e-7,1.6201571649468285e-6,0.9691797741428834,0.9985423728050674 +46.0,0.426716582914649,-5.657635155369028e-7,2.5823127375500367e-6,0.9691791110412861,0.9985422239167021 +47.0,0.426716469373307,-1.9999268662048253e-7,9.147186185214705e-7,0.9691802668813723,0.9985424834412481 +48.0,0.4267162674880653,4.53529758154388e-7,-2.0535417063955472e-6,0.9691823220506339,0.9985429448951083 +49.0,0.42671627788979505,4.201179072135318e-7,-1.900860288767501e-6,0.9691822161561675,0.9985429211182781 +50.0,0.42671649186340843,-2.7150027993614184e-7,1.2440747380756681e-6,0.9691800379462864,0.9985424320377062 +51.0,0.4267164371442758,-9.590264835717333e-8,4.410989446171615e-7,0.9691805949726369,0.9985425571086503 +52.0,0.42671627454943717,4.2799006477805363e-7,-1.9470013902130833e-6,0.9691822501358911,0.9985429287478564 +53.0,0.4267164118674504,-1.515429881156514e-8,7.058019617525471e-8,0.9691808522520705,0.998542614876441 +54.0,0.4267164314292611,-7.842218520117033e-8,3.5813674655576225e-7,0.9691806531145852,0.9985425701634516 +55.0,0.4267163402967335,2.1536057882219336e-7,-9.805249063054177e-7,0.9691815808243099,0.9985427784651432 +56.0,0.4267164064309929,2.026849024044358e-9,-8.917426690141202e-9,0.969180907581668,0.998542627299776 +57.0,0.4267164283592435,-6.872496312372469e-8,3.1325564224967763e-7,0.969180684354161,0.9985425771777755 +58.0,0.4267163726304823,1.1096730955813961e-7,-5.053996636778452e-7,0.9691812516656547,0.9985427045580763 +59.0,0.4267164022669418,1.5383256861379745e-8,-7.001281739503044e-8,0.9691809499676411,0.9985426368168369 +60.0,0.4267164303322644,-7.51289506815351e-8,3.42286373147847e-7,0.9691806642650668,0.9985425726671056 +61.0,0.42671639252407795,4.678946740843496e-8,-2.1312670437383657e-7,0.9691810491487283,0.9985426590862899 +62.0,0.4267163906803671,5.2732295751289435e-8,-2.40207629435881e-7,0.9691810679165382,0.9985426633002876 +63.0,0.42671643093445266,-7.707951798376247e-8,3.51143486333012e-7,0.9691806581331516,0.9985425712902867 +64.0,0.4267164128814658,-1.8863795872089294e-8,8.593863522149074e-8,0.9691808419110224,0.9985426125545317 +65.0,0.42671637686662756,9.727501741359907e-8,-4.4313284520207667e-7,0.9691812085384706,0.9985426948745889 +66.0,0.4267164200453857,-4.196688650706194e-8,1.9118201954589875e-7,0.9691807689818369,0.9985425961795036 +67.0,0.42671643358050243,-8.561468651784963e-8,3.900185546253598e-7,0.9691806311955823,0.9985425652419063 +68.0,0.4267163868830551,6.497344448890763e-8,-2.9598552756642434e-7,0.9691811065713162,0.9985426719795709 +69.0,0.4267163912524428,5.0882866998259076e-8,-2.317959767734765e-7,0.9691810620904003,0.9985426619921258 +70.0,0.42671644043115114,-1.0770701760595323e-7,4.906592850593068e-7,0.969180561455502,0.9985425495829385 +71.0,0.4267164167376279,-3.130102261908867e-8,1.425919497941743e-7,0.9691808026534119,0.9985426037398922 +72.0,0.4267163669919889,1.291167181759515e-7,-5.881901500074731e-7,0.9691813090586004,0.9985427174447016 +73.0,0.4267164200016294,-4.1827178724749095e-8,1.9054393033467677e-7,0.9691807694245668,0.9985425962789112 +74.0,0.42671644240995465,-1.1408876512607039e-7,5.19730932389067e-7,0.9691805413101093,0.9985425450596278 +75.0,0.4267163840322124,7.416555460008109e-8,-3.3786040346975136e-7,0.9691811355893997,0.9985426784950959 +76.0,0.42671638308439874,7.72215823075342e-8,-3.5178174842226147e-7,0.9691811452364474,0.998542680661179 +77.0,0.42671644336728115,-1.1717662911843815e-7,5.337977747629874e-7,0.969180531562473,0.9985425428709592 +78.0,0.42671641776404695,-3.461230115131663e-8,1.5767640473892867e-7,0.9691807922007798,0.9985426013929286 +79.0,0.4267163612928776,1.4749353363619526e-7,-6.719051144173975e-7,0.9691813670703036,0.9985427304702585 +80.0,0.4267164203694237,-4.3014784946728866e-8,1.9595428752147932e-7,0.9691807656757546,0.9985425954371782 +81.0,0.4267164406054988,-1.0827141116253247e-7,4.932300540646792e-7,0.9691805596746412,0.9985425491830761 +82.0,0.4267163787730171,9.112354076019481e-8,-4.151121391980163e-7,0.9691811891224043,0.9985426905150361 +83.0,0.42671638837990683,6.01430724791526e-8,-2.73980456025726e-7,0.9691810913234238,0.998542668555912 +84.0,0.4267164428101249,-1.1538169475281397e-7,5.256211091987005e-7,0.96918053722903,0.9985425441432898 +85.0,0.4267164096512028,-8.452095193740493e-9,3.85038789026454e-8,0.969180874783399,0.9985426199354737 +86.0,0.42671636500544596,1.3551953690584167e-7,-6.173575814057123e-7,0.9691813292713266,0.998542721983131 +87.0,0.42671642647384933,-6.270190499547489e-8,2.856388664368029e-7,0.9691807035280883,0.9985425814829599 +88.0,0.4267164339544428,-8.682512094420266e-8,3.9553170304887336e-7,0.96918062737645,0.998542564384384 +89.0,0.4267163734271133,1.083610174179625e-7,-4.936373932413709e-7,0.9691812435379789,0.9985427027331426 +90.0,0.42671638393595407,7.447198602603216e-8,-3.392558129647435e-7,0.9691811365573083,0.9985426787124235 diff --git a/examples/example-model.json b/examples/example-model.json new file mode 100644 index 0000000..a0d0c27 --- /dev/null +++ b/examples/example-model.json @@ -0,0 +1 @@ +{"T":[{"tname":"exp"},{"tname":"conv"},{"tname":"rec"},{"tname":"death"}],"S":[{"sname":"S"},{"sname":"E"},{"sname":"I"},{"sname":"R"},{"sname":"D"}],"I":[{"it":1,"is":1},{"it":1,"is":3},{"it":2,"is":2},{"it":3,"is":3},{"it":4,"is":3}],"O":[{"ot":1,"os":2},{"ot":1,"os":3},{"ot":2,"os":3},{"ot":3,"os":4},{"ot":4,"os":5}]} diff --git a/examples/request-calibrate.json b/examples/request-calibrate.json index e8c3691..23411cd 100644 --- a/examples/request-calibrate.json +++ b/examples/request-calibrate.json @@ -1,25 +1,23 @@ { - "model": "{\"T\":[{\"tname\":\"exp\"},{\"tname\":\"conv\"},{\"tname\":\"rec\"},{\"tname\":\"death\"}],\"S\":[{\"sname\":\"S\"},{\"sname\":\"E\"},{\"sname\":\"I\"},{\"sname\":\"R\"},{\"sname\":\"D\"}],\"I\":[{\"it\":1,\"is\":1},{\"it\":1,\"is\":3},{\"it\":2,\"is\":2},{\"it\":3,\"is\":3},{\"it\":4,\"is\":3}],\"O\":[{\"ot\":1,\"os\":2},{\"ot\":1,\"os\":3},{\"ot\":2,\"os\":3},{\"ot\":3,\"os\":4},{\"ot\":4,\"os\":5}]}", - "initials": { - "S": 0.49457800495224524, - "E": 0.26745259325403603, - "I": 0.4497387877393193, - "R": 0.32807705995998604, - "D": 0.8545934885162726 + "model_config_id": "f20ea672-bfa4-422b-884f-0a4c09356fd0", + "dataset": { + "id": "2ea2d39f-866f-46f6-beec-972ed2136ed5", + "filename": "dataset.csv" }, - "params": { - "exp": 0.16207166221196045, - "conv": 0.7009195813964052, - "rec": 0.7040317196117394, - "death": 0.15807853921067516 - }, - "feature_mappings": { - "S": "S", - "Exp": "E", - "R": "R", - "Inf": "I", - "D": "D" - }, - "timesteps_column": "timestepz", - "dataset": "timestepz,S,Exp,Inf,R,D\n0.0,0.49457800495224524,0.26745259325403603,0.4497387877393193,0.32807705995998604,0.8545934885162726\n1.0,0.4665022246157727,0.15226661609143755,0.282128632173281,0.5819467770808255,0.9115956844605425\n2.0,0.44995568708559147,0.0870753734907285,0.17176626841178605,0.7388229924583731,0.9468196129753799\n3.0,0.4402579969265885,0.049958082609506564,0.10268280170774824,0.8334700316632883,0.9680710215147275\n4.0,0.4345928780439725,0.02873264900876653,0.060681564306912315,0.8897296629609703,0.9807031801012375\n5.0,0.43129133897514976,0.016554961390151432,0.03559634907740039,0.9228561146017267,0.9881411703774309\n6.0,0.42937079614049717,0.00955084027561801,0.020781137104605475,0.9422429917414922,0.9924941691596463\n7.0,0.42825523174624236,0.005515226366193871,0.012094002808321005,0.9535438792052197,0.9950315942958823\n8.0,0.42760803965578864,0.003187203614737658,0.007024405738798128,0.9601135757214116,0.996506709691123\n9.0,0.42723286848143166,0.0018428391922203655,0.004074861364167079,0.9639265223459347,0.9973628430381053\n10.0,0.42701542110237617,0.001065625665672367,0.002361473964384682,0.966138016624911,0.997859397064515\n11.0,0.42688949053651354,0.0006163374574758091,0.0013677486708266896,0.9674192754645957,0.9981470822924473\n12.0,0.4268166164339979,0.0003566562089632792,0.0007921677242322537,0.968160893781643,0.9983136002730226\n13.0,0.426774399020502,0.00020632531989352596,0.0004585439285171233,0.9685905857420082,0.9984100804109383\n14.0,0.4267499641987114,0.00011936317441910912,0.00026537658614250037,0.9688393044325163,0.9984659260300698\n15.0,0.4267358390828414,6.911064486184507e-5,0.00015368573473018547,0.9689830886008763,0.9984982103585492\n16.0,0.42672764295323584,3.995637251879534e-5,8.886936321230337e-5,0.9690665218374893,0.9985169438954028\n17.0,0.42672291793839023,2.315252702782855e-5,5.149885891478391e-5,0.969114621277837,0.9985277438196892\n18.0,0.42672016837248206,1.3374421742842271e-5,2.975169371631378e-5,0.9691426114006284,0.9985340285332894\n19.0,0.4267185903383033,7.76324275234401e-6,1.72697551963783e-5,0.96915867560426,0.9985376354813472\n20.0,0.42671766524571453,4.473700806675521e-6,9.952514996761569e-6,0.9691680929676613,0.9985397499926799\n21.0,0.4267171396866097,2.605039389810057e-6,5.795292710617236e-6,0.9691734431227009,0.9985409512804482\n22.0,0.4267168292213564,1.5011076626621485e-6,3.339530654332321e-6,0.9691766036403886,0.9985416609217972\n23.0,0.4267166503308677,8.650367137104304e-7,1.9244997328791993e-6,0.9691784247361698,0.998542069818375\n24.0,0.4267165513975358,5.133018515249044e-7,1.1418963787668045e-6,0.9691794318721828,0.9985422959539103\n25.0,0.42671648849345345,2.8961248029392266e-7,6.443470300592253e-7,0.9691800722327397,0.9985424397361555\n26.0,0.4267164534150905,1.6488872179188643e-7,3.668730947050761e-7,0.9691804293288433,0.9985425199161089\n27.0,0.4267164363527159,1.0425224374051472e-7,2.3187776609803832e-7,0.9691806030229255,0.9985425589162079\n28.0,0.4267164237650669,5.94635822904114e-8,1.323403435661362e-7,0.969180731164592,0.9985425876882744\n29.0,0.42671641532016785,2.938997840309349e-8,6.558731960133238e-8,0.9691808171332872,0.9985426069911061\n30.0,0.4267164119320277,1.7383990307292018e-8,3.8745941399490847e-8,0.9691808516243932,0.9985426147355065\n31.0,0.42671641142993344,1.567980166379535e-8,3.4693296446806386e-8,0.9691808567356694,0.9985426158831582\n32.0,0.4267164097239014,9.397997811086657e-9,2.141426794359566e-8,0.9691808741029923,0.9985426197826998\n33.0,0.4267164077513253,1.6630451169718799e-9,6.5322673177872115e-9,0.9691808941837284,0.998542624291493\n34.0,0.4267164068327058,-1.7144298561756136e-9,-6.228678598180968e-10,0.969180903535234,0.998542626391217\n35.0,0.42671640713709813,1.995841690399913e-10,9.531889471946351e-10,0.9691809004365325,0.9985426256954555\n36.0,0.4267164080167858,4.441082888563269e-9,6.797923212261653e-9,0.9691808914813484,0.9985426236847188\n37.0,0.42671640835547514,3.2138485814499743e-9,1.190846777114169e-8,0.9691808880335046,0.9985426229105632\n38.0,0.42671641412844163,-2.6272280610911653e-8,1.0758565137248983e-7,0.9691808292649736,0.9985426097150731\n39.0,0.42671642043471775,-5.7124386784960156e-8,2.107435751597396e-7,0.9691807650673759,0.9985425953005771\n40.0,0.42671642069465143,-6.101555879931584e-8,2.1761506634755097e-7,0.9691807624212634,0.9985425947064368\n41.0,0.42671641308224717,-2.98748098503364e-8,9.919283597417899e-8,0.9691808399151859,0.9985426121064\n42.0,0.42671640052516574,2.41129225278762e-8,-9.87705210985136e-8,0.9691809677456914,0.9985426408086006\n43.0,0.4267163907047244,6.850677646347266e-8,-2.557625261850519e-7,0.9691810677173265,0.998542663255558\n44.0,0.42671639824152524,4.377942441187322e-8,-1.4462071904623285e-7,0.9691809909931911,0.9985426460284376\n45.0,0.4267165177761834,-3.5045943998565953e-7,1.6201571649468285e-6,0.9691797741428834,0.9985423728050674\n46.0,0.426716582914649,-5.657635155369028e-7,2.5823127375500367e-6,0.9691791110412861,0.9985422239167021\n47.0,0.426716469373307,-1.9999268662048253e-7,9.147186185214705e-7,0.9691802668813723,0.9985424834412481\n48.0,0.4267162674880653,4.53529758154388e-7,-2.0535417063955472e-6,0.9691823220506339,0.9985429448951083\n49.0,0.42671627788979505,4.201179072135318e-7,-1.900860288767501e-6,0.9691822161561675,0.9985429211182781\n50.0,0.42671649186340843,-2.7150027993614184e-7,1.2440747380756681e-6,0.9691800379462864,0.9985424320377062\n51.0,0.4267164371442758,-9.590264835717333e-8,4.410989446171615e-7,0.9691805949726369,0.9985425571086503\n52.0,0.42671627454943717,4.2799006477805363e-7,-1.9470013902130833e-6,0.9691822501358911,0.9985429287478564\n53.0,0.4267164118674504,-1.515429881156514e-8,7.058019617525471e-8,0.9691808522520705,0.998542614876441\n54.0,0.4267164314292611,-7.842218520117033e-8,3.5813674655576225e-7,0.9691806531145852,0.9985425701634516\n55.0,0.4267163402967335,2.1536057882219336e-7,-9.805249063054177e-7,0.9691815808243099,0.9985427784651432\n56.0,0.4267164064309929,2.026849024044358e-9,-8.917426690141202e-9,0.969180907581668,0.998542627299776\n57.0,0.4267164283592435,-6.872496312372469e-8,3.1325564224967763e-7,0.969180684354161,0.9985425771777755\n58.0,0.4267163726304823,1.1096730955813961e-7,-5.053996636778452e-7,0.9691812516656547,0.9985427045580763\n59.0,0.4267164022669418,1.5383256861379745e-8,-7.001281739503044e-8,0.9691809499676411,0.9985426368168369\n60.0,0.4267164303322644,-7.51289506815351e-8,3.42286373147847e-7,0.9691806642650668,0.9985425726671056\n61.0,0.42671639252407795,4.678946740843496e-8,-2.1312670437383657e-7,0.9691810491487283,0.9985426590862899\n62.0,0.4267163906803671,5.2732295751289435e-8,-2.40207629435881e-7,0.9691810679165382,0.9985426633002876\n63.0,0.42671643093445266,-7.707951798376247e-8,3.51143486333012e-7,0.9691806581331516,0.9985425712902867\n64.0,0.4267164128814658,-1.8863795872089294e-8,8.593863522149074e-8,0.9691808419110224,0.9985426125545317\n65.0,0.42671637686662756,9.727501741359907e-8,-4.4313284520207667e-7,0.9691812085384706,0.9985426948745889\n66.0,0.4267164200453857,-4.196688650706194e-8,1.9118201954589875e-7,0.9691807689818369,0.9985425961795036\n67.0,0.42671643358050243,-8.561468651784963e-8,3.900185546253598e-7,0.9691806311955823,0.9985425652419063\n68.0,0.4267163868830551,6.497344448890763e-8,-2.9598552756642434e-7,0.9691811065713162,0.9985426719795709\n69.0,0.4267163912524428,5.0882866998259076e-8,-2.317959767734765e-7,0.9691810620904003,0.9985426619921258\n70.0,0.42671644043115114,-1.0770701760595323e-7,4.906592850593068e-7,0.969180561455502,0.9985425495829385\n71.0,0.4267164167376279,-3.130102261908867e-8,1.425919497941743e-7,0.9691808026534119,0.9985426037398922\n72.0,0.4267163669919889,1.291167181759515e-7,-5.881901500074731e-7,0.9691813090586004,0.9985427174447016\n73.0,0.4267164200016294,-4.1827178724749095e-8,1.9054393033467677e-7,0.9691807694245668,0.9985425962789112\n74.0,0.42671644240995465,-1.1408876512607039e-7,5.19730932389067e-7,0.9691805413101093,0.9985425450596278\n75.0,0.4267163840322124,7.416555460008109e-8,-3.3786040346975136e-7,0.9691811355893997,0.9985426784950959\n76.0,0.42671638308439874,7.72215823075342e-8,-3.5178174842226147e-7,0.9691811452364474,0.998542680661179\n77.0,0.42671644336728115,-1.1717662911843815e-7,5.337977747629874e-7,0.969180531562473,0.9985425428709592\n78.0,0.42671641776404695,-3.461230115131663e-8,1.5767640473892867e-7,0.9691807922007798,0.9985426013929286\n79.0,0.4267163612928776,1.4749353363619526e-7,-6.719051144173975e-7,0.9691813670703036,0.9985427304702585\n80.0,0.4267164203694237,-4.3014784946728866e-8,1.9595428752147932e-7,0.9691807656757546,0.9985425954371782\n81.0,0.4267164406054988,-1.0827141116253247e-7,4.932300540646792e-7,0.9691805596746412,0.9985425491830761\n82.0,0.4267163787730171,9.112354076019481e-8,-4.151121391980163e-7,0.9691811891224043,0.9985426905150361\n83.0,0.42671638837990683,6.01430724791526e-8,-2.73980456025726e-7,0.9691810913234238,0.998542668555912\n84.0,0.4267164428101249,-1.1538169475281397e-7,5.256211091987005e-7,0.96918053722903,0.9985425441432898\n85.0,0.4267164096512028,-8.452095193740493e-9,3.85038789026454e-8,0.969180874783399,0.9985426199354737\n86.0,0.42671636500544596,1.3551953690584167e-7,-6.173575814057123e-7,0.9691813292713266,0.998542721983131\n87.0,0.42671642647384933,-6.270190499547489e-8,2.856388664368029e-7,0.9691807035280883,0.9985425814829599\n88.0,0.4267164339544428,-8.682512094420266e-8,3.9553170304887336e-7,0.96918062737645,0.998542564384384\n89.0,0.4267163734271133,1.083610174179625e-7,-4.936373932413709e-7,0.9691812435379789,0.9985427027331426\n90.0,0.42671638393595407,7.447198602603216e-8,-3.392558129647435e-7,0.9691811365573083,0.9985426787124235" + "timespan": [101.0, 108.0], + "extra": { + "initials": { + "S": 0.49457800495224524, + "E": 0.26745259325403603, + "I": 0.4497387877393193, + "R": 0.32807705995998604, + "D": 0.8545934885162726 + }, + "params": { + "exp": 0.16207166221196045, + "conv": 0.7009195813964052, + "rec": 0.7040317196117394, + "death": 0.15807853921067516 + } + } } \ No newline at end of file diff --git a/examples/request-simulate.json b/examples/request-simulate.json index a6a1964..40db652 100644 --- a/examples/request-simulate.json +++ b/examples/request-simulate.json @@ -1,17 +1,19 @@ { - "model": "{\"T\":[{\"tname\":\"exp\"},{\"tname\":\"conv\"},{\"tname\":\"rec\"},{\"tname\":\"death\"}],\"S\":[{\"sname\":\"S\"},{\"sname\":\"E\"},{\"sname\":\"I\"},{\"sname\":\"R\"},{\"sname\":\"D\"}],\"I\":[{\"it\":1,\"is\":1},{\"it\":1,\"is\":3},{\"it\":2,\"is\":2},{\"it\":3,\"is\":3},{\"it\":4,\"is\":3}],\"O\":[{\"ot\":1,\"os\":2},{\"ot\":1,\"os\":3},{\"ot\":2,\"os\":3},{\"ot\":3,\"os\":4},{\"ot\":4,\"os\":5}]}", - "initials": { - "S": 0.49457800495224524, - "E": 0.26745259325403603, - "I": 0.4497387877393193, - "R": 0.32807705995998604, - "D": 0.8545934885162726 - }, - "tspan": [0.0,90.0], - "params": { - "exp": 0.16207166221196045, - "conv": 0.7009195813964052, - "rec": 0.7040317196117394, - "death": 0.15807853921067516 - } + "model_config_id": "f20ea672-bfa4-422b-884f-0a4c09356fd0", + "timespan": [101.0, 108.0], + "extra": { + "initials": { + "S": 0.49457800495224524, + "E": 0.26745259325403603, + "I": 0.4497387877393193, + "R": 0.32807705995998604, + "D": 0.8545934885162726 + }, + "params": { + "exp": 0.16207166221196045, + "conv": 0.7009195813964052, + "rec": 0.7040317196117394, + "death": 0.15807853921067516 + } + } } \ No newline at end of file diff --git a/paths.yaml b/paths.yaml new file mode 100644 index 0000000..c5c86a9 --- /dev/null +++ b/paths.yaml @@ -0,0 +1,159 @@ +/simulate: + post: + summary: Perform a simulation + operationId: simulateModel + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + engine: + type: string + enum: [SciML, CIEMSS] + example: CIEMSS + model_config_id: + type: string + example: "ba8da8d4-047d-11ee-be56" + timespan: + type: array + items: + type: integer + example: [0, 90] + extra: + description: optional extra system specific arguments for advanced use cases + type: object + required: + - engine + - model_config_id + - timespan + responses: + '200': + description: Simulation created successfully + content: + application/json: + schema: + type: object + properties: + simulation_id: + type: string + example: fc5d80e4-0483-11ee-be56 + +/calibrate: + post: + summary: Calibrate a model to data then simulate + description: Calibrate a model and perform a "fitting" simulation over the time range of the calibration dataset and a simulation of a time range specified in `timespan` if `timespan` is specified. The SciML engine allows for calibration without additional simulation; in that case `timespan` is left off the payload. + operationId: calibratesimulateModel + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + engine: + type: string + enum: [SciML, CIEMSS] + example: CIEMSS + model_config_id: + type: string + example: "c1cd941a-047d-11ee-be56" + dataset: + type: object + properties: + id: + type: string + example: "cd339570-047d-11ee-be55" + filename: + type: string + example: "dataset.csv" + timespan: + type: array + items: + type: integer + example: [0, 90] + extra: + description: optional extra system specific arguments for advanced use cases + type: object + required: + - engine + - model_config_id + - dataset_id + responses: + '200': + description: Calibration/simulation created successfully + content: + application/json: + schema: + type: object + properties: + simulation_id: + type: string + example: fc5d80e4-0483-11ee-be56 +/ensemble: + post: + summary: Perform an ensemble simulation + operationId: createEnsemble + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + engine: + type: string + enum: [SciML, CIEMSS] + example: CIEMSS + model_config_ids: + type: array + items: + type: string + example: ["ba8da8d4-047d-11ee-be56", "c1cd941a-047d-11ee-be56", "c4b9f88a-047d-11ee-be56"] + timespan: + type: array + items: + type: integer + example: [0, 90] + extra: + description: optional extra system specific arguments for advanced use cases + type: object + required: + - engine + - model_config_id + - timespan + responses: + '200': + description: Ensemble created successfully + content: + application/json: + schema: + type: object + properties: + simulation_id: + type: string + example: fc5d80e4-0483-11ee-be56 + +/status/{simulation_id}: + get: + summary: Retrieve the status of a simulation + operationId: getStatus + parameters: + - name: simulation_id + in: path + required: true + schema: + type: string + example: fc5d80e4-0483-11ee-be56 + responses: + '200': + description: Status retrieved successfully + content: + application/json: + schema: + type: object + properties: + status: + type: string + enum: [queued, running, complete, error] diff --git a/src/Settings.jl b/src/Settings.jl index 2fa4c33..a18fc39 100644 --- a/src/Settings.jl +++ b/src/Settings.jl @@ -60,16 +60,12 @@ macro setting(name::Symbol) :(@setting($name, String, nothing)) end -@setting SHOULD_LOG false +@setting RABBITMQ_ENABLED false @setting RABBITMQ_LOGIN "guest" @setting RABBITMQ_PASSWORD "guest" @setting RABBITMQ_ROUTE "terarium" @setting RABBITMQ_PORT 5672 -@setting ENABLE_REMOTE_DATA_HANDLING false +@setting ENABLE_TDS true @setting TDS_URL "http://localhost:8001" -@setting FILE_STORE "http://localhost:9000" -@setting BUCKET "jataware-sim-service-test" -@setting AWS_ACCESS_KEY_ID "miniouser" -@setting AWS_SECRET_ACCESS_KEY "miniopass" end # module Settings diff --git a/src/SimulationService.jl b/src/SimulationService.jl index 1b11f70..d81a4b4 100644 --- a/src/SimulationService.jl +++ b/src/SimulationService.jl @@ -7,260 +7,40 @@ import AlgebraicPetri: LabelledPetriNet import Symbolics import Oxygen: serveparallel, resetstate, json, setschema, terminate, @post, @get import SwaggerMarkdown: build, @swagger, OpenAPI, validate_spec, openApiToDict, DOCS -import YAML: load -import CSV: write +import YAML import JSON3 as JSON -import DataFrames: DataFrame -import HTTP: Request, Response import JobSchedulers: scheduler_start, set_scheduler, scheduler_stop, submit!, job_query, result, generate_id, update_queue!, Job, JobSchedulers -include("./contracts/Interface.jl"); import .Interface: get_operation, use_operation, conversions_for_valid_inputs, Context -include("./service/Service.jl"); import .Service.ArgIO: prepare_output, prepare_input; import .Service.Queuing: publish_to_rabbitmq +include("./service/Service.jl") +import .Service: make_deterministic_run, retrieve_job include("./Settings.jl"); import .Settings: settings export start!, stop! +OPENAPI_SPEC = "paths.yaml" + """ Print out settings """ function health_check() tds_url = settings["TDS_URL"] - enable_remote = settings["ENABLE_REMOTE_DATA_HANDLING"] mq_route = settings["RABBITMQ_ROUTE"] - return "Simulation-service. TDS_URL=$tds_url, RABBITMQ_ROUTE=$mq_route, ENABLE_REMOTE_DATA_HANDLING=$enable_remote" -end - -""" -Generate the task to run with the correct context -""" -function contextualize_prog(context) - prepare_output(context) ∘ use_operation(context) ∘ prepare_input(context) -end - -""" -Schedule a sim run given an operation -""" -function make_deterministic_run(req::Request, operation::String) - # TODO(five): Spawn remote workers and run jobs on them - # TODO(five): Handle Python so a probabilistic case can work - if isnothing(get_operation(operation)) - return Response( - 404, - ["Content-Type" => "text/plain; charset=utf-8"], - body="Operation not found" - ) - end - - publish_hook = settings["SHOULD_LOG"] ? publish_to_rabbitmq : (args...) -> nothing - - context = Context( - generate_id(), - publish_hook, - Symbol(operation), - ) - prog = contextualize_prog(context) - sim_run = Job(@task(prog(req))) - sim_run.id = context.job_id - submit!(sim_run) - Response( - 201, - ["Content-Type" => "application/json; charset=utf-8"], - body=JSON.write("id" => sim_run.id) - ) -end - -""" -Get status of sim -""" -function retrieve_job(_, id::Int64, element::String) - job = job_query(id) - if isnothing(job) - return Response( - 404, - ["Content-Type" => "text/plain; charset=utf-8"], - body="Job does not exist" - ) - end - if element == "status" - return Dict("status" => job.state) - elseif element == "result" - if job.state == :done - return result(job) - else - return Response( - 400, - ["Content-Type" => "text/plain; charset=utf-8"], - body="Job has not completed" - ) - end - else - return Response( - 404, - ["Content-Type" => "text/plain; charset=utf-8"], - body="Element not found" - ) - end + return "Simulation-service. TDS_URL=$tds_url, RABBITMQ_ROUTE=$mq_route" end - - """ Specify endpoint to function mappings """ function register!() - @swagger """ - /: - get: - summary: Healthcheck - description: A basic healthcheck for the simulation scheduler - responses: - '200': - description: Returns notice that service has started - - """ @get "/" health_check + @get "/{element}/{uuid}" retrieve_job + @post "/{operation}" make_deterministic_run - @swagger """ - /runs/{id}/status: - get: - summary: Simulation status - description: Get status of specified job - parameters: - - name: id - in: path - required: true - description: ID of the simulation job - schema: - type: number - responses: - '200': - description: JSON containing status of the job - '404': - description: Job does not exist - /runs/{id}/result: - get: - summary: Simulation results - description: Get the resulting CSV from a job - parameters: - - name: id - in: path - required: true - description: ID of the simulation job - schema: - type: number - responses: - '200': - description: CSV containing timesteps for each compartment - '400': - description: Job has not yet completed - '404': - description: Job does not exist - """ - @get "/runs/{id}/{element}" retrieve_job - - - @swagger """ - /calls/simulate: - post: - summary: Simulation simulate - description: Create simulate job - requestBody: - description: Arguments to pass into simulate function - required: true - content: - application/json: - schema: - type: object - properties: - model: - type: string - initials: - type: object - properties: - compartment: - type: number - params: - type: object - properties: - variable: - type: number - tspan: - type: array - items: - type: number - required: - - model - - initials - - params - - tspan - example: - model: "{}" - initials: {"compartment_a": 100.1, "compartment_b": 200} - params: {"alpha": 0.5, "beta": 0.1} - tspan: [0,20] - responses: - '201': - description: The ID of the job created - /calls/calibrate: - post: - summary: Simulation calibrate - description: Create calibrate job - requestBody: - description: Arguments to pass into simulate function. - required: true - content: - application/json: - schema: - type: object - properties: - model: - type: string - initials: - type: object - properties: - compartment: - type: number - params: - type: object - properties: - variable: - type: number - timesteps_column: - type: string - feature_mappings: - type: object - properties: - fromkey: - type: array - items: - type: string - dataset: - type: string - required: - - model - - initials - - params - - timesteps_column - - feature_mappings - - dataset - example: - model: "{}" - initials: {"compartment_a": 100.1, "compartment_b": 200} - params: {"alpha": 0.5, "beta": 0.1} - timesteps_column: [] - feature_mappings: {} - dataset: "" - responses: - '201': - description: The ID of the job created - """ - @post "/calls/{operation}" make_deterministic_run - - info = Dict("title" => "Simulation Service", "version" => "0.1.0") + info = Dict("title" => "Simulation Service", "version" => "0.6.0") openAPI = OpenAPI("3.0.0", info) - openAPI.paths = load(join(DOCS)) # NOTE: Has to be done manually because it's broken in SwaggerMarkdown + openAPI.paths = YAML.load_file(OPENAPI_SPEC) documentation = build(openAPI) setschema(documentation) end diff --git a/src/contracts/Available.jl b/src/contracts/Available.jl new file mode 100644 index 0000000..af947ee --- /dev/null +++ b/src/contracts/Available.jl @@ -0,0 +1,72 @@ +""" +Operations interface for the simulation service +""" +module Available + +import CSV +import AlgebraicPetri: LabelledPetriNet, AbstractPetriNet +import DataFrames: DataFrame + +include("./ProblemInputs.jl"); import .ProblemInputs: conversions_for_valid_inputs +include("./SystemInputs.jl"); import .SystemInputs: Context +include("../operations/Operations.jl"); import .Operations +include("../Settings.jl"); import .Settings: settings + +export available_operations + +available_operations = Dict{String, Function}() + +""" +Retrieve internal atomic operations +""" +function get_operation(operation::Symbol) + if in(operation, names(Operations; all=false, imported=true)) + return getfield(Operations, operation) + else + return nothing + end +end + +function simulate(; + model::AbstractPetriNet, + params::Dict{String,Float64}, + initials::Dict{String,Float64}, + timespan=(0.0, 100.0)::Tuple{Float64,Float64}, + context +) + Dict("result" => get_operation(:simulate)(;model=model, params=params, initials=initials, timespan=timespan, context)) +end + +available_operations["simulate"] = simulate + +function calibrate_then_simulate(; + model::AbstractPetriNet, # TODO(five): Remove from exports and rename + params::Dict{String,Float64}, + initials::Dict{String,Float64}, + dataset::DataFrame, + timespan::Union{Nothing, Tuple{Float64, Float64}} = nothing, + context, +) + results = Dict{String, Any}() + calibrated_params = get_operation(:calibrate)(;model=model, params=params, initials=initials, dataset=dataset, context=context) + results["parameters"] = calibrated_params + if in(NaN, values(calibrated_params)) throw("NaN adjustment") + return results + end + adjusted_params = Dict(string(key.val) => value for (key,value) in calibrated_params) + tmin, tmax = (dataset.timestep[1], dataset.timestep[end]) + results["simulation"] = get_operation(:simulate)(;model=model, params=adjusted_params, initials=initials, timespan=(tmin, tmax), context=context) + if isnothing(timespan) + return results + end + results["extra-simulation"] = get_operation(:simulate)(;model=model, params=adjusted_params, initials=initials, timespan=timespan, context=context) + results +end + +available_operations["calibrate"] = calibrate_then_simulate + + + + + +end # module Available diff --git a/src/contracts/Interface.jl b/src/contracts/Interface.jl index e3e6315..1f97d3a 100644 --- a/src/contracts/Interface.jl +++ b/src/contracts/Interface.jl @@ -7,35 +7,28 @@ import CSV include("./ProblemInputs.jl"); import .ProblemInputs: conversions_for_valid_inputs include("./SystemInputs.jl"); import .SystemInputs: Context +include("./Available.jl"); import .Available: available_operations include("../operations/Operations.jl"); import .Operations include("../Settings.jl"); import .Settings: settings -export get_operation, conversions_for_valid_inputs, Context - -""" -Sim runs that can be created using the `/runs/sciml/{operation}` endpoint. -""" -function get_operation(raw_operation) - operation = Symbol(raw_operation) - if in(operation, names(Operations; all=false, imported=true)) - return getfield(Operations, operation) - else - return nothing - end -end +export use_operation, available_operations, conversions_for_valid_inputs, Context """ Return an operation wrapped with necessary handlers """ function use_operation(context::Context) - operation = get_operation(context.operation) + operation = available_operations[string(context.operation)] + + method = collect(methods(operation))[1] + inputs = ccall(:jl_uncompress_argnames, Vector{Symbol}, (Any,), method.slot_syms)[2:end] + # NOTE: This runs inside the job so we can't use it to validate on request ATM function coerced_operation(arglist::Dict{Symbol, Any}) # TODO(five): Fail properly on extra params fixed_args = Dict( - name => conversions_for_valid_inputs[name](value) - for (name, value) in arglist + name => conversions_for_valid_inputs[name](arglist[name]) + for name in inputs if name != :context ) operation(;fixed_args..., context=context) end diff --git a/src/contracts/ProblemInputs.jl b/src/contracts/ProblemInputs.jl index 5873088..e2f5e5a 100644 --- a/src/contracts/ProblemInputs.jl +++ b/src/contracts/ProblemInputs.jl @@ -20,17 +20,26 @@ Act as identity since the value is already coerced """ coerce_dataset(val::DataFrame) = val +""" +Transform unstructured payload into ACSet +""" +coerce_model(val) = parse_json_acset(PropertyLabelledReactionNet{Number, Number, Dict}, val) + +""" +Coerce timespan +""" +coerce_timespan(val) = !isnothing(val) ? Tuple{Float64,Float64}(val) : nothing + """ Inputs converted from payload to arguments expanded in operations. """ conversions_for_valid_inputs = Dict{Symbol,Function}( - :model => (val) -> parse_json_acset(PropertyLabelledReactionNet{Number, Number, Dict}, val), # hack for mira - :tspan => (val) -> Tuple{Float64,Float64}(val), + :model => coerce_model, + :models => val -> coerce_model.(val), + :timespan => coerce_timespan, :params => (val) -> Dict{String,Float64}(val), :initials => (val) -> Dict{String,Float64}(val), :dataset => coerce_dataset, - :feature_mappings => (val) -> Dict{String, String}(val), - :timesteps_column => (val) -> String(val) ) end # module ProblemInputs \ No newline at end of file diff --git a/src/contracts/SystemInputs.jl b/src/contracts/SystemInputs.jl index 36f0034..dba521c 100644 --- a/src/contracts/SystemInputs.jl +++ b/src/contracts/SystemInputs.jl @@ -9,6 +9,10 @@ struct Context job_id::Int64 interactivity_hook::Function operation::Symbol + raw_args + function Context(job_id::Int64, interactivity_hook::Function, operation::Symbol, raw_args) + new(job_id, interactivity_hook, operation, deepcopy(raw_args)) + end end function Base.iterate(context::Context, state=nothing) diff --git a/src/operations/Operations.jl b/src/operations/Operations.jl index 1daac31..131604f 100644 --- a/src/operations/Operations.jl +++ b/src/operations/Operations.jl @@ -13,8 +13,8 @@ import EasyModelAnalysis include("./Utils.jl"); import .Utils: to_prob, unzip, symbolize_args, select_data -# NOTE: Export symbols here are automatically made available to the API (`/calls/{name}`) -export simulate, calibrate +# NOTE: Operations exposed to the rest of the Simulation Service +export simulate, calibrate, ensemble """ Simulate a scenario from a PetriNet @@ -22,10 +22,10 @@ Simulate a scenario from a PetriNet function simulate(; model::AbstractPetriNet, params::Dict{String,Float64}, initials::Dict{String,Float64}, - tspan=(0.0, 100.0)::Tuple{Float64,Float64}, + timespan=(0.0, 100.0)::Tuple{Float64,Float64}, context )::DataFrame - sol = solve(to_prob(model, params, initials, tspan); progress = true, progress_steps = 1) + sol = solve(to_prob(model, params, initials, timespan); progress = true, progress_steps = 1) DataFrame(sol) end @@ -38,11 +38,9 @@ function calibrate(; model::AbstractPetriNet, params::Dict{String,Float64}, initials::Dict{String,Float64}, dataset::DataFrame, - feature_mappings::Dict{String, String}, - timesteps_column::String = "timestamp", context, ) - timesteps, data = select_data(dataset, feature_mappings, timesteps_column) + timesteps, data = select_data(dataset) prob = to_prob(model, params, initials, extrema(timesteps)) sys = prob.f.sys p = symbolize_args(params, parameters(sys)) # this ends up being a second call to symbolize_args 🤷 @@ -56,6 +54,13 @@ function calibrate(; model::AbstractPetriNet, fitp end +""" +NOT IMPLEMENTED +""" +function ensemble(; models::AbstractArray{AbstractPetriNet}, timespan=(0.0, 100.0)::Tuple{Float64, Float64}) + throw("ENSEMBLE IS NOT YET IMPLEMENTED") +end + "long running functions like global_datafit and sensitivity wrappers will need to be refactored to share callback info incrementally" function _global_datafit(; model::LabelledPetriNet, parameter_bounds::Dict{String,Tuple{Float64,Float64}}, diff --git a/src/operations/Utils.jl b/src/operations/Utils.jl index a19ef98..be77ed9 100644 --- a/src/operations/Utils.jl +++ b/src/operations/Utils.jl @@ -3,7 +3,7 @@ Unexposed helper functions for operations """ module Utils -import DataFrames: DataFrame +import DataFrames: DataFrame, names import ModelingToolkit: ODESystem, ODEProblem import Symbolics: getname import SymbolicIndexingInterface: states, parameters @@ -52,12 +52,13 @@ end """ Generate data and timestep list from a dataframe """ -function select_data(dataframe::DataFrame, feature_mappings:: Dict{String, String}, timesteps_column::String) +function select_data(dataframe::DataFrame) data = Dict( - to => dataframe[!, from] - for (from, to) in feature_mappings + key => dataframe[!, key] + for key in names(dataframe) if key != "timestep" ) - dataframe[!, timesteps_column], data + + dataframe[!, "timestep"], data end diff --git a/src/service/ArgIO.jl b/src/service/ArgIO.jl index cd21d2b..947e93b 100644 --- a/src/service/ArgIO.jl +++ b/src/service/ArgIO.jl @@ -4,13 +4,13 @@ Provide external awareness / service-related side-effects to SciML operations module ArgIO import Symbolics -import DataFrames: DataFrame, rename! +import DataFrames: rename!, transform!, DataFrame, ByRow import CSV import HTTP: Request -import Oxygen: serveparallel, serve, resetstate, json, setschema, @post, @get +import JSON3 as JSON include("../Settings.jl"); import .Settings: settings -include("./AssetManager.jl"); import .AssetManager: fetch_dataset, fetch_model, upload +include("./AssetManager.jl"); import .AssetManager: fetch_dataset, fetch_model, update_simulation, upload export prepare_input, prepare_output @@ -20,14 +20,25 @@ Transform requests into arguments to be used by operation Optionally, IDs are hydrated with the corresponding entity from TDS. """ -function prepare_input(req::Request; context...) - args = json(req, Dict{Symbol,Any}) - if settings["ENABLE_REMOTE_DATA_HANDLING"] - if in(:model, keys(args)) - args[:model] = fetch_model(string(args[:model])) - end - if in(:dataset, keys(args)) - args[:dataset] = fetch_dataset(string(args[:dataset])) +function prepare_input(args; context...) + if settings["ENABLE_TDS"] + update_simulation(context[:job_id], Dict([:status=>"running", :start_time => time()])) + end + if in(:model_config_id, keys(args)) + args[:model] = fetch_model(args[:model_config_id]) + end + if in(:dataset, keys(args)) && !isa(args[:dataset], String) + args[:dataset] = fetch_dataset(args[:dataset]["id"], args[:dataset]["filename"]) + end + if in(:model_config_ids, keys(args)) + args[:models] = fetch_model.(map(string, args[:model_ids])) + end + if !in(:timespan, keys(args)) + args[:timespan] = nothing + end + if in(:extra, keys(args)) + for (key, value) in Dict(args[:extra]) + args[Symbol(key)] = value end end args @@ -37,8 +48,8 @@ end Generate a `prepare_input` function that is already contextualized """ function prepare_input(context) - function contextualized_prepare_input(req::Request) - prepare_input(req; context...) + function contextualized_prepare_input(args) + prepare_input(args; context...) end end @@ -47,26 +58,42 @@ Normalize the header of the resulting dataframe and return a CSV Optionally, the CSV is saved to TDS instead an the coreresponding ID is returned. """ -function prepare_output(dataframe::DataFrame; context...) +function prepare_output(dataframe::DataFrame; name="0", context...) stripped_names = names(dataframe) .=> (r -> replace(r, "(t)"=>"")).(names(dataframe)) rename!(dataframe, stripped_names) - rename!(dataframe, "timestamp" => "timestep") - if !settings["ENABLE_REMOTE_DATA_HANDLING"] + if !settings["ENABLE_TDS"] io = IOBuffer() # TODO(five): Write to remote server CSV.write(io, dataframe) return String(take!(io)) else - return upload(dataframe, context[:job_id]) + return upload(dataframe, context[:job_id]; name=name) end end """ -Coerces NaN values to nothing for each parameter. +Coerces NaN values to nothing for each parameter """ -function prepare_output(params::Vector{Pair{Symbolics.Num, Float64}}; context...) +function prepare_output(params::Vector{Pair{Symbolics.Num, Float64}}; name="0", context...) nan_to_nothing(value) = isnan(value) ? nothing : value - Dict(key => nan_to_nothing(value) for (key, value) in params) + fixed_params = Dict(key => nan_to_nothing(value) for (key, value) in params) + if settings["ENABLE_TDS"] + return upload(fixed_params, context[:job_id]; name=name) + end +end + + +""" +Coerces NaN values to nothing for each parameter +""" +function prepare_output(results::Dict{String}; context...) + if settings["ENABLE_TDS"] + urls = [] + for (name, value) in results + append!(urls, [prepare_output(value; context..., name=name)]) + end + update_simulation(context[:job_id], Dict([:status => "complete", :result_files => urls, :completed_time => time()])) + end end """ diff --git a/src/service/AssetManager.jl b/src/service/AssetManager.jl index 85261ae..29bc34f 100644 --- a/src/service/AssetManager.jl +++ b/src/service/AssetManager.jl @@ -7,55 +7,99 @@ import DataFrames: DataFrame import CSV, Downloads, HTTP import OpenAPI.Clients: Client import JSON3 as JSON -using AWS -include("./MinIO.jl"); using .MinIO +import UUIDs: UUID include("../Settings.jl"); import .Settings: settings -@service S3 -export fetch_dataset, fetch_model, upload +export fetch_dataset, fetch_model, update_simulation, upload + +""" +Generate UUID with prefix +""" +function gen_uuid(job_id) + "sciml-" * string(UUID(job_id)) +end """ Return model JSON as string from TDS by ID """ function fetch_model(model_id::String) - response = HTTP.get("$(settings["TDS_URL"])/models/$model_id", ["Content-Type" => "application/json"]) + response = HTTP.get("$(settings["TDS_URL"])/model_configurations/$model_id", ["Content-Type" => "application/json"]) body = response.body |> JSON.read ∘ String - body.content + body.configuration end """ Return csv from TDS by ID """ -function fetch_dataset(dataset_id::String) - url = "$(settings["TDS_URL"])/datasets/$dataset_id/file" +function fetch_dataset(dataset_id::String, filename::String) + # TODO(five): Select name dynamicially + url = "$(settings["TDS_URL"])/datasets/$dataset_id/download-url?filename=$filename" + response = HTTP.get(url, ["Content-Type" => "application/json"]) + body = response.body |> JSON.read ∘ String io = IOBuffer() - Downloads.download(url, io) + Downloads.download(body.url, io) seekstart(io) CSV.read(io, DataFrame) end +""" +Report the job as completed +""" +function update_simulation(job_id::Int64, updated_fields::Dict{Symbol}) + uuid = gen_uuid(job_id) + response = nothing + remaining_retries = 10 # TODO(five)??: Set this with environment variable + while remaining_retries != 0 + remaining_retries -= 1 + sleep(2) + try + response = HTTP.get("$(settings["TDS_URL"])/simulations/$uuid", ["Content-Type" => "application/json"]) + break + catch exception + if isa(exception,HTTP.Exceptions.StatusError) && exception.status == 404 + response = nothing + else + throw(exception) + end + end + end + if isnothing(response) + throw("Job cannot finish because it does not exist in TDS") + end + body = response.body |> Dict ∘ JSON.read ∘ String + for field in updated_fields + body[field.first] = field.second + end + HTTP.put("$(settings["TDS_URL"])/simulations/$uuid", ["Content-Type" => "application/json"], body=JSON.write(body)) +end + """ Upload a CSV to S3/MinIO """ -function upload(output::DataFrame, job_id) +function upload(output::DataFrame, job_id; name="result") + uuid = gen_uuid(job_id) + response = HTTP.get("$(settings["TDS_URL"])/simulations/$uuid/upload-url?filename=$name.csv", ["Content-Type" => "application/json"]) # TODO(five): Stream so there isn't duplication - CONTENT_TYPE = "text/csv" io = IOBuffer() CSV.write(io, output) seekstart(io) - params = Dict( - "body" => take!(io), - "content-type" => CONTENT_TYPE - ) - - handle = "$job_id.csv" - - # TODO(five): Call once - AWS.global_aws_config(config) - - S3.put_object(settings["BUCKET"], handle, params) - - return Dict("data_path" => handle) + url = JSON.read(response.body)[:url] + HTTP.put(url, ["Content-Type" => "application/json"], body = take!(io)) + bare_url = split(url, "?")[1] + bare_url +end + + +""" +Upload a JSON to S3/MinIO +""" +function upload(output::Dict, job_id; name="result") + uuid = gen_uuid(job_id) + response = HTTP.get("$(settings["TDS_URL"])/simulations/$uuid/upload-url?filename=$name.json", ["Content-Type" => "application/json"]) + url = JSON.read(response.body)[:url] + HTTP.put(url, ["Content-Type" => "application/json"], body = JSON.write(output)) + bare_url = split(url, "?")[1] + bare_url end end # module AssetManager diff --git a/src/service/Execution.jl b/src/service/Execution.jl new file mode 100644 index 0000000..be9e3a0 --- /dev/null +++ b/src/service/Execution.jl @@ -0,0 +1,113 @@ +""" +Manage jobs +""" +module Execution + +import HTTP: Request, Response +import JSON3 as JSON +import Oxygen: json +import JobSchedulers: submit!, job_query, result, generate_id, Job, JobSchedulers +import UUIDs: UUID + +include("../contracts/Interface.jl"); import .Interface: available_operations, use_operation, Context +include("./AssetManager.jl"); import .AssetManager: update_simulation +include("./ArgIO.jl"); import .ArgIO: prepare_input, prepare_output +include("./Queuing.jl"); import .Queuing: publish_to_rabbitmq +include("../Settings.jl"); import .Settings: settings + +export make_deterministic_run, retrieve_job + +SCHEDULER_TO_API_STATUS_MAP = Dict( + JobSchedulers.QUEUING => :queued, + JobSchedulers.RUNNING => :running, + JobSchedulers.DONE => :complete, + JobSchedulers.FAILED => :error, + JobSchedulers.CANCELLED => :cancelled, +) + +""" +Generate the task to run with the correct context +""" +function contextualize_prog(context) + function prog(args) + try + (prepare_output(context) ∘ use_operation(context) ∘ prepare_input(context))(args) + catch exception + if settings["ENABLE_TDS"] + update_simulation(context.job_id, Dict([:status=>"error"])) + end + throw(exception) + end + end +end + +""" +Schedule a sim run given an operation +""" +function make_deterministic_run(req::Request, operation::String) + # TODO(five): Spawn remote workers and run jobs on them + if !in(operation, keys(available_operations)) + return Response( + 404, + ["Content-Type" => "text/plain; charset=utf-8"], + body="Operation not found" + ) + end + + publish_hook = settings["RABBITMQ_ENABLED"] ? publish_to_rabbitmq : (_...) -> nothing + + args = json(req, Dict{Symbol,Any}) + context = Context( + generate_id(), + publish_hook, + Symbol(operation), + args + ) + prog = contextualize_prog(context) + sim_run = Job(@task(prog(args))) + sim_run.id = context.job_id + submit!(sim_run) + uuid = "sciml-" * string(UUID(sim_run.id)) + Response( + 201, + ["Content-Type" => "application/json; charset=utf-8"], + body=JSON.write("simulation_id" => uuid) + ) +end + + +""" +Get status of sim +""" +function retrieve_job(_, uuid::String, element::String) + id = Int64(UUID(split(uuid, "sciml-")[2]).value) + job = job_query(id) + if isnothing(job) + return Response( + 404, + ["Content-Type" => "text/plain; charset=utf-8"], + body="Job does not exist" + ) + end + if element == "status" + return Dict("status" => SCHEDULER_TO_API_STATUS_MAP[job.state]) + elseif element == "result" + if job.state == :done + return result(job) + else + return Response( + 400, + ["Content-Type" => "text/plain; charset=utf-8"], + body="Job has not completed" + ) + end + else + return Response( + 404, + ["Content-Type" => "text/plain; charset=utf-8"], + body="Element not found" + ) + end +end + +end # module Execution diff --git a/src/service/MinIO.jl b/src/service/MinIO.jl deleted file mode 100644 index f09722a..0000000 --- a/src/service/MinIO.jl +++ /dev/null @@ -1,53 +0,0 @@ -""" -MinIO/S3 Handler -""" -module MinIO -import AWS: AWS, AbstractAWSConfig - -include("../Settings.jl") -import .Settings: settings - -export MinioConfig, region, credentials, Credentials, check_credentials, generate_service_url, config - -""" -MinIO/S3 Configuration -""" -struct MinioConfig <: AbstractAWSConfig - endpoint::String - region::String - creds -end -AWS.region(config::MinioConfig) = config.region -AWS.credentials(config::MinioConfig) = config.creds - -""" -Generic Credentials that work for MinIO and S3 -""" -struct Credentials - access_key_id::String - secret_key::String - token::String -end -AWS.check_credentials(c::Credentials) = c - -function AWS.generate_service_url(aws::MinioConfig, service::String, resource::String) - service == "s3" || throw(ArgumentError("Can only handle S3 requests")) - SERVICE_HOST = "amazonaws.com" - reg = AWS.region(aws) - if length(settings["FILE_STORE"]) == 0 - return string( - "https://", service, ".", isempty(reg) ? "" : "$reg.", SERVICE_HOST, resource - ) - else - return string(settings["FILE_STORE"], resource) - end -end - - -config = MinioConfig( - settings["FILE_STORE"], - AWS.DEFAULT_REGION, - Credentials(settings["AWS_ACCESS_KEY_ID"], settings["AWS_SECRET_ACCESS_KEY"], "") -) - -end # module MinIO diff --git a/src/service/Service.jl b/src/service/Service.jl index bbb5793..167f604 100644 --- a/src/service/Service.jl +++ b/src/service/Service.jl @@ -6,6 +6,7 @@ include("../Settings.jl") include("./AssetManager.jl") include("./ArgIO.jl") include("./Queuing.jl") -@reexport import .ArgIO, .AssetManager, .Queuing +include("./Execution.jl") +@reexport using .ArgIO, .AssetManager, .Queuing, .Execution end # module Service diff --git a/test/sciml.jl b/test/sciml.jl index 9e12bb4..6a67aa9 100644 --- a/test/sciml.jl +++ b/test/sciml.jl @@ -1,4 +1,4 @@ -using SimulationService, AlgebraicPetri, DataFrames, DifferentialEquations, ModelingToolkit, Symbolics, EasyModelAnalysis, Catlab, Catlab.CategoricalAlgebra, JSON3, UnPack, SimulationService.Interface.Operations +using SimulationService, AlgebraicPetri, DataFrames, DifferentialEquations, ModelingToolkit, Symbolics, EasyModelAnalysis, Catlab, Catlab.CategoricalAlgebra, JSON3, UnPack, SimulationService.Service.Execution.Interface.Operations using CSV, DataFrames, JSONTables using ForwardDiff @@ -21,31 +21,32 @@ u0 = string.(snames(petri)) .=> petri[:concentration] params = Dict(ps) initials = Dict(u0) -tspan = (0.0, 100.0) +timespan = (0.0, 100.0) -nt = (; model=petri, params, initials, tspan) +nt = (; model=petri, params, initials, timespan) body = Dict(pairs(nt)) j = JSON3.write(body) forecast_fn = _log("forecast.json") write(forecast_fn, j) -df = SimulationService.Interface.get_operation(:simulate)(; nt..., context=nothing) +df = SimulationService.Service.Execution.Interface.Available.get_operation(:simulate)(; nt..., context=nothing) @test df isa DataFrame params["t1"] = 0.1 -nt = (; model = petri, params, initials, tspan) -df2 = SimulationService.Interface.get_operation(:simulate)(; nt..., context=nothing) +nt = (; model = petri, params, initials, timespan) +df2 = SimulationService.Service.Execution.Interface.Available.get_operation(:simulate)(; nt..., context=nothing) timesteps = df.timestamp data = Dict(["Susceptible" => df[:, 2]]) -fit_args = (; model=petri, params, initials, dataset=df, timesteps_column="timestamp", feature_mappings=Dict("Susceptible(t)"=>"Susceptible")) +rename!(df, Dict("Susceptible(t)" => "Susceptible", "timestamp" => "timestep")) +fit_args = (; model=petri, params, initials, dataset=df[:, ["timestep", "Susceptible"]]) fit_body = Dict(pairs(fit_args)) fit_j = JSON3.write(fit_body) calibrate_fn = _log("calibrate.json") write(calibrate_fn, fit_j) -fitp = SimulationService.Interface.get_operation(:calibrate)(; fit_args..., context=nothing) -prob = SimulationService.Interface.Operations.Utils.to_prob(petri, params, initials, extrema(timesteps)) +fitp = SimulationService.Service.Execution.Interface.Available.get_operation(:calibrate)(; fit_args..., context=nothing) +prob = SimulationService.Service.Execution.Interface.Operations.Utils.to_prob(petri, params, initials, extrema(timesteps)) sys = prob.f.sys # example of dloss/dp