From 3042ca40c70ab375fc7e0b46eae296339f0cc65f Mon Sep 17 00:00:00 2001 From: fbrand-new Date: Mon, 18 Dec 2023 16:02:33 +0100 Subject: [PATCH 1/3] [ILLM] Added function support on ILLM interface, nws, nwc Tests corrected --- .../fake/fakeLLMDevice/fakeLLMDevice.cpp | 29 +++++++--- .../fake/fakeLLMDevice/fakeLLMDevice.h | 6 +- src/devices/messages/ILLMMsgs/ILLMMsgs.thrift | 13 +++-- .../idl_generated_code/ILLMMsgs_index.txt | 2 - .../yarp/dev/llm/return_ask.cpp | 14 ++--- .../yarp/dev/llm/return_ask.h | 5 +- .../yarp/dev/llm/return_getConversation.cpp | 2 +- .../yarp/dev/llm/return_getConversation.h | 6 +- .../LLM_nwc_yarp/LLM_nwc_yarp.cpp | 8 +-- .../LLM_nwc_yarp/LLM_nwc_yarp.h | 5 +- .../LLM_nws_yarp/ILLMServerImpl.cpp | 18 +++--- src/libYARP_dev/src/CMakeLists.txt | 1 + src/libYARP_dev/src/idl/LLM_Message.thrift | 20 +++++++ .../idl_generated_code/LLM_Message_index.txt | 2 + .../yarp/dev/LLM_Message.cpp} | 58 +++++++++---------- .../yarp/dev/LLM_Message.h} | 40 ++++++++----- src/libYARP_dev/src/yarp/dev/ILLM.h | 8 +-- src/libYARP_dev/src/yarp/dev/tests/ILLMTest.h | 14 ++++- src/yarpllmgui/ConversationModel.cpp | 8 +-- src/yarpllmgui/ConversationModel.h | 1 + 20 files changed, 155 insertions(+), 105 deletions(-) create mode 100644 src/libYARP_dev/src/idl/LLM_Message.thrift create mode 100644 src/libYARP_dev/src/idl_generated_code/LLM_Message_index.txt rename src/{devices/messages/ILLMMsgs/idl_generated_code/yarp/dev/llm/Message.cpp => libYARP_dev/src/idl_generated_code/yarp/dev/LLM_Message.cpp} (63%) rename src/{devices/messages/ILLMMsgs/idl_generated_code/yarp/dev/llm/Message.h => libYARP_dev/src/idl_generated_code/yarp/dev/LLM_Message.h} (61%) diff --git a/src/devices/fake/fakeLLMDevice/fakeLLMDevice.cpp b/src/devices/fake/fakeLLMDevice/fakeLLMDevice.cpp index 1ca396e42c1..af5967261a6 100644 --- a/src/devices/fake/fakeLLMDevice/fakeLLMDevice.cpp +++ b/src/devices/fake/fakeLLMDevice/fakeLLMDevice.cpp @@ -15,17 +15,17 @@ bool fakeLLMDevice::setPrompt(const std::string &prompt) return false; } - m_conversation.push_back(std::make_pair("system", prompt)); + m_conversation.push_back(yarp::dev::LLM_Message{"system", prompt,""}); return true; } bool fakeLLMDevice::readPrompt(std::string &oPrompt) { - for (const auto &[author, content] : m_conversation) + for (const auto &message : m_conversation) { - if (author == "system") + if (message.type == "system") { - oPrompt = content; + oPrompt = message.content; return true; } } @@ -33,17 +33,28 @@ bool fakeLLMDevice::readPrompt(std::string &oPrompt) return false; } -bool fakeLLMDevice::ask(const std::string &question, std::string &oAnswer) +bool fakeLLMDevice::ask(const std::string &question, yarp::dev::LLM_Message &oAnswer) { // In the fake device we ignore the question - std::string answer = "Fatti non foste per viver come bruti ma per seguir virtute e canoscenza"; - m_conversation.push_back(std::make_pair("user", question)); - m_conversation.push_back(std::make_pair("assistant", answer)); + yarp::dev::LLM_Message answer; + if(question == "function") + { + std::string function_name = "FakeFunction"; + std::string function_args = "{'arg1':'yes','arg2':'no'}"; + answer = yarp::dev::LLM_Message{"function",function_name,function_args}; + } + else + { + std::string answer_content = "Fatti non foste per viver come bruti ma per seguir virtute e canoscenza"; + answer = yarp::dev::LLM_Message{"assistant", answer_content,""}; + } + m_conversation.push_back(yarp::dev::LLM_Message{"user", question,""}); + m_conversation.push_back(answer); oAnswer = answer; return true; } -bool fakeLLMDevice::getConversation(std::vector>& oConversation) +bool fakeLLMDevice::getConversation(std::vector& oConversation) { oConversation = m_conversation; return true; diff --git a/src/devices/fake/fakeLLMDevice/fakeLLMDevice.h b/src/devices/fake/fakeLLMDevice/fakeLLMDevice.h index dbae5d4c11a..ff99cbf84d6 100644 --- a/src/devices/fake/fakeLLMDevice/fakeLLMDevice.h +++ b/src/devices/fake/fakeLLMDevice/fakeLLMDevice.h @@ -24,12 +24,12 @@ class fakeLLMDevice : public yarp::dev::ILLM, fakeLLMDevice() : m_conversation{} {} bool setPrompt(const std::string &prompt) override; bool readPrompt(std::string &oPromp) override; - bool ask(const std::string &question, std::string &oAnswer) override; - bool getConversation(std::vector> &oConversation) override; + bool ask(const std::string &question, yarp::dev::LLM_Message &oAnswer) override; + bool getConversation(std::vector &oConversation) override; bool deleteConversation() noexcept override; private: - std::vector> m_conversation; + std::vector m_conversation; }; #endif diff --git a/src/devices/messages/ILLMMsgs/ILLMMsgs.thrift b/src/devices/messages/ILLMMsgs/ILLMMsgs.thrift index 058f9404c9d..43bdfd70fa2 100644 --- a/src/devices/messages/ILLMMsgs/ILLMMsgs.thrift +++ b/src/devices/messages/ILLMMsgs/ILLMMsgs.thrift @@ -5,10 +5,11 @@ namespace yarp yarp.dev.llm -struct Message { - string sender; - string content; -} +struct LLM_Message { +} ( + yarp.name = "yarp::dev::LLM_Message" + yarp.includefile="yarp/dev/ILLM.h" +) struct return_readPrompt{ 1: bool ret = false; @@ -17,12 +18,12 @@ struct return_readPrompt{ struct return_ask{ 1: bool ret = false; - 2: string answer; + 2: LLM_Message answer; } struct return_getConversation{ 1: bool ret = false; - 2: list conversation; + 2: list conversation; } service ILLMMsgs { diff --git a/src/devices/messages/ILLMMsgs/idl_generated_code/ILLMMsgs_index.txt b/src/devices/messages/ILLMMsgs/idl_generated_code/ILLMMsgs_index.txt index 22797a43cda..2df1ad9c7c5 100644 --- a/src/devices/messages/ILLMMsgs/idl_generated_code/ILLMMsgs_index.txt +++ b/src/devices/messages/ILLMMsgs/idl_generated_code/ILLMMsgs_index.txt @@ -1,5 +1,3 @@ -yarp/dev/llm/Message.h -yarp/dev/llm/Message.cpp yarp/dev/llm/return_readPrompt.h yarp/dev/llm/return_readPrompt.cpp yarp/dev/llm/return_ask.h diff --git a/src/devices/messages/ILLMMsgs/idl_generated_code/yarp/dev/llm/return_ask.cpp b/src/devices/messages/ILLMMsgs/idl_generated_code/yarp/dev/llm/return_ask.cpp index 8dfbe4d8127..329f21195b8 100644 --- a/src/devices/messages/ILLMMsgs/idl_generated_code/yarp/dev/llm/return_ask.cpp +++ b/src/devices/messages/ILLMMsgs/idl_generated_code/yarp/dev/llm/return_ask.cpp @@ -14,7 +14,7 @@ namespace yarp::dev::llm { // Constructor with field values return_ask::return_ask(const bool ret, - const std::string& answer) : + const yarp::dev::LLM_Message& answer) : WirePortable(), ret(ret), answer(answer) @@ -27,7 +27,7 @@ bool return_ask::read(yarp::os::idl::WireReader& reader) if (!read_ret(reader)) { return false; } - if (!read_answer(reader)) { + if (!nested_read_answer(reader)) { return false; } if (reader.isError()) { @@ -55,7 +55,7 @@ bool return_ask::write(const yarp::os::idl::WireWriter& writer) const if (!write_ret(writer)) { return false; } - if (!write_answer(writer)) { + if (!nested_write_answer(writer)) { return false; } if (writer.isError()) { @@ -130,7 +130,7 @@ bool return_ask::read_answer(yarp::os::idl::WireReader& reader) reader.fail(); return false; } - if (!reader.readString(answer)) { + if (!reader.read(answer)) { reader.fail(); return false; } @@ -140,7 +140,7 @@ bool return_ask::read_answer(yarp::os::idl::WireReader& reader) // write answer field bool return_ask::write_answer(const yarp::os::idl::WireWriter& writer) const { - if (!writer.writeString(answer)) { + if (!writer.write(answer)) { return false; } return true; @@ -153,7 +153,7 @@ bool return_ask::nested_read_answer(yarp::os::idl::WireReader& reader) reader.fail(); return false; } - if (!reader.readString(answer)) { + if (!reader.readNested(answer)) { reader.fail(); return false; } @@ -163,7 +163,7 @@ bool return_ask::nested_read_answer(yarp::os::idl::WireReader& reader) // write (nested) answer field bool return_ask::nested_write_answer(const yarp::os::idl::WireWriter& writer) const { - if (!writer.writeString(answer)) { + if (!writer.writeNested(answer)) { return false; } return true; diff --git a/src/devices/messages/ILLMMsgs/idl_generated_code/yarp/dev/llm/return_ask.h b/src/devices/messages/ILLMMsgs/idl_generated_code/yarp/dev/llm/return_ask.h index ec6ef66e989..b610b003a60 100644 --- a/src/devices/messages/ILLMMsgs/idl_generated_code/yarp/dev/llm/return_ask.h +++ b/src/devices/messages/ILLMMsgs/idl_generated_code/yarp/dev/llm/return_ask.h @@ -13,6 +13,7 @@ #include #include +#include namespace yarp::dev::llm { @@ -22,14 +23,14 @@ class return_ask : public: // Fields bool ret{false}; - std::string answer{}; + yarp::dev::LLM_Message answer{}; // Default constructor return_ask() = default; // Constructor with field values return_ask(const bool ret, - const std::string& answer); + const yarp::dev::LLM_Message& answer); // Read structure on a Wire bool read(yarp::os::idl::WireReader& reader) override; diff --git a/src/devices/messages/ILLMMsgs/idl_generated_code/yarp/dev/llm/return_getConversation.cpp b/src/devices/messages/ILLMMsgs/idl_generated_code/yarp/dev/llm/return_getConversation.cpp index 4f884d6abc7..281310b767e 100644 --- a/src/devices/messages/ILLMMsgs/idl_generated_code/yarp/dev/llm/return_getConversation.cpp +++ b/src/devices/messages/ILLMMsgs/idl_generated_code/yarp/dev/llm/return_getConversation.cpp @@ -14,7 +14,7 @@ namespace yarp::dev::llm { // Constructor with field values return_getConversation::return_getConversation(const bool ret, - const std::vector& conversation) : + const std::vector& conversation) : WirePortable(), ret(ret), conversation(conversation) diff --git a/src/devices/messages/ILLMMsgs/idl_generated_code/yarp/dev/llm/return_getConversation.h b/src/devices/messages/ILLMMsgs/idl_generated_code/yarp/dev/llm/return_getConversation.h index 0a4edfc83d9..e1246583a3e 100644 --- a/src/devices/messages/ILLMMsgs/idl_generated_code/yarp/dev/llm/return_getConversation.h +++ b/src/devices/messages/ILLMMsgs/idl_generated_code/yarp/dev/llm/return_getConversation.h @@ -13,7 +13,7 @@ #include #include -#include +#include namespace yarp::dev::llm { @@ -23,14 +23,14 @@ class return_getConversation : public: // Fields bool ret{false}; - std::vector conversation{}; + std::vector conversation{}; // Default constructor return_getConversation() = default; // Constructor with field values return_getConversation(const bool ret, - const std::vector& conversation); + const std::vector& conversation); // Read structure on a Wire bool read(yarp::os::idl::WireReader& reader) override; diff --git a/src/devices/networkWrappers/LLM_nwc_yarp/LLM_nwc_yarp.cpp b/src/devices/networkWrappers/LLM_nwc_yarp/LLM_nwc_yarp.cpp index 137cb5bdd99..07b3e0aae6a 100644 --- a/src/devices/networkWrappers/LLM_nwc_yarp/LLM_nwc_yarp.cpp +++ b/src/devices/networkWrappers/LLM_nwc_yarp/LLM_nwc_yarp.cpp @@ -73,7 +73,7 @@ bool LLM_nwc_yarp::readPrompt(std::string &oPrompt) return ret.ret; } -bool LLM_nwc_yarp::ask(const std::string &question, std::string &oAnswer) +bool LLM_nwc_yarp::ask(const std::string &question, yarp::dev::LLM_Message &oAnswer) { auto ret = m_LLM_RPC.ask(question); @@ -81,12 +81,12 @@ bool LLM_nwc_yarp::ask(const std::string &question, std::string &oAnswer) return ret.ret; } -bool LLM_nwc_yarp::getConversation(std::vector> &oConversation) +bool LLM_nwc_yarp::getConversation(std::vector &oConversation) { auto ret = m_LLM_RPC.getConversation(); - for (const auto &[author, message] : ret.conversation) + for (const auto &message : ret.conversation) { - oConversation.push_back(std::make_pair(author, message)); + oConversation.push_back(message); } return true; diff --git a/src/devices/networkWrappers/LLM_nwc_yarp/LLM_nwc_yarp.h b/src/devices/networkWrappers/LLM_nwc_yarp/LLM_nwc_yarp.h index 31e76a4c354..c7216957ff7 100644 --- a/src/devices/networkWrappers/LLM_nwc_yarp/LLM_nwc_yarp.h +++ b/src/devices/networkWrappers/LLM_nwc_yarp/LLM_nwc_yarp.h @@ -7,6 +7,7 @@ #include #include #include +#include /** * @ingroup dev_impl_network_clients @@ -38,7 +39,7 @@ class LLM_nwc_yarp : public yarp::dev::DeviceDriver, //From ILLM bool setPrompt(const std::string& prompt) override; bool readPrompt(std::string& oPrompt) override; - bool ask(const std::string& question, std::string& oAnswer) override; - bool getConversation(std::vector>& oConversation) override; + bool ask(const std::string& question, yarp::dev::LLM_Message& oAnswer) override; + bool getConversation(std::vector& oConversation) override; bool deleteConversation() override; }; diff --git a/src/devices/networkWrappers/LLM_nws_yarp/ILLMServerImpl.cpp b/src/devices/networkWrappers/LLM_nws_yarp/ILLMServerImpl.cpp index a288661a3c7..508a25b57dd 100644 --- a/src/devices/networkWrappers/LLM_nws_yarp/ILLMServerImpl.cpp +++ b/src/devices/networkWrappers/LLM_nws_yarp/ILLMServerImpl.cpp @@ -55,7 +55,7 @@ yarp::dev::llm::return_ask ILLMRPCd::ask(const std::string& question) return ret; } - ret.ret = m_iLlm->ask(question, ret.answer); + ret.ret = m_iLlm->ask(question,ret.answer); if (ret.ret) { m_stream_conversation(); @@ -66,7 +66,7 @@ yarp::dev::llm::return_ask ILLMRPCd::ask(const std::string& question) void ILLMRPCd::m_stream_conversation() { - std::vector> conversation; + std::vector conversation; if (!m_iLlm->getConversation(conversation)) { yCError(LLMSERVER, "Unable to retrieve the conversation"); return; @@ -74,10 +74,11 @@ void ILLMRPCd::m_stream_conversation() auto& bot = m_streaming_port.prepare(); auto& list = bot.addList(); - for (const auto& [author, message] : conversation) { + for (const auto& message : conversation) { auto& message_bot = list.addList(); - message_bot.addString(author); - message_bot.addString(message); + message_bot.addString(message.type); + message_bot.addString(message.content); + message_bot.addString(message.arguments); } m_streaming_port.write(); @@ -94,12 +95,9 @@ yarp::dev::llm::return_getConversation ILLMRPCd::getConversation() return ret; } - std::vector> conversation; + std::vector conversation; ret.ret = m_iLlm->getConversation(conversation); - - for (const auto& [author, message] : conversation) { - ret.conversation.push_back(yarp::dev::llm::Message(author, message)); - } + ret.conversation = conversation; return ret; } diff --git a/src/libYARP_dev/src/CMakeLists.txt b/src/libYARP_dev/src/CMakeLists.txt index 844cb7efe1e..b76e836528d 100644 --- a/src/libYARP_dev/src/CMakeLists.txt +++ b/src/libYARP_dev/src/CMakeLists.txt @@ -267,6 +267,7 @@ set(YARP_dev_IDL idl/MobileBaseVelocity.thrift idl/AudioPlayerStatus.thrift idl/AudioRecorderStatus.thrift + idl/LLM_Message.thrift ) if(TARGET YARP::YARP_math) diff --git a/src/libYARP_dev/src/idl/LLM_Message.thrift b/src/libYARP_dev/src/idl/LLM_Message.thrift new file mode 100644 index 00000000000..da5e9efc6bd --- /dev/null +++ b/src/libYARP_dev/src/idl/LLM_Message.thrift @@ -0,0 +1,20 @@ +/* + * SPDX-FileCopyrightText: 2006-2021 Istituto Italiano di Tecnologia (IIT) + * SPDX-License-Identifier: BSD-3-Clause + */ + +namespace yarp yarp.dev + +struct LLM_Message +{ + /** type of message (can be prompt, assistant, function, ...) */ + 1: string type; + /** content of the message */ + 2: string content; + /** possible extra arguments */ + 3: string arguments; +} +( + yarp.api.include = "yarp/dev/api.h" + yarp.api.keyword = "YARP_dev_API" +) diff --git a/src/libYARP_dev/src/idl_generated_code/LLM_Message_index.txt b/src/libYARP_dev/src/idl_generated_code/LLM_Message_index.txt new file mode 100644 index 00000000000..db54107def0 --- /dev/null +++ b/src/libYARP_dev/src/idl_generated_code/LLM_Message_index.txt @@ -0,0 +1,2 @@ +yarp/dev/LLM_Message.h +yarp/dev/LLM_Message.cpp diff --git a/src/devices/messages/ILLMMsgs/idl_generated_code/yarp/dev/llm/Message.cpp b/src/libYARP_dev/src/idl_generated_code/yarp/dev/LLM_Message.cpp similarity index 63% rename from src/devices/messages/ILLMMsgs/idl_generated_code/yarp/dev/llm/Message.cpp rename to src/libYARP_dev/src/idl_generated_code/yarp/dev/LLM_Message.cpp index 15ddb1377be..522a1bd5665 100644 --- a/src/devices/messages/ILLMMsgs/idl_generated_code/yarp/dev/llm/Message.cpp +++ b/src/libYARP_dev/src/idl_generated_code/yarp/dev/LLM_Message.cpp @@ -8,23 +8,23 @@ // This is an automatically generated file. // It could get re-generated if the ALLOW_IDL_GENERATION flag is on. -#include +#include -namespace yarp::dev::llm { +namespace yarp::dev { // Constructor with field values -Message::Message(const std::string& sender, - const std::string& content) : +LLM_Message::LLM_Message(const std::string& type, + const std::string& content) : WirePortable(), - sender(sender), + type(type), content(content) { } // Read structure on a Wire -bool Message::read(yarp::os::idl::WireReader& reader) +bool LLM_Message::read(yarp::os::idl::WireReader& reader) { - if (!read_sender(reader)) { + if (!read_type(reader)) { return false; } if (!read_content(reader)) { @@ -37,7 +37,7 @@ bool Message::read(yarp::os::idl::WireReader& reader) } // Read structure on a Connection -bool Message::read(yarp::os::ConnectionReader& connection) +bool LLM_Message::read(yarp::os::ConnectionReader& connection) { yarp::os::idl::WireReader reader(connection); if (!reader.readListHeader(2)) { @@ -50,9 +50,9 @@ bool Message::read(yarp::os::ConnectionReader& connection) } // Write structure on a Wire -bool Message::write(const yarp::os::idl::WireWriter& writer) const +bool LLM_Message::write(const yarp::os::idl::WireWriter& writer) const { - if (!write_sender(writer)) { + if (!write_type(writer)) { return false; } if (!write_content(writer)) { @@ -65,7 +65,7 @@ bool Message::write(const yarp::os::idl::WireWriter& writer) const } // Write structure on a Connection -bool Message::write(yarp::os::ConnectionWriter& connection) const +bool LLM_Message::write(yarp::os::ConnectionWriter& connection) const { yarp::os::idl::WireWriter writer(connection); if (!writer.writeListHeader(2)) { @@ -78,7 +78,7 @@ bool Message::write(yarp::os::ConnectionWriter& connection) const } // Convert to a printable string -std::string Message::toString() const +std::string LLM_Message::toString() const { yarp::os::Bottle b; if (!yarp::os::Portable::copyPortable(*this, b)) { @@ -87,54 +87,54 @@ std::string Message::toString() const return b.toString(); } -// read sender field -bool Message::read_sender(yarp::os::idl::WireReader& reader) +// read type field +bool LLM_Message::read_type(yarp::os::idl::WireReader& reader) { if (reader.noMore()) { reader.fail(); return false; } - if (!reader.readString(sender)) { + if (!reader.readString(type)) { reader.fail(); return false; } return true; } -// write sender field -bool Message::write_sender(const yarp::os::idl::WireWriter& writer) const +// write type field +bool LLM_Message::write_type(const yarp::os::idl::WireWriter& writer) const { - if (!writer.writeString(sender)) { + if (!writer.writeString(type)) { return false; } return true; } -// read (nested) sender field -bool Message::nested_read_sender(yarp::os::idl::WireReader& reader) +// read (nested) type field +bool LLM_Message::nested_read_type(yarp::os::idl::WireReader& reader) { if (reader.noMore()) { reader.fail(); return false; } - if (!reader.readString(sender)) { + if (!reader.readString(type)) { reader.fail(); return false; } return true; } -// write (nested) sender field -bool Message::nested_write_sender(const yarp::os::idl::WireWriter& writer) const +// write (nested) type field +bool LLM_Message::nested_write_type(const yarp::os::idl::WireWriter& writer) const { - if (!writer.writeString(sender)) { + if (!writer.writeString(type)) { return false; } return true; } // read content field -bool Message::read_content(yarp::os::idl::WireReader& reader) +bool LLM_Message::read_content(yarp::os::idl::WireReader& reader) { if (reader.noMore()) { reader.fail(); @@ -148,7 +148,7 @@ bool Message::read_content(yarp::os::idl::WireReader& reader) } // write content field -bool Message::write_content(const yarp::os::idl::WireWriter& writer) const +bool LLM_Message::write_content(const yarp::os::idl::WireWriter& writer) const { if (!writer.writeString(content)) { return false; @@ -157,7 +157,7 @@ bool Message::write_content(const yarp::os::idl::WireWriter& writer) const } // read (nested) content field -bool Message::nested_read_content(yarp::os::idl::WireReader& reader) +bool LLM_Message::nested_read_content(yarp::os::idl::WireReader& reader) { if (reader.noMore()) { reader.fail(); @@ -171,7 +171,7 @@ bool Message::nested_read_content(yarp::os::idl::WireReader& reader) } // write (nested) content field -bool Message::nested_write_content(const yarp::os::idl::WireWriter& writer) const +bool LLM_Message::nested_write_content(const yarp::os::idl::WireWriter& writer) const { if (!writer.writeString(content)) { return false; @@ -179,4 +179,4 @@ bool Message::nested_write_content(const yarp::os::idl::WireWriter& writer) cons return true; } -} // namespace yarp::dev::llm +} // namespace yarp::dev diff --git a/src/devices/messages/ILLMMsgs/idl_generated_code/yarp/dev/llm/Message.h b/src/libYARP_dev/src/idl_generated_code/yarp/dev/LLM_Message.h similarity index 61% rename from src/devices/messages/ILLMMsgs/idl_generated_code/yarp/dev/llm/Message.h rename to src/libYARP_dev/src/idl_generated_code/yarp/dev/LLM_Message.h index 475567a0c91..c0fa0738308 100644 --- a/src/devices/messages/ILLMMsgs/idl_generated_code/yarp/dev/llm/Message.h +++ b/src/libYARP_dev/src/idl_generated_code/yarp/dev/LLM_Message.h @@ -8,28 +8,36 @@ // This is an automatically generated file. // It could get re-generated if the ALLOW_IDL_GENERATION flag is on. -#ifndef YARP_THRIFT_GENERATOR_STRUCT_MESSAGE_H -#define YARP_THRIFT_GENERATOR_STRUCT_MESSAGE_H +#ifndef YARP_THRIFT_GENERATOR_STRUCT_LLM_MESSAGE_H +#define YARP_THRIFT_GENERATOR_STRUCT_LLM_MESSAGE_H + +#include #include #include -namespace yarp::dev::llm { +namespace yarp::dev { -class Message : +class YARP_dev_API LLM_Message : public yarp::os::idl::WirePortable { public: // Fields - std::string sender{}; + /** + * type of message (can be prompt, assistant, function, ...) + */ + std::string type{}; + /** + * content of the message + */ std::string content{}; // Default constructor - Message() = default; + LLM_Message() = default; // Constructor with field values - Message(const std::string& sender, - const std::string& content); + LLM_Message(const std::string& type, + const std::string& content); // Read structure on a Wire bool read(yarp::os::idl::WireReader& reader) override; @@ -47,14 +55,14 @@ class Message : std::string toString() const; // If you want to serialize this class without nesting, use this helper - typedef yarp::os::idl::Unwrapped unwrapped; + typedef yarp::os::idl::Unwrapped unwrapped; private: - // read/write sender field - bool read_sender(yarp::os::idl::WireReader& reader); - bool write_sender(const yarp::os::idl::WireWriter& writer) const; - bool nested_read_sender(yarp::os::idl::WireReader& reader); - bool nested_write_sender(const yarp::os::idl::WireWriter& writer) const; + // read/write type field + bool read_type(yarp::os::idl::WireReader& reader); + bool write_type(const yarp::os::idl::WireWriter& writer) const; + bool nested_read_type(yarp::os::idl::WireReader& reader); + bool nested_write_type(const yarp::os::idl::WireWriter& writer) const; // read/write content field bool read_content(yarp::os::idl::WireReader& reader); @@ -63,6 +71,6 @@ class Message : bool nested_write_content(const yarp::os::idl::WireWriter& writer) const; }; -} // namespace yarp::dev::llm +} // namespace yarp::dev -#endif // YARP_THRIFT_GENERATOR_STRUCT_MESSAGE_H +#endif // YARP_THRIFT_GENERATOR_STRUCT_LLM_MESSAGE_H diff --git a/src/libYARP_dev/src/yarp/dev/ILLM.h b/src/libYARP_dev/src/yarp/dev/ILLM.h index 1ee41f568ab..7cebbea6e37 100644 --- a/src/libYARP_dev/src/yarp/dev/ILLM.h +++ b/src/libYARP_dev/src/yarp/dev/ILLM.h @@ -7,13 +7,11 @@ #define YARP_DEV_ILLM_H #include +#include #include #include -typedef std::string Author; -typedef std::string Content; - namespace yarp::dev { class ILLM; } // namespace yarp @@ -51,14 +49,14 @@ class YARP_dev_API yarp::dev::ILLM * @param answer the returned answer * @return true/false */ - virtual bool ask(const std::string& question, std::string& answer) = 0; + virtual bool ask(const std::string& question, yarp::dev::LLM_Message& answer) = 0; /** * Retrieves the whole conversation * @param conversation the conversation * @return true/false */ - virtual bool getConversation(std::vector>& conversation) = 0; + virtual bool getConversation(std::vector& conversation) = 0; /** * Delete the conversation and clear the system context from any internally stored context. diff --git a/src/libYARP_dev/src/yarp/dev/tests/ILLMTest.h b/src/libYARP_dev/src/yarp/dev/tests/ILLMTest.h index 3d992f92a7c..c991a10175e 100644 --- a/src/libYARP_dev/src/yarp/dev/tests/ILLMTest.h +++ b/src/libYARP_dev/src/yarp/dev/tests/ILLMTest.h @@ -7,6 +7,7 @@ #define ILLMTEST_H #include +#include #include using namespace yarp::dev; @@ -25,11 +26,20 @@ inline void exec_iLLM_test_1(yarp::dev::ILLM* illm) b = illm->readPrompt(prompt); CHECK(b); - std::string answer; + yarp::dev::LLM_Message answer; b = illm->ask("A question", answer); CHECK(b); + CHECK(answer.type == "assistant"); + CHECK(answer.content != ""); + CHECK(answer.arguments == ""); // No arguments in a non-function call - std::vector> conversation; + b = illm->ask("function",answer); + CHECK(b); + CHECK(answer.type == "function"); + CHECK(answer.content != ""); + CHECK(answer.arguments != ""); + + std::vector conversation; b = illm->getConversation(conversation); CHECK(b); } diff --git a/src/yarpllmgui/ConversationModel.cpp b/src/yarpllmgui/ConversationModel.cpp index d3a849829ab..84e3e264948 100644 --- a/src/yarpllmgui/ConversationModel.cpp +++ b/src/yarpllmgui/ConversationModel.cpp @@ -132,15 +132,15 @@ void ConversationModel::refreshConversation() return; } - std::vector> conversation; + std::vector conversation; if (!m_iLlm->getConversation(conversation)) { yError() << "Unable to get conversation"; return; } - for (const auto& [author, message] : conversation) { - addMessage(Message(QString::fromStdString(author), QString::fromStdString(message))); + for (const auto& message : conversation) { + addMessage(Message(QString::fromStdString(message.type), QString::fromStdString(message.content))); } } @@ -154,7 +154,7 @@ void ConversationModel::addMessage(const Message& message) void ConversationModel::sendMessage(const QString& message) { - std::string answer; + yarp::dev::LLM_Message answer; if (!m_iLlm) { yError() << m_no_device_message; diff --git a/src/yarpllmgui/ConversationModel.h b/src/yarpllmgui/ConversationModel.h index 5e09eb2e759..b83b7b8f704 100644 --- a/src/yarpllmgui/ConversationModel.h +++ b/src/yarpllmgui/ConversationModel.h @@ -13,6 +13,7 @@ #include #include +#include #include #include From ffb7f82cf914f19af106b85f02dec12690d042be Mon Sep 17 00:00:00 2001 From: fbrand-new Date: Thu, 11 Jan 2024 12:16:46 +0100 Subject: [PATCH 2/3] [ILLM] LLM message has function call params and args as lists Fake device and test correction --- doc/release/master.md | 2 + .../fake/fakeLLMDevice/fakeLLMDevice.cpp | 11 +- .../LLM_nws_yarp/ILLMServerImpl.cpp | 12 +- src/libYARP_dev/src/idl/LLM_Message.thrift | 6 +- .../yarp/dev/LLM_Message.cpp | 204 +++++++++++++++++- .../idl_generated_code/yarp/dev/LLM_Message.h | 24 ++- src/libYARP_dev/src/yarp/dev/tests/ILLMTest.h | 11 +- 7 files changed, 254 insertions(+), 16 deletions(-) diff --git a/doc/release/master.md b/doc/release/master.md index f646cf6762c..bd7d6d5b418 100644 --- a/doc/release/master.md +++ b/doc/release/master.md @@ -34,6 +34,8 @@ New Features * Added new command line tool `yarpDeviceParamParserGenerator`. See official yarp documentation (cmd_yarpDeviceParamParserGenerator.dox) +* Added LLM_Message data type to propagate LLM answers + ### Devices #### deviceBundler diff --git a/src/devices/fake/fakeLLMDevice/fakeLLMDevice.cpp b/src/devices/fake/fakeLLMDevice/fakeLLMDevice.cpp index af5967261a6..d98c6b3e2fa 100644 --- a/src/devices/fake/fakeLLMDevice/fakeLLMDevice.cpp +++ b/src/devices/fake/fakeLLMDevice/fakeLLMDevice.cpp @@ -15,7 +15,7 @@ bool fakeLLMDevice::setPrompt(const std::string &prompt) return false; } - m_conversation.push_back(yarp::dev::LLM_Message{"system", prompt,""}); + m_conversation.push_back(yarp::dev::LLM_Message("system", prompt,{},{})); return true; } @@ -40,15 +40,16 @@ bool fakeLLMDevice::ask(const std::string &question, yarp::dev::LLM_Message &oAn if(question == "function") { std::string function_name = "FakeFunction"; - std::string function_args = "{'arg1':'yes','arg2':'no'}"; - answer = yarp::dev::LLM_Message{"function",function_name,function_args}; + std::vector function_params = {"arg1","arg2"}; + std::vector function_args = {"yes","no"}; + answer = yarp::dev::LLM_Message("function",function_name,function_params,function_args); } else { std::string answer_content = "Fatti non foste per viver come bruti ma per seguir virtute e canoscenza"; - answer = yarp::dev::LLM_Message{"assistant", answer_content,""}; + answer = yarp::dev::LLM_Message("assistant", answer_content,{},{}); } - m_conversation.push_back(yarp::dev::LLM_Message{"user", question,""}); + m_conversation.push_back(yarp::dev::LLM_Message("user", question,{},{})); m_conversation.push_back(answer); oAnswer = answer; return true; diff --git a/src/devices/networkWrappers/LLM_nws_yarp/ILLMServerImpl.cpp b/src/devices/networkWrappers/LLM_nws_yarp/ILLMServerImpl.cpp index 508a25b57dd..ffafa552b44 100644 --- a/src/devices/networkWrappers/LLM_nws_yarp/ILLMServerImpl.cpp +++ b/src/devices/networkWrappers/LLM_nws_yarp/ILLMServerImpl.cpp @@ -78,7 +78,17 @@ void ILLMRPCd::m_stream_conversation() auto& message_bot = list.addList(); message_bot.addString(message.type); message_bot.addString(message.content); - message_bot.addString(message.arguments); + auto& params_bot = message_bot.addList(); + for(const auto& params: message.parameters) + { + params_bot.addString(params); + } + + auto& args_bot = message_bot.addList(); + for(const auto& args: message.arguments) + { + args_bot.addString(args); + } } m_streaming_port.write(); diff --git a/src/libYARP_dev/src/idl/LLM_Message.thrift b/src/libYARP_dev/src/idl/LLM_Message.thrift index da5e9efc6bd..2dbc78323db 100644 --- a/src/libYARP_dev/src/idl/LLM_Message.thrift +++ b/src/libYARP_dev/src/idl/LLM_Message.thrift @@ -11,8 +11,10 @@ struct LLM_Message 1: string type; /** content of the message */ 2: string content; - /** possible extra arguments */ - 3: string arguments; + /** possible parameters */ + 3: list parameters = []; + /** possible arguments of the parameters */ + 4: list arguments = []; } ( yarp.api.include = "yarp/dev/api.h" diff --git a/src/libYARP_dev/src/idl_generated_code/yarp/dev/LLM_Message.cpp b/src/libYARP_dev/src/idl_generated_code/yarp/dev/LLM_Message.cpp index 522a1bd5665..2f27d1cccfc 100644 --- a/src/libYARP_dev/src/idl_generated_code/yarp/dev/LLM_Message.cpp +++ b/src/libYARP_dev/src/idl_generated_code/yarp/dev/LLM_Message.cpp @@ -14,10 +14,14 @@ namespace yarp::dev { // Constructor with field values LLM_Message::LLM_Message(const std::string& type, - const std::string& content) : + const std::string& content, + const std::vector& parameters, + const std::vector& arguments) : WirePortable(), type(type), - content(content) + content(content), + parameters(parameters), + arguments(arguments) { } @@ -30,6 +34,12 @@ bool LLM_Message::read(yarp::os::idl::WireReader& reader) if (!read_content(reader)) { return false; } + if (!read_parameters(reader)) { + return false; + } + if (!read_arguments(reader)) { + return false; + } if (reader.isError()) { return false; } @@ -40,7 +50,7 @@ bool LLM_Message::read(yarp::os::idl::WireReader& reader) bool LLM_Message::read(yarp::os::ConnectionReader& connection) { yarp::os::idl::WireReader reader(connection); - if (!reader.readListHeader(2)) { + if (!reader.readListHeader(4)) { return false; } if (!read(reader)) { @@ -58,6 +68,12 @@ bool LLM_Message::write(const yarp::os::idl::WireWriter& writer) const if (!write_content(writer)) { return false; } + if (!write_parameters(writer)) { + return false; + } + if (!write_arguments(writer)) { + return false; + } if (writer.isError()) { return false; } @@ -68,7 +84,7 @@ bool LLM_Message::write(const yarp::os::idl::WireWriter& writer) const bool LLM_Message::write(yarp::os::ConnectionWriter& connection) const { yarp::os::idl::WireWriter writer(connection); - if (!writer.writeListHeader(2)) { + if (!writer.writeListHeader(4)) { return false; } if (!write(writer)) { @@ -179,4 +195,184 @@ bool LLM_Message::nested_write_content(const yarp::os::idl::WireWriter& writer) return true; } +// read parameters field +bool LLM_Message::read_parameters(yarp::os::idl::WireReader& reader) +{ + size_t _csize; + yarp::os::idl::WireState _etype; + reader.readListBegin(_etype, _csize); + // WireReader removes BOTTLE_TAG_LIST from the tag + constexpr int expected_tag = ((BOTTLE_TAG_STRING) & (~BOTTLE_TAG_LIST)); + if constexpr (expected_tag != 0) { + if (_csize != 0 && _etype.code != expected_tag) { + return false; + } + } + parameters.resize(_csize); + for (size_t _i = 0; _i < _csize; ++_i) { + if (reader.noMore()) { + reader.fail(); + return false; + } + if (!reader.readString(parameters[_i])) { + reader.fail(); + return false; + } + } + reader.readListEnd(); + return true; +} + +// write parameters field +bool LLM_Message::write_parameters(const yarp::os::idl::WireWriter& writer) const +{ + if (!writer.writeListBegin(BOTTLE_TAG_STRING, parameters.size())) { + return false; + } + for (const auto& _item : parameters) { + if (!writer.writeString(_item, true)) { + return false; + } + } + if (!writer.writeListEnd()) { + return false; + } + return true; +} + +// read (nested) parameters field +bool LLM_Message::nested_read_parameters(yarp::os::idl::WireReader& reader) +{ + size_t _csize; + yarp::os::idl::WireState _etype; + reader.readListBegin(_etype, _csize); + // WireReader removes BOTTLE_TAG_LIST from the tag + constexpr int expected_tag = ((BOTTLE_TAG_STRING) & (~BOTTLE_TAG_LIST)); + if constexpr (expected_tag != 0) { + if (_csize != 0 && _etype.code != expected_tag) { + return false; + } + } + parameters.resize(_csize); + for (size_t _i = 0; _i < _csize; ++_i) { + if (reader.noMore()) { + reader.fail(); + return false; + } + if (!reader.readString(parameters[_i])) { + reader.fail(); + return false; + } + } + reader.readListEnd(); + return true; +} + +// write (nested) parameters field +bool LLM_Message::nested_write_parameters(const yarp::os::idl::WireWriter& writer) const +{ + if (!writer.writeListBegin(BOTTLE_TAG_STRING, parameters.size())) { + return false; + } + for (const auto& _item : parameters) { + if (!writer.writeString(_item, true)) { + return false; + } + } + if (!writer.writeListEnd()) { + return false; + } + return true; +} + +// read arguments field +bool LLM_Message::read_arguments(yarp::os::idl::WireReader& reader) +{ + size_t _csize; + yarp::os::idl::WireState _etype; + reader.readListBegin(_etype, _csize); + // WireReader removes BOTTLE_TAG_LIST from the tag + constexpr int expected_tag = ((BOTTLE_TAG_STRING) & (~BOTTLE_TAG_LIST)); + if constexpr (expected_tag != 0) { + if (_csize != 0 && _etype.code != expected_tag) { + return false; + } + } + arguments.resize(_csize); + for (size_t _i = 0; _i < _csize; ++_i) { + if (reader.noMore()) { + reader.fail(); + return false; + } + if (!reader.readString(arguments[_i])) { + reader.fail(); + return false; + } + } + reader.readListEnd(); + return true; +} + +// write arguments field +bool LLM_Message::write_arguments(const yarp::os::idl::WireWriter& writer) const +{ + if (!writer.writeListBegin(BOTTLE_TAG_STRING, arguments.size())) { + return false; + } + for (const auto& _item : arguments) { + if (!writer.writeString(_item, true)) { + return false; + } + } + if (!writer.writeListEnd()) { + return false; + } + return true; +} + +// read (nested) arguments field +bool LLM_Message::nested_read_arguments(yarp::os::idl::WireReader& reader) +{ + size_t _csize; + yarp::os::idl::WireState _etype; + reader.readListBegin(_etype, _csize); + // WireReader removes BOTTLE_TAG_LIST from the tag + constexpr int expected_tag = ((BOTTLE_TAG_STRING) & (~BOTTLE_TAG_LIST)); + if constexpr (expected_tag != 0) { + if (_csize != 0 && _etype.code != expected_tag) { + return false; + } + } + arguments.resize(_csize); + for (size_t _i = 0; _i < _csize; ++_i) { + if (reader.noMore()) { + reader.fail(); + return false; + } + if (!reader.readString(arguments[_i])) { + reader.fail(); + return false; + } + } + reader.readListEnd(); + return true; +} + +// write (nested) arguments field +bool LLM_Message::nested_write_arguments(const yarp::os::idl::WireWriter& writer) const +{ + if (!writer.writeListBegin(BOTTLE_TAG_STRING, arguments.size())) { + return false; + } + for (const auto& _item : arguments) { + if (!writer.writeString(_item, true)) { + return false; + } + } + if (!writer.writeListEnd()) { + return false; + } + return true; +} + } // namespace yarp::dev diff --git a/src/libYARP_dev/src/idl_generated_code/yarp/dev/LLM_Message.h b/src/libYARP_dev/src/idl_generated_code/yarp/dev/LLM_Message.h index c0fa0738308..e9f7b3f458a 100644 --- a/src/libYARP_dev/src/idl_generated_code/yarp/dev/LLM_Message.h +++ b/src/libYARP_dev/src/idl_generated_code/yarp/dev/LLM_Message.h @@ -31,13 +31,23 @@ class YARP_dev_API LLM_Message : * content of the message */ std::string content{}; + /** + * possible parameters + */ + std::vector parameters{}; + /** + * possible arguments of the parameters + */ + std::vector arguments{}; // Default constructor LLM_Message() = default; // Constructor with field values LLM_Message(const std::string& type, - const std::string& content); + const std::string& content, + const std::vector& parameters, + const std::vector& arguments); // Read structure on a Wire bool read(yarp::os::idl::WireReader& reader) override; @@ -69,6 +79,18 @@ class YARP_dev_API LLM_Message : bool write_content(const yarp::os::idl::WireWriter& writer) const; bool nested_read_content(yarp::os::idl::WireReader& reader); bool nested_write_content(const yarp::os::idl::WireWriter& writer) const; + + // read/write parameters field + bool read_parameters(yarp::os::idl::WireReader& reader); + bool write_parameters(const yarp::os::idl::WireWriter& writer) const; + bool nested_read_parameters(yarp::os::idl::WireReader& reader); + bool nested_write_parameters(const yarp::os::idl::WireWriter& writer) const; + + // read/write arguments field + bool read_arguments(yarp::os::idl::WireReader& reader); + bool write_arguments(const yarp::os::idl::WireWriter& writer) const; + bool nested_read_arguments(yarp::os::idl::WireReader& reader); + bool nested_write_arguments(const yarp::os::idl::WireWriter& writer) const; }; } // namespace yarp::dev diff --git a/src/libYARP_dev/src/yarp/dev/tests/ILLMTest.h b/src/libYARP_dev/src/yarp/dev/tests/ILLMTest.h index c991a10175e..07518a46177 100644 --- a/src/libYARP_dev/src/yarp/dev/tests/ILLMTest.h +++ b/src/libYARP_dev/src/yarp/dev/tests/ILLMTest.h @@ -8,6 +8,7 @@ #include #include + #include using namespace yarp::dev; @@ -31,13 +32,17 @@ inline void exec_iLLM_test_1(yarp::dev::ILLM* illm) CHECK(b); CHECK(answer.type == "assistant"); CHECK(answer.content != ""); - CHECK(answer.arguments == ""); // No arguments in a non-function call + CHECK(answer.parameters.empty()); // No parameters in a non-function call + CHECK(answer.arguments.empty()); // No arguments in a non-function call - b = illm->ask("function",answer); + // The fake device returns a function_call if the question is "function" + b = illm->ask("function", answer); CHECK(b); CHECK(answer.type == "function"); CHECK(answer.content != ""); - CHECK(answer.arguments != ""); + CHECK(!answer.parameters.empty()); + CHECK(!answer.arguments.empty()); + std::vector conversation; b = illm->getConversation(conversation); From f6a03143ac31bac6c2efc7f2ee6bc0873ad53590 Mon Sep 17 00:00:00 2001 From: Marco Randazzo Date: Fri, 9 Feb 2024 16:49:17 +0100 Subject: [PATCH 3/3] python bindings fix --- bindings/yarp.i | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bindings/yarp.i b/bindings/yarp.i index 16573b9237d..92ed71a9c2c 100644 --- a/bindings/yarp.i +++ b/bindings/yarp.i @@ -1584,8 +1584,8 @@ typedef yarp::os::BufferedPort BufferedPortImageRgbFloat; return self->readPrompt(oPropmt[0]); } - bool ask(const std::string& question, std::vector& answer) { - return self->ask(question, answer[0]); + bool ask(const std::string& question, yarp::dev::LLM_Message& answer) { + return self->ask(question, answer); } }