feat(skymp5-server): add binary animations encoding (#1700)

skymp5-server/cpp/CMakeLists.txt

@@ -21,6 +21,19 @@ if(WIN32)
   add_dependencies(skyrim-platform MpClientPlugin)
 endif()
 
+#
+# messages
+#
+
+file(GLOB_RECURSE src "${CMAKE_CURRENT_SOURCE_DIR}/messages/*")
+list(APPEND src "${CMAKE_SOURCE_DIR}/.clang-format")
+add_library(messages STATIC ${src})
+target_include_directories(messages PUBLIC "${CMAKE_CURRENT_LIST_DIR}/messages")
+target_link_libraries(messages PUBLIC viet)
+target_link_libraries(messages PUBLIC papyrus-vm-lib) # Utils.h: stricmp
+apply_default_settings(TARGETS messages)
+list(APPEND VCPKG_DEPENDENT messages)
+
 #
 # mp_common
 #
@@ -30,8 +43,7 @@ list(APPEND src "${CMAKE_SOURCE_DIR}/.clang-format")
 add_library(mp_common STATIC ${src})
 target_compile_definitions(mp_common PUBLIC MAX_PLAYERS=1000)
 target_include_directories(mp_common PUBLIC "${CMAKE_CURRENT_LIST_DIR}/mp_common")
-target_include_directories(mp_common PUBLIC "${CMAKE_CURRENT_LIST_DIR}/third_party")
-target_link_libraries(mp_common PUBLIC viet)
+target_link_libraries(mp_common PUBLIC messages viet)
 apply_default_settings(TARGETS mp_common)
 list(APPEND VCPKG_DEPENDENT mp_common)
 if(WIN32)

skymp5-server/cpp/client/main.cpp

@@ -1,11 +1,43 @@
+#include "MessageSerializerFactory.h"
 #include "MpClientPlugin.h"
 #include <cstdint>
+#include <nlohmann/json.hpp>
 
 namespace {
 MpClientPlugin::State& GetState()
 {
-  static MpClientPlugin::State state;
-  return state;
+  static MpClientPlugin::State g_state;
+  return g_state;
+}
+
+MessageSerializer& GetMessageSerializer()
+{
+  static std::shared_ptr<MessageSerializer> g_serializer =
+    MessageSerializerFactory::CreateMessageSerializer();
+  return *g_serializer;
+}
+
+void MySerializeMessage(const char* jsonContent,
+                        SLNet::BitStream& outputStream)
+{
+  GetMessageSerializer().Serialize(jsonContent, outputStream);
+}
+
+bool MyDeserializeMessage(const uint8_t* data, size_t length,
+                          std::string& outJsonContent)
+{
+  std::optional<DeserializeResult> result =
+    GetMessageSerializer().Deserialize(data, length);
+  if (!result) {
+    return false;
+  }
+
+  // TODO(perf): there should be a faster way to get JS object from binary
+  // (without extra json building)
+  nlohmann::json outJson;
+  result->message->WriteJson(outJson);
+  outJsonContent = outJson.dump();
+  return true;
 }
 }
 
@@ -34,28 +66,13 @@ __declspec(dllexport) bool IsConnected()
 __declspec(dllexport) void Tick(MpClientPlugin::OnPacket onPacket, void* state)
 {
 
-  return MpClientPlugin::Tick(GetState(), onPacket, state);
+  return MpClientPlugin::Tick(GetState(), onPacket, MyDeserializeMessage,
+                              state);
 }
 
 __declspec(dllexport) void Send(const char* jsonContent, bool reliable)
 {
-  return MpClientPlugin::Send(GetState(), jsonContent, reliable);
-}
-
-__declspec(dllexport) bool SKSEPlugin_Query(void* skse, void* info)
-{
-  struct PluginInfo
-  {
-    uint32_t infoVersion = 1;
-    const char* name = "MpClientPlugin";
-    uint32_t version = 1;
-  };
-  new (info) PluginInfo;
-  return true;
-}
-
-__declspec(dllexport) bool SKSEPlugin_Load(void* skse)
-{
-  return true;
+  return MpClientPlugin::Send(GetState(), jsonContent, reliable,
+                              MySerializeMessage);
 }
 }

skymp5-server/cpp/messages/MessageBase.cpp

@@ -0,0 +1,3 @@
+#include "MessageBase.h"
+
+MessageBase::~MessageBase() = default;

skymp5-server/cpp/messages/MessageBase.h

@@ -0,0 +1,14 @@
+#pragma once
+#include <nlohmann/json_fwd.hpp>
+#include <slikenet/types.h>
+
+struct MessageBase
+{
+  virtual ~MessageBase();
+
+  virtual void WriteBinary(SLNet::BitStream& stream) const = 0;
+  virtual void ReadBinary(SLNet::BitStream& stream) = 0;
+
+  virtual void WriteJson(nlohmann::json& json) const = 0;
+  virtual void ReadJson(const nlohmann::json& json) = 0;
+};

skymp5-server/cpp/messages/MessageSerializerFactory.cpp

@@ -0,0 +1,186 @@
+#include "MessageSerializerFactory.h"
+#include "Messages.h"
+#include "MinPacketId.h"
+#include "MsgType.h"
+#include <nlohmann/json.hpp>
+#include <slikenet/BitStream.h>
+#include <spdlog/spdlog.h>
+
+namespace {
+template <class Message>
+void Serialize(const nlohmann::json& inputJson, SLNet::BitStream& outputStream)
+{
+  Message message;
+  message.ReadJson(
+    inputJson); // may throw. we shouldn't pollute outputStream in this case
+
+  outputStream.Write(static_cast<uint8_t>(Networking::MinPacketId));
+  outputStream.Write(static_cast<uint8_t>(Message::kHeaderByte));
+  message.WriteBinary(outputStream);
+}
+
+template <class Message>
+std::optional<DeserializeResult> Deserialize(
+  const uint8_t* rawMessageJsonOrBinary, size_t length)
+{
+  if (length >= 2 && rawMessageJsonOrBinary[1] == Message::kHeaderByte) {
+    // BitStream requires non-const ref even though it doesn't modify it
+    SLNet::BitStream stream(
+      const_cast<unsigned char*>(rawMessageJsonOrBinary) + 2, length - 2,
+      /*copyData*/ false);
+
+    Message message;
+    message.ReadBinary(stream);
+
+    DeserializeResult result;
+    result.msgType = static_cast<MsgType>(Message::kMsgType);
+    result.message = std::make_unique<Message>(std::move(message));
+    result.format = DeserializeInputFormat::Binary;
+    return result;
+  }
+
+  std::string str(reinterpret_cast<const char*>(rawMessageJsonOrBinary + 1),
+                  length - 1);
+  nlohmann::json json = nlohmann::json::parse(str);
+
+  auto msgTypeIt = json.find("t");
+  if (msgTypeIt == json.end()) {
+    // Messages produced by the server use string "type" instead of integer "t"
+    // We will refactor them out at some point
+    return std::nullopt;
+  }
+  int msgType = msgTypeIt->get<int>();
+
+  if (msgType != Message::kMsgType) {
+    // In case of JSON we keep searching in deserializers array
+    return std::nullopt;
+  }
+
+  Message message;
+  message.ReadJson(json);
+
+  DeserializeResult result;
+  result.msgType = static_cast<MsgType>(msgType);
+  result.message = std::make_unique<Message>(std::move(message));
+  result.format = DeserializeInputFormat::Json;
+  return result;
+}
+} // namespace
+
+#define REGISTER_MESSAGE(Message)                                             \
+  serializeFns[static_cast<size_t>(Message::kMsgType)] = Serialize<Message>;  \
+  deserializeFns[static_cast<size_t>(Message::kHeaderByte)] =                 \
+    Deserialize<Message>;
+
+std::shared_ptr<MessageSerializer>
+MessageSerializerFactory::CreateMessageSerializer()
+{
+  constexpr auto kSerializeFnMax = static_cast<size_t>(MsgType::Max);
+
+  // A hack to support MovementMessage, the first message that was ported to
+  // binary back in 2021 MovementMessage uses 'M' char instead of MsgType to
+  // identify itself in binary encoded packets
+  constexpr auto kDeserializeFnMax = std::max(
+    static_cast<size_t>(MovementMessage::kHeaderByte) + 1, kSerializeFnMax);
+
+  std::vector<MessageSerializer::SerializeFn> serializeFns(kSerializeFnMax);
+  std::vector<MessageSerializer::DeserializeFn> deserializeFns(
+    kDeserializeFnMax);
+
+  REGISTER_MESSAGE(MovementMessage)
+  REGISTER_MESSAGE(UpdateAnimationMessage)
+
+  // make_shared isn't working for private constructors
+  return std::shared_ptr<MessageSerializer>(
+    new MessageSerializer(serializeFns, deserializeFns));
+}
+
+MessageSerializer::MessageSerializer(
+  std::vector<SerializeFn> serializerFns_,
+  std::vector<DeserializeFn> deserializerFns_)
+  : serializerFns(serializerFns_)
+  , deserializerFns(deserializerFns_)
+{
+}
+
+void MessageSerializer::Serialize(const char* jsonContent,
+                                  SLNet::BitStream& outputStream)
+{
+  // TODO(perf): consider using simdjson for parsing OR even use JsValue
+  // directly
+
+  // TODO: logging and write raw instead of throwing exception
+  const auto parsedJson = nlohmann::json::parse(jsonContent);
+
+  // TODO: logging and write raw instead of throwing exception
+  const auto msgType = static_cast<MsgType>(parsedJson.at("t").get<int>());
+
+  auto index = static_cast<size_t>(msgType);
+  if (index >= serializerFns.size()) {
+    // TODO: logging
+    outputStream.Write(static_cast<uint8_t>(Networking::MinPacketId));
+    outputStream.Write(jsonContent, strlen(jsonContent));
+    return;
+  }
+
+  auto serializerFn = serializerFns[index];
+  if (!serializerFn) {
+    // TODO: logging
+    outputStream.Write(static_cast<uint8_t>(Networking::MinPacketId));
+    outputStream.Write(jsonContent, strlen(jsonContent));
+    return;
+  }
+
+  serializerFn(parsedJson, outputStream);
+}
+
+std::optional<DeserializeResult> MessageSerializer::Deserialize(
+  const uint8_t* rawMessageJsonOrBinary, size_t length)
+{
+  if (length < 2) {
+    spdlog::trace("MessageSerializer::Deserialize - Length < 2");
+    return std::nullopt;
+  }
+
+  auto headerByte = rawMessageJsonOrBinary[1];
+  if (headerByte == '{') {
+    spdlog::trace("MessageSerializer::Deserialize - Encountered JSON message");
+    for (auto fn : deserializerFns) {
+      if (fn) {
+        auto result = fn(rawMessageJsonOrBinary, length);
+        if (result) {
+          spdlog::trace("MessageSerializer::Deserialize - Deserialized");
+          return result;
+        }
+      }
+    }
+    spdlog::trace("MessageSerializer::Deserialize - Failed to deserialize, "
+                  "falling back to PacketParser.cpp");
+    return std::nullopt;
+  }
+
+  if (headerByte >= deserializerFns.size()) {
+    spdlog::trace(
+      "MessageSerializer::Deserialize - {} >= deserializerFns.size() ",
+      static_cast<int>(headerByte));
+    return std::nullopt;
+  }
+
+  auto deserializerFn = deserializerFns[headerByte];
+  if (!deserializerFn) {
+    spdlog::trace("MessageSerializer::Deserialize - deserializerFn not found "
+                  "for headerByte {}",
+                  static_cast<int>(headerByte));
+    return std::nullopt;
+  }
+
+  auto result = deserializerFn(rawMessageJsonOrBinary, length);
+  if (result == std::nullopt) {
+    spdlog::trace("MessageSerializer::Deserialize - deserializerFn returned "
+                  "nullopt for headerByte {}",
+                  static_cast<int>(headerByte));
+    return std::nullopt;
+  }
+
+  return result;
+}

skymp5-server/cpp/messages/MessageSerializerFactory.h

@@ -0,0 +1,55 @@
+#pragma once
+
+#include "MessageBase.h"
+#include "MsgType.h"
+#include <cstdint>
+#include <memory>
+#include <nlohmann/json_fwd.hpp>
+#include <optional>
+#include <slikenet/types.h>
+#include <utility>
+#include <vector>
+
+class MessageSerializer;
+
+class MessageSerializerFactory
+{
+public:
+  static std::shared_ptr<MessageSerializer> CreateMessageSerializer();
+};
+
+enum class DeserializeInputFormat
+{
+  Json,
+  Binary
+};
+
+struct DeserializeResult
+{
+  MsgType msgType = MsgType::Invalid;
+  std::unique_ptr<MessageBase> message;
+  DeserializeInputFormat format = DeserializeInputFormat::Json;
+};
+
+class MessageSerializer
+{
+  friend class MessageSerializerFactory;
+
+public:
+  void Serialize(const char* jsonContent, SLNet::BitStream& outputStream);
+
+  std::optional<DeserializeResult> Deserialize(
+    const uint8_t* rawMessageJsonOrBinary, size_t length);
+
+private:
+  typedef void (*SerializeFn)(const nlohmann::json& inputJson,
+                              SLNet::BitStream& outputStream);
+  typedef std::optional<DeserializeResult> (*DeserializeFn)(
+    const uint8_t* rawMessageJsonOrBinary, size_t length);
+
+  MessageSerializer(std::vector<SerializeFn> serializerFns,
+                    std::vector<DeserializeFn> deserializerFns);
+
+  const std::vector<SerializeFn> serializerFns;
+  const std::vector<DeserializeFn> deserializerFns;
+};

skymp5-server/cpp/messages/Messages.h

@@ -0,0 +1,3 @@
+#pragma once
+#include "MovementMessage.h"
+#include "UpdateAnimationMessage.h"

skymp5-server/cpp/messages/MinPacketId.h

@@ -0,0 +1,3 @@
+namespace Networking {
+constexpr unsigned char MinPacketId = 134;
+}