diff --git a/CMakeLists.txt b/CMakeLists.txt index 90ad239..b20c23d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,6 +35,8 @@ endif() target_precompile_headers(lib_spellcheck PRIVATE ${src_loc}/spellcheck/spellcheck_pch.h) nice_target_sources(lib_spellcheck ${src_loc} PRIVATE + highlighting/highlighting.qrc + spellcheck/platform/linux/language_linux.cpp spellcheck/platform/linux/language_linux.h spellcheck/platform/linux/linux_enchant.cpp @@ -57,19 +59,23 @@ PRIVATE spellcheck/third_party/hunspell_controller.h spellcheck/third_party/spellcheck_hunspell.cpp spellcheck/third_party/spellcheck_hunspell.h + spellcheck/spellcheck_types.h + spellcheck/spellcheck_highlight_syntax.cpp + spellcheck/spellcheck_highlight_syntax.h spellcheck/spellcheck_utils.cpp spellcheck/spellcheck_utils.h - spellcheck/spellcheck_types.h + spellcheck/spellcheck_value.cpp + spellcheck/spellcheck_value.h spellcheck/spelling_highlighter.cpp spellcheck/spelling_highlighter.h spellcheck/spelling_highlighter_helper.cpp spellcheck/spelling_highlighter_helper.h - spellcheck/spellcheck_value.cpp - spellcheck/spellcheck_value.h spellcheck/spellcheck_pch.h ) +target_prepare_qrc(lib_spellcheck) + if (system_spellchecker) remove_target_sources(lib_spellcheck ${src_loc} spellcheck/third_party/spellcheck_hunspell.cpp @@ -126,9 +132,11 @@ PRIVATE desktop-app::lib_base desktop-app::lib_rpl desktop-app::lib_crl + desktop-app::lib_prisma desktop-app::external_qt desktop-app::external_ranges desktop-app::external_gsl + desktop-app::external_xxhash ) if (LINUX AND use_enchant) diff --git a/highlighting/grammars.dat b/highlighting/grammars.dat new file mode 100644 index 0000000..b164570 Binary files /dev/null and b/highlighting/grammars.dat differ diff --git a/highlighting/highlighting.qrc b/highlighting/highlighting.qrc new file mode 100644 index 0000000..0346f28 --- /dev/null +++ b/highlighting/highlighting.qrc @@ -0,0 +1,5 @@ + + + grammars.dat + + diff --git a/spellcheck/spellcheck_highlight_syntax.cpp b/spellcheck/spellcheck_highlight_syntax.cpp new file mode 100644 index 0000000..5fcd5bd --- /dev/null +++ b/spellcheck/spellcheck_highlight_syntax.cpp @@ -0,0 +1,288 @@ +// This file is part of Desktop App Toolkit, +// a set of libraries for developing nice desktop applications. +// +// For license and copyright information please follow this link: +// https://github.com/desktop-app/legal/blob/master/LEGAL +// +#include "spellcheck/spellcheck_highlight_syntax.h" + +#include "base/debug_log.h" +#include "base/flat_map.h" +#include "crl/crl_object_on_queue.h" + +#include "SyntaxHighlighter.h" + +#include + +#include +#include +#include + +#include + +void spellchecker_InitHighlightingResource() { +#ifdef Q_OS_MAC // Use resources from the .app bundle on macOS. + + base::RegisterBundledResources(u"lib_spellcheck.rcc"_q); + +#else // Q_OS_MAC + + Q_INIT_RESOURCE(highlighting); + +#endif // Q_OS_MAC +} + +namespace Spellchecker { +namespace { + +base::flat_map Cache; +HighlightProcessId ProcessIdAutoIncrement/* = 0*/; +rpl::event_stream ReadyStream; + +class QueuedHighlighter final { +public: + QueuedHighlighter(); + + struct Request { + uint64 hash = 0; + QString text; + QString language; + }; + void process(Request request); + void notify(HighlightProcessId id); + +private: + using Task = std::variant; + + std::vector _tasks; + + std::unique_ptr _highlighter; + +}; + +[[nodiscard]] crl::object_on_queue &Highlighter() { + static auto result = crl::object_on_queue(); + return result; +} + +QueuedHighlighter::QueuedHighlighter() { + spellchecker_InitHighlightingResource(); +} + +void QueuedHighlighter::process(Request request) { + if (!_highlighter) { + auto file = QFile(":/misc/grammars.dat"); + const auto size = file.size(); + const auto ok1 = file.open(QIODevice::ReadOnly); + auto grammars = std::string(); + grammars.resize(size); + const auto ok2 = (file.read(grammars.data(), size) == size); + Assert(ok1 && ok2); + + _highlighter = std::make_unique(grammars); + } + + const auto text = request.text.toStdString(); + const auto language = request.language.toStdString(); + const auto tokens = _highlighter->tokenize(text, language); + + static const auto colors = base::flat_map{ + { "comment" , 1 }, + { "block-comment", 1 }, + { "prolog" , 1 }, + { "doctype" , 1 }, + { "cdata" , 1 }, + { "punctuation" , 2 }, + { "property" , 3 }, + { "tag" , 3 }, + { "boolean" , 3 }, + { "number" , 3 }, + { "constant" , 3 }, + { "symbol" , 3 }, + { "deleted" , 3 }, + { "selector" , 4 }, + { "attr-name" , 4 }, + { "string" , 4 }, + { "char" , 4 }, + { "builtin" , 4 }, + { "inserted" , 4 }, + { "operator" , 5 }, + { "entity" , 5 }, + { "url" , 5 }, + { "atrule" , 6 }, + { "attr-value" , 6 }, + { "keyword" , 6 }, + { "function" , 6 }, + { "class-name" , 7 }, + }; + + auto offset = 0; + auto entities = EntitiesInText(); + auto rebuilt = QString(); + rebuilt.reserve(request.text.size()); + const auto enumerate = [&]( + const TokenList &list, + const std::string &type, + auto &&self) -> void { + for (const auto &node : list) { + if (node.isSyntax()) { + const auto &syntax = static_cast(node); + self(syntax.children(), syntax.type(), self); + } else { + const auto text = static_cast(node).value(); + const auto utf16 = QString::fromUtf8( + text.data(), + text.size()); + const auto length = utf16.size(); + rebuilt.append(utf16); + if (!type.empty()) { + const auto i = colors.find(type); + if (i != end(colors)) { + entities.push_back(EntityInText( + EntityType::Colorized, + offset, + length, + QChar(ushort(i->second)))); + } + } + offset += length; + } + } + }; + enumerate(tokens, std::string(), enumerate); + const auto hash = request.hash; + if (offset != request.text.size()) { + // Something went wrong. + LOG(("Highlighting Error: for language '%1', text: %2" + ).arg(request.language + ).arg(request.text)); + entities.clear(); + } + crl::on_main([hash, entities = std::move(entities)]() mutable { + Cache.emplace(hash, std::move(entities)); + }); +} + +void QueuedHighlighter::notify(HighlightProcessId id) { + crl::on_main([=] { + ReadyStream.fire_copy(id); + }); +} + +struct CacheResult { + uint64 hash = 0; + const EntitiesInText *list = nullptr; + + explicit operator bool() const { + return list != nullptr; + } +}; +[[nodiscard]] CacheResult FindInCache( + const TextWithEntities &text, + EntitiesInText::const_iterator i) { + const auto view = QStringView(text.text).mid(i->offset(), i->length()); + const auto language = i->data(); + + struct Destroyer { + void operator()(XXH64_state_t *state) { + if (state) { + XXH64_freeState(state); + } + } + }; + static const auto S = std::unique_ptr( + XXH64_createState()); + + const auto state = S.get(); + XXH64_reset(state, 0); + XXH64_update(state, view.data(), view.size() * sizeof(ushort)); + XXH64_update(state, language.data(), language.size() * sizeof(ushort)); + const auto hash = XXH64_digest(state); + + const auto j = Cache.find(hash); + return { hash, (j != Cache.cend()) ? &j->second : nullptr }; +} + +EntitiesInText::iterator Insert( + TextWithEntities &text, + EntitiesInText::iterator i, + const EntitiesInText &entities) { + auto next = i + 1; + if (entities.empty()) { + return next; + } + const auto offset = i->offset(); + if (next != text.entities.cend() + && next->type() == entities.front().type() + && next->offset() == offset + entities.front().offset()) { + return next; + } + const auto length = i->length(); + for (const auto &entity : entities) { + if (entity.offset() + entity.length() >= length) { + break; + } + auto j = text.entities.insert(next, entity); + j->shiftRight(offset); + next = j + 1; + } + return next; +} + +void Schedule( + uint64 hash, + const TextWithEntities &text, + EntitiesInText::const_iterator i) { + Highlighter().with([ + hash, + text = text.text.mid(i->offset(), i->length()), + language = i->data() + ](QueuedHighlighter &instance) mutable { + instance.process({ hash, std::move(text), std::move(language) }); + }); +} + +void Notify(uint64 processId) { + Highlighter().with([processId](QueuedHighlighter &instance) { + instance.notify(processId); + }); +} + +} // namespace + +HighlightProcessId TryHighlightSyntax(TextWithEntities &text) { + auto b = text.entities.begin(); + auto i = b; + auto e = text.entities.end(); + const auto checking = [](const EntityInText &entity) { + return (entity.type() == EntityType::Pre) + && !entity.data().isEmpty(); + }; + auto processId = HighlightProcessId(); + while (true) { + i = std::find_if(i, e, checking); + if (i == e) { + break; + } else if (const auto already = FindInCache(text, i)) { + i = Insert(text, i, *already.list); + b = text.entities.begin(); + e = text.entities.end(); + } else { + Schedule(already.hash, text, i); + if (!processId) { + processId = ++ProcessIdAutoIncrement; + } + ++i; + } + } + if (processId) { + Notify(processId); + } + return processId; +} + +rpl::producer HighlightReady() { + return ReadyStream.events(); +} + +} // namespace Spellchecker diff --git a/spellcheck/spellcheck_highlight_syntax.h b/spellcheck/spellcheck_highlight_syntax.h new file mode 100644 index 0000000..7cc1068 --- /dev/null +++ b/spellcheck/spellcheck_highlight_syntax.h @@ -0,0 +1,19 @@ +// This file is part of Desktop App Toolkit, +// a set of libraries for developing nice desktop applications. +// +// For license and copyright information please follow this link: +// https://github.com/desktop-app/legal/blob/master/LEGAL +// +#pragma once + +#include "ui/text/text_entity.h" + +namespace Spellchecker { + +using HighlightProcessId = uint64; + +// Returning zero means we highlighted everything already. +[[nodiscard]] HighlightProcessId TryHighlightSyntax(TextWithEntities &text); +[[nodiscard]] rpl::producer HighlightReady(); + +} // namespace Spellchecker