Add syntax highlighting with libprisma.

CMakeLists.txt

@@ -35,6 +35,8 @@ endif()
 target_precompile_headers(lib_spellcheck PRIVATE ${src_loc}/spellcheck/spellcheck_pch.h)
 nice_target_sources(lib_spellcheck ${src_loc}
 PRIVATE
+    highlighting/highlighting.qrc
+
     spellcheck/platform/linux/language_linux.cpp
     spellcheck/platform/linux/language_linux.h
     spellcheck/platform/linux/linux_enchant.cpp
@@ -57,19 +59,23 @@ PRIVATE
     spellcheck/third_party/hunspell_controller.h
     spellcheck/third_party/spellcheck_hunspell.cpp
     spellcheck/third_party/spellcheck_hunspell.h
+    spellcheck/spellcheck_types.h
+    spellcheck/spellcheck_highlight_syntax.cpp
+    spellcheck/spellcheck_highlight_syntax.h
     spellcheck/spellcheck_utils.cpp
     spellcheck/spellcheck_utils.h
-    spellcheck/spellcheck_types.h
+    spellcheck/spellcheck_value.cpp
+    spellcheck/spellcheck_value.h
     spellcheck/spelling_highlighter.cpp
     spellcheck/spelling_highlighter.h
     spellcheck/spelling_highlighter_helper.cpp
     spellcheck/spelling_highlighter_helper.h
-    spellcheck/spellcheck_value.cpp
-    spellcheck/spellcheck_value.h
 
     spellcheck/spellcheck_pch.h
 )
 
+target_prepare_qrc(lib_spellcheck)
+
 if (system_spellchecker)
     remove_target_sources(lib_spellcheck ${src_loc}
         spellcheck/third_party/spellcheck_hunspell.cpp
@@ -126,9 +132,11 @@ PRIVATE
     desktop-app::lib_base
     desktop-app::lib_rpl
     desktop-app::lib_crl
+    desktop-app::lib_prisma
     desktop-app::external_qt
     desktop-app::external_ranges
     desktop-app::external_gsl
+    desktop-app::external_xxhash
 )
 
 if (LINUX AND use_enchant)

highlighting/highlighting.qrc

@@ -0,0 +1,5 @@
+<RCC>
+	<qresource prefix="/misc">
+		<file alias="grammars.dat">grammars.dat</file>
+	</qresource>
+</RCC>

spellcheck/spellcheck_highlight_syntax.cpp

@@ -0,0 +1,288 @@
+// This file is part of Desktop App Toolkit,
+// a set of libraries for developing nice desktop applications.
+//
+// For license and copyright information please follow this link:
+// https://github.com/desktop-app/legal/blob/master/LEGAL
+//
+#include "spellcheck/spellcheck_highlight_syntax.h"
+
+#include "base/debug_log.h"
+#include "base/flat_map.h"
+#include "crl/crl_object_on_queue.h"
+
+#include "SyntaxHighlighter.h"
+
+#include <QtCore/QFile>
+
+#include <xxhash.h>
+#include <variant>
+#include <string>
+
+#include <windows.h>
+
+void spellchecker_InitHighlightingResource() {
+#ifdef Q_OS_MAC // Use resources from the .app bundle on macOS.
+
+	base::RegisterBundledResources(u"lib_spellcheck.rcc"_q);
+
+#else // Q_OS_MAC
+
+	Q_INIT_RESOURCE(highlighting);
+
+#endif // Q_OS_MAC
+}
+
+namespace Spellchecker {
+namespace {
+
+base::flat_map<XXH64_hash_t, EntitiesInText> Cache;
+HighlightProcessId ProcessIdAutoIncrement/* = 0*/;
+rpl::event_stream<HighlightProcessId> ReadyStream;
+
+class QueuedHighlighter final {
+public:
+	QueuedHighlighter();
+
+	struct Request {
+		uint64 hash = 0;
+		QString text;
+		QString language;
+	};
+	void process(Request request);
+	void notify(HighlightProcessId id);
+
+private:
+	using Task = std::variant<Request, HighlightProcessId>;
+
+	std::vector<Task> _tasks;
+
+	std::unique_ptr<SyntaxHighlighter> _highlighter;
+
+};
+
+[[nodiscard]] crl::object_on_queue<QueuedHighlighter> &Highlighter() {
+	static auto result = crl::object_on_queue<QueuedHighlighter>();
+	return result;
+}
+
+QueuedHighlighter::QueuedHighlighter() {
+	spellchecker_InitHighlightingResource();
+}
+
+void QueuedHighlighter::process(Request request) {
+	if (!_highlighter) {
+		auto file = QFile(":/misc/grammars.dat");
+		const auto size = file.size();
+		const auto ok1 = file.open(QIODevice::ReadOnly);
+		auto grammars = std::string();
+		grammars.resize(size);
+		const auto ok2 = (file.read(grammars.data(), size) == size);
+		Assert(ok1 && ok2);
+
+		_highlighter = std::make_unique<SyntaxHighlighter>(grammars);
+	}
+
+	const auto text = request.text.toStdString();
+	const auto language = request.language.toStdString();
+	const auto tokens = _highlighter->tokenize(text, language);
+
+	static const auto colors = base::flat_map<std::string, int>{
+		{ "comment"      , 1 },
+		{ "block-comment", 1 },
+		{ "prolog"       , 1 },
+		{ "doctype"      , 1 },
+		{ "cdata"        , 1 },
+		{ "punctuation"  , 2 },
+		{ "property"     , 3 },
+		{ "tag"          , 3 },
+		{ "boolean"      , 3 },
+		{ "number"       , 3 },
+		{ "constant"     , 3 },
+		{ "symbol"       , 3 },
+		{ "deleted"      , 3 },
+		{ "selector"     , 4 },
+		{ "attr-name"    , 4 },
+		{ "string"       , 4 },
+		{ "char"         , 4 },
+		{ "builtin"      , 4 },
+		{ "inserted"     , 4 },
+		{ "operator"     , 5 },
+		{ "entity"       , 5 },
+		{ "url"          , 5 },
+		{ "atrule"       , 6 },
+		{ "attr-value"   , 6 },
+		{ "keyword"      , 6 },
+		{ "function"     , 6 },
+		{ "class-name"   , 7 },
+	};
+
+	auto offset = 0;
+	auto entities = EntitiesInText();
+	auto rebuilt = QString();
+	rebuilt.reserve(request.text.size());
+	const auto enumerate = [&](
+			const TokenList &list,
+			const std::string &type,
+			auto &&self) -> void {
+		for (const auto &node : list) {
+			if (node.isSyntax()) {
+				const auto &syntax = static_cast<const Syntax&>(node);
+				self(syntax.children(), syntax.type(), self);
+			} else {
+				const auto text = static_cast<const Text&>(node).value();
+				const auto utf16 = QString::fromUtf8(
+					text.data(),
+					text.size());
+				const auto length = utf16.size();
+				rebuilt.append(utf16);
+				if (!type.empty()) {
+					const auto i = colors.find(type);
+					if (i != end(colors)) {
+						entities.push_back(EntityInText(
+							EntityType::Colorized,
+							offset,
+							length,
+							QChar(ushort(i->second))));
+					}
+				}
+				offset += length;
+			}
+		}
+	};
+	enumerate(tokens, std::string(), enumerate);
+	const auto hash = request.hash;
+	if (offset != request.text.size()) {
+		// Something went wrong.
+		LOG(("Highlighting Error: for language '%1', text: %2"
+			).arg(request.language
+			).arg(request.text));
+		entities.clear();
+	}
+	crl::on_main([hash, entities = std::move(entities)]() mutable {
+		Cache.emplace(hash, std::move(entities));
+	});
+}
+
+void QueuedHighlighter::notify(HighlightProcessId id) {
+	crl::on_main([=] {
+		ReadyStream.fire_copy(id);
+	});
+}
+
+struct CacheResult {
+	uint64 hash = 0;
+	const EntitiesInText *list = nullptr;
+
+	explicit operator bool() const {
+		return list != nullptr;
+	}
+};
+[[nodiscard]] CacheResult FindInCache(
+		const TextWithEntities &text,
+		EntitiesInText::const_iterator i) {
+	const auto view = QStringView(text.text).mid(i->offset(), i->length());
+	const auto language = i->data();
+
+	struct Destroyer {
+		void operator()(XXH64_state_t *state) {
+			if (state) {
+				XXH64_freeState(state);
+			}
+		}
+	};
+	static const auto S = std::unique_ptr<XXH64_state_t, Destroyer>(
+		XXH64_createState());
+
+	const auto state = S.get();
+	XXH64_reset(state, 0);
+	XXH64_update(state, view.data(), view.size() * sizeof(ushort));
+	XXH64_update(state, language.data(), language.size() * sizeof(ushort));
+	const auto hash = XXH64_digest(state);
+
+	const auto j = Cache.find(hash);
+	return { hash, (j != Cache.cend()) ? &j->second : nullptr };
+}
+
+EntitiesInText::iterator Insert(
+		TextWithEntities &text,
+		EntitiesInText::iterator i,
+		const EntitiesInText &entities) {
+	auto next = i + 1;
+	if (entities.empty()) {
+		return next;
+	}
+	const auto offset = i->offset();
+	if (next != text.entities.cend()
+		&& next->type() == entities.front().type()
+		&& next->offset() == offset + entities.front().offset()) {
+		return next;
+	}
+	const auto length = i->length();
+	for (const auto &entity : entities) {
+		if (entity.offset() + entity.length() >= length) {
+			break;
+		}
+		auto j = text.entities.insert(next, entity);
+		j->shiftRight(offset);
+		next = j + 1;
+	}
+	return next;
+}
+
+void Schedule(
+		uint64 hash,
+		const TextWithEntities &text,
+		EntitiesInText::const_iterator i) {
+	Highlighter().with([
+		hash,
+		text = text.text.mid(i->offset(), i->length()),
+		language = i->data()
+	](QueuedHighlighter &instance) mutable {
+		instance.process({ hash, std::move(text), std::move(language) });
+	});
+}
+
+void Notify(uint64 processId) {
+	Highlighter().with([processId](QueuedHighlighter &instance) {
+		instance.notify(processId);
+	});
+}
+
+} // namespace
+
+HighlightProcessId TryHighlightSyntax(TextWithEntities &text) {
+	auto b = text.entities.begin();
+	auto i = b;
+	auto e = text.entities.end();
+	const auto checking = [](const EntityInText &entity) {
+		return (entity.type() == EntityType::Pre)
+			&& !entity.data().isEmpty();
+	};
+	auto processId = HighlightProcessId();
+	while (true) {
+		i = std::find_if(i, e, checking);
+		if (i == e) {
+			break;
+		} else if (const auto already = FindInCache(text, i)) {
+			i = Insert(text, i, *already.list);
+			b = text.entities.begin();
+			e = text.entities.end();
+		} else {
+			Schedule(already.hash, text, i);
+			if (!processId) {
+				processId = ++ProcessIdAutoIncrement;
+			}
+			++i;
+		}
+	}
+	if (processId) {
+		Notify(processId);
+	}
+	return processId;
+}
+
+rpl::producer<HighlightProcessId> HighlightReady() {
+	return ReadyStream.events();
+}
+
+} // namespace Spellchecker

spellcheck/spellcheck_highlight_syntax.h

@@ -0,0 +1,19 @@
+// This file is part of Desktop App Toolkit,
+// a set of libraries for developing nice desktop applications.
+//
+// For license and copyright information please follow this link:
+// https://github.com/desktop-app/legal/blob/master/LEGAL
+//
+#pragma once
+
+#include "ui/text/text_entity.h"
+
+namespace Spellchecker {
+
+using HighlightProcessId = uint64;
+
+// Returning zero means we highlighted everything already.
+[[nodiscard]] HighlightProcessId TryHighlightSyntax(TextWithEntities &text);
+[[nodiscard]] rpl::producer<HighlightProcessId> HighlightReady();
+
+} // namespace Spellchecker