From c99d1c8d8bb255db8014337b6082643e597c12c9 Mon Sep 17 00:00:00 2001 From: "Dirk-Jan C. Binnema" Date: Mon, 16 Sep 2024 19:52:43 +0300 Subject: [PATCH] experiment: try avoiding multi-thread with xapian Try to avoid multi-threaded operation with Xapian. This remove the thread workers during indexing, and avoids the indexing background thread. So, mu4e has to wait once again during indexing. We can improve upon that, but first we need to know if it avoids the problem of issue #2756. --- lib/mu-indexer.cc | 53 +++++++++++++++++++++++++++++++++-------------- lib/mu-server.cc | 33 +++++++++++++++++++++-------- meson.build | 10 ++++++++- meson_options.txt | 52 ++++++++++++++++++++++++++++++++-------------- 4 files changed, 106 insertions(+), 42 deletions(-) diff --git a/lib/mu-indexer.cc b/lib/mu-indexer.cc index ec4ef013f..02b1be605 100644 --- a/lib/mu-indexer.cc +++ b/lib/mu-indexer.cc @@ -105,6 +105,7 @@ struct Indexer::Private { bool handler(const std::string& fullpath, struct stat* statbuf, Scanner::HandleType htype); void maybe_start_worker(); + void item_worker(); void scan_worker(); @@ -135,6 +136,8 @@ struct Indexer::Private { Type type; }; + void handle_item(WorkItem&& item); + AsyncQueue todos_; Progress progress_{}; @@ -193,7 +196,11 @@ Indexer::Private::handler(const std::string& fullpath, struct stat* statbuf, return true; } case Scanner::HandleType::LeaveDir: { +#ifdef XAPIAN_SINGLE_THREADED + handle_item({fullpath, WorkItem::Type::Dir}); +#else todos_.push({fullpath, WorkItem::Type::Dir}); +#endif /*XAPIAN_SINGLE_THREADED*/ return true; } @@ -210,9 +217,13 @@ Indexer::Private::handler(const std::string& fullpath, struct stat* statbuf, if (statbuf->st_ctime <= dirstamp_ && store_.contains_message(fullpath)) return false; +#ifdef XAPIAN_SINGLE_THREADED + handle_item({fullpath, WorkItem::Type::File}); +#else // push the remaining messages to our "todo" queue for // (re)parsing and adding/updating to the database. todos_.push({fullpath, WorkItem::Type::File}); +#endif return true; } default: @@ -260,6 +271,30 @@ Indexer::Private::add_message(const std::string& path) return true; } + +void +Indexer::Private::handle_item(WorkItem&& item) +{ + try { + switch (item.type) { + case WorkItem::Type::File: { + if (G_LIKELY(add_message(item.full_path))) + ++progress_.updated; + } break; + case WorkItem::Type::Dir: + store_.set_dirstamp(item.full_path, ::time(NULL)); + break; + default: + g_warn_if_reached(); + break; + } + } catch (const Mu::Error& er) { + mu_warning("error adding message @ {}: {}", item.full_path, er.what()); + } +} + + + void Indexer::Private::item_worker() { @@ -270,22 +305,8 @@ Indexer::Private::item_worker() while (state_ == IndexState::Scanning) { if (!todos_.pop(item, 250ms)) continue; - try { - switch (item.type) { - case WorkItem::Type::File: { - if (G_LIKELY(add_message(item.full_path))) - ++progress_.updated; - } break; - case WorkItem::Type::Dir: - store_.set_dirstamp(item.full_path, ::time(NULL)); - break; - default: - g_warn_if_reached(); - break; - } - } catch (const Mu::Error& er) { - mu_warning("error adding message @ {}: {}", item.full_path, er.what()); - } + + handle_item(std::move(item)); maybe_start_worker(); std::this_thread::yield(); diff --git a/lib/mu-server.cc b/lib/mu-server.cc index 9d21cab25..cd8535735 100644 --- a/lib/mu-server.cc +++ b/lib/mu-server.cc @@ -149,6 +149,7 @@ struct Server::Private { Store& store() { return store_; } const Store& store() const { return store_; } Indexer& indexer() { return store().indexer(); } + void do_index(const Indexer::Config& conf); //CommandMap& command_map() const { return command_map_; } // @@ -761,6 +762,20 @@ get_stats(const Indexer::Progress& stats, const std::string& state) return sexp; } +void +Server::Private::do_index(const Indexer::Config& conf) +{ + StopWatch sw{"indexing"}; + indexer().start(conf); + while (indexer().is_running()) { + std::this_thread::sleep_for(std::chrono::milliseconds(2000)); + output_sexp(get_stats(indexer().progress(), "running"), + Server::OutputFlags::Flush); + } + output_sexp(get_stats(indexer().progress(), "complete"), + Server::OutputFlags::Flush); +} + void Server::Private::index_handler(const Command& cmd) { @@ -770,22 +785,22 @@ Server::Private::index_handler(const Command& cmd) // ignore .noupdate with an empty store. conf.ignore_noupdate = store().empty(); +#ifdef XAPIAN_SINGLE_THREADED + // nothing to do + if (indexer().is_running()) + return; + do_index(conf); +#else indexer().stop(); if (index_thread_.joinable()) index_thread_.join(); // start a background track. index_thread_ = std::thread([this, conf = std::move(conf)] { - StopWatch sw{"indexing"}; - indexer().start(conf); - while (indexer().is_running()) { - std::this_thread::sleep_for(std::chrono::milliseconds(2000)); - output_sexp(get_stats(indexer().progress(), "running"), - Server::OutputFlags::Flush); - } - output_sexp(get_stats(indexer().progress(), "complete"), - Server::OutputFlags::Flush); + do_index(conf); }); +#endif /*XAPIAN_SINGLE_THREADED */ + } void diff --git a/meson.build b/meson.build index c7b07fe67..7aba9890e 100644 --- a/meson.build +++ b/meson.build @@ -133,7 +133,13 @@ add_project_arguments(['-DHAVE_CONFIG_H'], language: 'cpp') config_h_dep=declare_dependency( include_directories: include_directories(['.'])) - +# +# single-threaded Xapian access? +# +if get_option('xapian-single-threaded') + config_h_data.set('XAPIAN_SINGLE_THREADED', true) + message('use Xapian only in a single thread') +endif # # d_type, d_ino are not available universally, so let's check # (we use them for optimizations in mu-scanner @@ -322,6 +328,8 @@ if gmime_dep.version() == '3.2.13' warning('See: https://github.com/jstedfast/gmime/issues/133') endif + + # Local Variables: # indent-tabs-mode: nil # End: diff --git a/meson_options.txt b/meson_options.txt index 93bc7dbca..d79a3778d 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -14,36 +14,56 @@ ## along with this program; if not, write to the Free Software Foundation, ## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -option('tests', + +option('cld2', type : 'feature', value: 'auto', - description: 'build unit tests') + description: 'Add support for language-detection through cld2') + +# +# emacs +# + +option('emacs', + type: 'string', + value: 'emacs', + description: 'name/path of the emacs executable (for byte-compilation)') + +option('lispdir', + type: 'string', + description: 'path under which to install emacs-lisp files') + + +# +# guile +# option('guile', type : 'feature', value: 'auto', description: 'build the guile scripting support (requires guile-3.x)') -option('cld2', - type : 'feature', - value: 'auto', - description: 'Compact Language Detector2') - # by default, this uses guile_dep.get_variable(pkgconfig: 'extensiondir') option('guile-extension-dir', type: 'string', description: 'custom install path for the guile extension module') + +# +# misc +# + +option('tests', + type : 'feature', + value: 'auto', + description: 'build unit tests') + +option('xapian-single-threaded', + type : 'boolean', + value: true, + description: 'only use Xapian from a single thread') + option('readline', type: 'feature', value: 'auto', description: 'enable readline support for the mu4e repl') - -option('emacs', - type: 'string', - value: 'emacs', - description: 'name/path of the emacs executable') - -option('lispdir', - type: 'string', - description: 'path under which to install emacs-lisp files')