Update to version v3.17.0

bindings/consts.go

@@ -2,7 +2,7 @@ package bindings
 
 const CInt32Max = int(^uint32(0) >> 1)
 
-const ReindexerVersion = "v3.16.0"
+const ReindexerVersion = "v3.17.0"
 
 // public go consts from type_consts.h and reindexer_ctypes.h
 const (

changelog.md

@@ -1,3 +1,26 @@
+# Version 3.17.0 (06.07.2023)
+## Core
+- [fea] Optimized namespaces' locks for queries to the system namespaces, containing explicit list of names (for example, `SELECT * FROM #memstats WHERE "name" IN ('ns1', 'nsx', 'ns19')` now requires shared locks for the listed namespaces only)
+- [fea] Added update queries for the arrays of objects (i.e. queries like `UPDATE ns SET object_array[11].field = 999`)
+- [fea] Added more fulltext index options for the base rankings tuning. Check `FtBaseRanking` struct in the [config doc](fulltext.md#configuration)
+- [fix] Fixed update queries for the sparse indexed arrays
+- [fix] Fixed selection plans and cost calculation for `NOT`-conditions in queries
+- [fix] Disabled composite indexes over non-indexed fields (except fulltext indexes) and sparse composite indexes. Previously those indexes could be created, but was not actually implemented, so queries with them could lead to unexpected errors
+- [fix] Fixed merging of the conditions by the same index in queries' optimizer (previously it could sometimes cause SIGABORT in case of the empty resulting set)
+- [fix] Disabled LIMIT for internal merged queries (it was not implemented and did not work properly on previous versions)
+- [fix] Fixed cache behavior for fulltext queires with JOINs and `enable_preselect_before_ft: true`
+
+## Reindexer server
+- [fea] Optimized perfstats config access
+- [fea] Added support for CSV results for select queries via HTTP (experimental, probably this API will be modified later on)
+
+## Go connector
+- [fea] Added `SortStPointDistance` and `SortStFieldDistance` (simplified aliases for the long constructions with `ST_Distance`)
+- [ref] Added `reindexer.Point` type instead of `[2]float64`
+
+## Build
+- [fea] Added support and deploy for RedOS 7.3.2
+
 # Version 3.16.0 (09.06.2023)
 ## Core
 - [fea] Added UUID field type for indexes

cpp_src/CMakeLists.txt

@@ -35,7 +35,7 @@ else()
   option (LINK_RESOURCES "Link web resources as binary data" ON)
 endif()
 
-set (REINDEXER_VERSION_DEFAULT "3.16.0")
+set (REINDEXER_VERSION_DEFAULT "3.17.0")
 
 if(NOT CMAKE_BUILD_TYPE)
   set(CMAKE_BUILD_TYPE "RelWithDebInfo")
@@ -529,7 +529,7 @@ set(CPACK_GENERATOR "TGZ")
 
 if (WIN32) 
   set (CPACK_GENERATOR "NSIS")
-elseif (LINUX_ISSUE MATCHES "Fedora" OR LINUX_ISSUE MATCHES "CentOS" OR LINUX_ISSUE MATCHES "Mandriva")
+elseif (LINUX_ISSUE MATCHES "Fedora" OR LINUX_ISSUE MATCHES "CentOS" OR LINUX_ISSUE MATCHES "Mandriva" OR LINUX_ISSUE MATCHES "RED OS")
   set(CPACK_GENERATOR "RPM")
   set(CPACK_PACKAGE_RELOCATABLE OFF)
 elseif (LINUX_ISSUE MATCHES "Ubuntu" OR LINUX_ISSUE MATCHES "Debian" OR LINUX_ISSUE MATCHES "Mint")
@@ -691,7 +691,7 @@ if (NOT WIN32)
     "core/queryresults/itemref.h" "core/namespace/stringsholder.h" "core/keyvalue/key_string.h" "core/key_value_type.h" "core/keyvalue/uuid.h"
     "core/expressiontree.h" "core/lsn.h" "core/cjson/tagspath.h" "core/cjson/ctag.h"
 	"estl/cow.h" "estl/overloaded.h" "estl/one_of.h" "estl/h_vector.h" "estl/mutex.h" "estl/intrusive_ptr.h" "estl/trivial_reverse_iterator.h"
-	"estl/span.h" "estl/chunk.h" "estl/fast_hash_traits.h" "estl/debug_macros.h"
+	"estl/span.h" "estl/chunk.h" "estl/fast_hash_traits.h" "estl/debug_macros.h" "estl/defines.h"
     "client/reindexer.h" "client/item.h" "client/reindexerconfig.h" "client/queryresults.h" "client/resultserializer.h"
     "client/internalrdxcontext.h" "client/transaction.h"
     "client/cororeindexer.h" "client/coroqueryresults.h" "client/corotransaction.h"

cpp_src/cmd/reindexer_tool/commandsexecutor.cc

@@ -8,6 +8,7 @@
 #include "tableviewscroller.h"
 #include "tools/fsops.h"
 #include "tools/jsontools.h"
+#include "tools/stringstools.h"
 
 namespace reindexer_tool {
 
@@ -558,7 +559,7 @@ Error CommandsExecutor<DBInterface>::commandSelect(const std::string& command) {
 						reindexer::h_vector<int, 1> maxW;
 						maxW.reserve(agg.fields.size());
 						for (const auto& field : agg.fields) {
-							maxW.push_back(field.length());
+							maxW.emplace_back(field.length());
 						}
 						for (auto& row : agg.facets) {
 							assertrx(row.values.size() == agg.fields.size());
@@ -707,7 +708,7 @@ Error CommandsExecutor<DBInterface>::commandDump(const std::string& command) {
 			auto ns = parser.NextToken();
 			auto nsDef = std::find_if(allNsDefs.begin(), allNsDefs.end(), [&ns](const NamespaceDef& nsDef) { return ns == nsDef.name; });
 			if (nsDef != allNsDefs.end()) {
-				doNsDefs.push_back(std::move(*nsDef));
+				doNsDefs.emplace_back(std::move(*nsDef));
 				allNsDefs.erase(nsDef);
 			} else {
 				std::cerr << "Namespace '" << ns << "' - skipped. (not found in storage)" << std::endl;
@@ -724,7 +725,7 @@ Error CommandsExecutor<DBInterface>::commandDump(const std::string& command) {
 
 	for (auto& nsDef : doNsDefs) {
 		// skip system namespaces, except #config
-		if (nsDef.name.length() > 0 && nsDef.name[0] == '#' && nsDef.name != "#config") continue;
+		if (reindexer::isSystemNamespaceNameFast(nsDef.name) && nsDef.name != "#config") continue;
 
 		wrser << "-- Dumping namespace '" << nsDef.name << "' ..." << '\n';
 

cpp_src/core/activity_context.cc

@@ -1,6 +1,7 @@
 #include "activity_context.h"
 #include <iomanip>
 #include <sstream>
+#include "activity_context.h"
 #include "cjson/jsonbuilder.h"
 #include "tools/stringstools.h"
 
@@ -9,46 +10,55 @@ namespace reindexer {
 using namespace std::string_view_literals;
 
 void ActivityContainer::Register(const RdxActivityContext* context) {
-	std::unique_lock<std::mutex> lck(mtx_);
+	std::unique_lock lck(mtx_);
 	const auto res = cont_.insert(context);
+	lck.unlock();
+
 	assertrx(res.second);
 	(void)res;
 }
 
 void ActivityContainer::Unregister(const RdxActivityContext* context) {
-	std::unique_lock<std::mutex> lck(mtx_);
+	std::unique_lock lck(mtx_);
 	const auto count = cont_.erase(context);
+	lck.unlock();
+
 	assertrx(count == 1u);
 	(void)count;
 }
 
 void ActivityContainer::Reregister(const RdxActivityContext* oldCtx, const RdxActivityContext* newCtx) {
 	if (oldCtx == newCtx) return;
-	std::unique_lock<std::mutex> lck(mtx_);
+
+	std::unique_lock lck(mtx_);
 	const auto eraseCount = cont_.erase(oldCtx);
-	assertrx(eraseCount == 1u);
 	const auto insertRes = cont_.insert(newCtx);
+	lck.unlock();
+
+	assertrx(eraseCount == 1u);
 	assertrx(insertRes.second);
 	(void)eraseCount;
 	(void)insertRes;
 }
 
 std::vector<Activity> ActivityContainer::List() {
 	std::vector<Activity> ret;
-	std::unique_lock<std::mutex> lck(mtx_);
-	ret.reserve(cont_.size());
-	for (const RdxActivityContext* ctx : cont_) ret.push_back(*ctx);
+	{
+		std::lock_guard lck(mtx_);
+		ret.reserve(cont_.size());
+		for (const RdxActivityContext* ctx : cont_) ret.emplace_back(*ctx);
+	}
 	return ret;
 }
 
 std::optional<std::string> ActivityContainer::QueryForIpConnection(int id) {
-	std::unique_lock<std::mutex> lck(mtx_);
+	std::lock_guard lck(mtx_);
 
 	for (const RdxActivityContext* ctx : cont_) {
 		if (ctx->CheckConnectionId(id)) {
 			std::string ret;
 			deepCopy(ret, ctx->Query());
-			return ret;
+			return std::optional{std::move(ret)};
 		}
 	}
 
@@ -126,16 +136,11 @@ RdxActivityContext::operator Activity() const {
 	return ret;
 }
 
-unsigned RdxActivityContext::serializeState(MutexMark mark) { return Activity::WaitLock | (static_cast<unsigned>(mark) << kStateShift); }
-unsigned RdxActivityContext::serializeState(Activity::State state) { return static_cast<unsigned>(state); }
-
 std::pair<Activity::State, std::string_view> RdxActivityContext::deserializeState(unsigned state) {
 	const Activity::State decodedState = static_cast<Activity::State>(state & kStateMask);
-	if (decodedState == Activity::WaitLock) {
-		return {decodedState, DescribeMutexMark(static_cast<MutexMark>(state >> kStateShift))};
-	} else {
-		return {decodedState, ""};
-	}
+	return decodedState == Activity::WaitLock
+			   ? std::make_pair(decodedState, DescribeMutexMark(static_cast<MutexMark>(state >> kStateShift)))
+			   : std::make_pair(decodedState, "");
 }
 
 unsigned RdxActivityContext::nextId() noexcept {

cpp_src/core/activity_context.h

@@ -56,23 +56,23 @@ class RdxActivityContext {
 
 	class Ward {
 	public:
-		Ward(RdxActivityContext* cont, Activity::State state) : context_(cont) {
+		Ward(RdxActivityContext* cont, Activity::State state) noexcept : context_(cont) {
 			if (context_) {
 				prevState_ = context_->state_.exchange(serializeState(state), std::memory_order_relaxed);
 #ifndef NDEBUG
 				context_->refCount_.fetch_add(1u, std::memory_order_relaxed);
 #endif
 			}
 		}
-		Ward(RdxActivityContext* cont, MutexMark mutexMark) : context_(cont) {
+		Ward(RdxActivityContext* cont, MutexMark mutexMark) noexcept : context_(cont) {
 			if (context_) {
 				prevState_ = context_->state_.exchange(serializeState(mutexMark), std::memory_order_relaxed);
 #ifndef NDEBUG
 				context_->refCount_.fetch_add(1u, std::memory_order_relaxed);
 #endif
 			}
 		}
-		Ward(Ward&& other) : context_(other.context_), prevState_(other.prevState_) { other.context_ = nullptr; }
+		Ward(Ward&& other) noexcept : context_(other.context_), prevState_(other.prevState_) { other.context_ = nullptr; }
 		~Ward() {
 			if (context_) {
 				context_->state_.store(prevState_, std::memory_order_relaxed);
@@ -107,15 +107,15 @@ class RdxActivityContext {
 
 	/// returning value of these functions should be assined to a local variable which will be destroyed after the waiting work complete
 	/// lifetime of the local variable should not exceed of the activityContext's
-	Ward BeforeLock(MutexMark mutexMark) { return Ward(this, mutexMark); }
-	Ward BeforeIndexWork() { return Ward(this, Activity::IndexesLookup); }
-	Ward BeforeSelectLoop() { return Ward(this, Activity::SelectLoop); }
+	Ward BeforeLock(MutexMark mutexMark) noexcept { return Ward(this, mutexMark); }
+	Ward BeforeIndexWork() noexcept { return Ward(this, Activity::IndexesLookup); }
+	Ward BeforeSelectLoop() noexcept { return Ward(this, Activity::SelectLoop); }
 
 	bool CheckConnectionId(int connectionId) const noexcept { return data_.connectionId == connectionId; }
 
 private:
-	static unsigned serializeState(MutexMark);
-	static unsigned serializeState(Activity::State);
+	static unsigned serializeState(MutexMark mark) noexcept { return Activity::WaitLock | (static_cast<unsigned>(mark) << kStateShift); }
+	static unsigned serializeState(Activity::State state) noexcept { return static_cast<unsigned>(state); }
 	static std::pair<Activity::State, std::string_view> deserializeState(unsigned state);
 	static unsigned nextId() noexcept;
 

cpp_src/core/cbinding/reindexer_c.cc

@@ -169,7 +169,7 @@ reindexer_error reindexer_modify_item_packed_tx(uintptr_t rx, uintptr_t tr, rein
 	unsigned preceptsCount = ser.GetVarUint();
 	std::vector<std::string> precepts;
 	while (preceptsCount--) {
-		precepts.push_back(std::string(ser.GetVString()));
+		precepts.emplace_back(ser.GetVString());
 	}
 	Error err = err_not_init;
 	auto item = trw->tr_.NewItem();

cpp_src/core/cjson/baseencoder.cc

@@ -4,6 +4,7 @@
 #include "cjsonbuilder.h"
 #include "cjsontools.h"
 #include "core/keyvalue/p_string.h"
+#include "csvbuilder.h"
 #include "jsonbuilder.h"
 #include "msgpackbuilder.h"
 #include "protobufbuilder.h"
@@ -126,6 +127,14 @@ bool BaseEncoder<Builder>::encode(ConstPayload* pl, Serializer& rdser, Builder&
 	const TagType tagType = tag.Type();
 
 	if (tagType == TAG_END) {
+		if constexpr (kWithFieldExtractor) {
+			if (visible && filter_ && indexedTagsPath_.size() && indexedTagsPath_.back().IsWithIndex()) {
+				const auto field = builder.TargetField();
+				if (field >= 0 && !builder.IsHavingOffset() && filter_->match(indexedTagsPath_)) {
+					builder.OnScopeEnd(fieldsoutcnt_[field]);
+				}
+			}
+		}
 		return false;
 	}
 	const int tagName = tag.Name();
@@ -179,7 +188,7 @@ bool BaseEncoder<Builder>::encode(ConstPayload* pl, Serializer& rdser, Builder&
 			case TAG_END:
 			case TAG_OBJECT:
 			case TAG_UUID:
-				if (visible) builder.Put(tagName, pl->Get(tagField, (*cnt)));
+				if (visible) builder.Put(tagName, pl->Get(tagField, (*cnt)), *cnt);
 				++(*cnt);
 				break;
 		}
@@ -232,7 +241,7 @@ bool BaseEncoder<Builder>::encode(ConstPayload* pl, Serializer& rdser, Builder&
 			case TAG_UUID:
 				if (visible) {
 					Variant value = rdser.GetRawVariant(KeyValueType{tagType});
-					builder.Put(tagName, std::move(value));
+					builder.Put(tagName, std::move(value), 0);
 				} else {
 					rdser.SkipRawVariant(KeyValueType{tagType});
 				}
@@ -340,5 +349,6 @@ template class BaseEncoder<CJsonBuilder>;
 template class BaseEncoder<MsgPackBuilder>;
 template class BaseEncoder<ProtobufBuilder>;
 template class BaseEncoder<FieldsExtractor>;
+template class BaseEncoder<CsvBuilder>;
 
 }  // namespace reindexer

cpp_src/core/cjson/baseencoder.h

@@ -14,6 +14,7 @@ class TagsMatcher;
 class JsonBuilder;
 class MsgPackBuilder;
 class ProtobufBuilder;
+class CsvBuilder;
 
 class IEncoderDatasourceWithJoins {
 public:
@@ -47,6 +48,7 @@ class BaseEncoder {
 protected:
 	using IndexedTagsPathInternalT = IndexedTagsPathImpl<16>;
 	constexpr static bool kWithTagsPathTracking = std::is_same_v<ProtobufBuilder, Builder>;
+	constexpr static bool kWithFieldExtractor = std::is_same_v<FieldsExtractor, Builder>;
 
 	struct DummyTagsPathScope {
 		DummyTagsPathScope(TagsPath & /*tagsPath*/, int16_t /*tagName*/) noexcept {}
@@ -74,5 +76,6 @@ using JsonEncoder = BaseEncoder<JsonBuilder>;
 using CJsonEncoder = BaseEncoder<CJsonBuilder>;
 using MsgPackEncoder = BaseEncoder<MsgPackBuilder>;
 using ProtobufEncoder = BaseEncoder<ProtobufBuilder>;
+using CsvEncoder = BaseEncoder<CsvBuilder>;
 
 }  // namespace reindexer