[#1488] [#1317] - clang-format; force-build-all

Restream · Aug 14, 2023 · 53cd8c6 · 53cd8c6
1 parent 6d4504f
commit 53cd8c6
Show file tree

Hide file tree

Showing 62 changed files with 1,636 additions and 1,192 deletions.
diff --git a/cpp_src/cmd/reindexer_server/test/test_storage_compatibility.sh b/cpp_src/cmd/reindexer_server/test/test_storage_compatibility.sh
@@ -0,0 +1,194 @@
+#!/bin/bash
+# Task: https://github.com/restream/reindexer/-/issues/1188
+set -e
+
+function KillAndRemoveServer {
+	local pid=$1
+	kill $pid
+	wait $pid
+	yum remove -y 'reindexer*' > /dev/null
+}
+
+function WaitForDB {
+	# wait until DB is loaded
+	set +e # disable "exit on error" so the script won't stop when DB's not loaded yet
+	is_connected=$(reindexer_tool --dsn $ADDRESS --command '\databases list');
+	while [[ $is_connected != "test" ]]
+		do
+			sleep 2
+			is_connected=$(reindexer_tool --dsn $ADDRESS --command '\databases list');
+		done
+	set -e
+}
+
+function CompareNamespacesLists {
+	local ns_list_actual=$1
+	local ns_list_expected=$2
+	local pid=$3
+
+	diff=$(echo ${ns_list_actual[@]} ${ns_list_expected[@]} | tr ' ' '\n' | sort | uniq -u) # compare in any order
+	if [ "$diff" == "" ]; then
+	    echo "## PASS: namespaces list not changed"
+	else
+	    echo "##### FAIL: namespaces list was changed"
+	    echo "expected: $ns_list_expected"
+	    echo "actual: $ns_list_actual"
+		KillAndRemoveServer $pid;
+	    exit 1
+    fi
+}
+
+function CompareMemstats {
+	local actual=$1
+	local expected=$2
+	local pid=$3
+	diff=$(echo ${actual[@]} ${expected[@]} | tr ' ' '\n' | sed 's/\(.*\),$/\1/' | sort | uniq -u) # compare in any order
+	if [ "$diff" == "" ]; then
+	    echo "## PASS: memstats not changed"
+	else
+	    echo "##### FAIL: memstats was changed"
+	    echo "expected: $expected"
+	    echo "actual: $actual"
+		KillAndRemoveServer $pid;
+	    exit 1
+    fi
+}
+
+
+RX_SERVER_CURRENT_VERSION_RPM="$(basename build/reindexer-*server*.rpm)"
+VERSION_FROM_RPM=$(echo "$RX_SERVER_CURRENT_VERSION_RPM" | grep -o '.*server-..')
+VERSION=$(echo ${VERSION_FROM_RPM: -2:1})   # one-digit version
+
+if [ $VERSION == 3 ]; then
+  LATEST_RELEASE=$(curl -s http://repo.restream.ru/itv-api-ng/7/x86_64/ | grep -o '>reindexer-server-.*.rpm' | tail -n1 | cut -c 2-)
+  namespaces_list_expected=$'purchase_options_ext_dict\nchild_account_recommendations\n#config\n#activitystats\nradio_channels\ncollections\n#namespaces\nwp_imports_tasks\nepg_genres\nrecom_media_items_personal\nrecom_epg_archive_default\n#perfstats\nrecom_epg_live_default\nmedia_view_templates\nasset_video_servers\nwp_tasks_schedule\nadmin_roles\n#clientsstats\nrecom_epg_archive_personal\nrecom_media_items_similars\nmenu_items\naccount_recommendations\nkaraoke_items\nmedia_items\nbanners\n#queriesperfstats\nrecom_media_items_default\nrecom_epg_live_personal\nservices\n#memstats\nchannels\nmedia_item_recommendations\nwp_tasks_tasks\nepg'
+elif [ $VERSION == 4 ]; then
+  LATEST_RELEASE=$(curl -s http://repo.restream.ru/itv-api-ng/7/x86_64/ | grep -o '>reindexer-4-server-.*.rpm' | tail -n1 | cut -c 2-)
+  # replicationstats ns added for v4
+  namespaces_list_expected=$'purchase_options_ext_dict\nchild_account_recommendations\n#config\n#activitystats\n#replicationstats\nradio_channels\ncollections\n#namespaces\nwp_imports_tasks\nepg_genres\nrecom_media_items_personal\nrecom_epg_archive_default\n#perfstats\nrecom_epg_live_default\nmedia_view_templates\nasset_video_servers\nwp_tasks_schedule\nadmin_roles\n#clientsstats\nrecom_epg_archive_personal\nrecom_media_items_similars\nmenu_items\naccount_recommendations\nkaraoke_items\nmedia_items\nbanners\n#queriesperfstats\nrecom_media_items_default\nrecom_epg_live_personal\nservices\n#memstats\nchannels\nmedia_item_recommendations\nwp_tasks_tasks\nepg'
+else
+  echo "Unknown version"
+  exit 1
+fi
+
+echo "## downloading latest release rpm file: $LATEST_RELEASE"
+curl "http://repo.itv.restr.im/itv-api-ng/7/x86_64/$LATEST_RELEASE" --output $LATEST_RELEASE;
+echo "## downloading example DB"
+curl "https://git.restream.ru/MaksimKravchuk/reindexer_testdata/-/raw/master/big.zip" --output big.zip;
+unzip -o big.zip # unzips into mydb_big.rxdump;
+
+ADDRESS="cproto://127.0.0.1:6534/"
+DB_NAME="test"
+
+memstats_expected=$'[
+{"replication":{"data_hash":24651210926,"data_count":3}},
+{"replication":{"data_hash":6252344969,"data_count":1}},
+{"replication":{"data_hash":37734732881,"data_count":28}},
+{"replication":{"data_hash":0,"data_count":0}},
+{"replication":{"data_hash":1024095024522,"data_count":1145}},
+{"replication":{"data_hash":8373644068,"data_count":1315}},
+{"replication":{"data_hash":0,"data_count":0}},
+{"replication":{"data_hash":0,"data_count":0}},
+{"replication":{"data_hash":0,"data_count":0}},
+{"replication":{"data_hash":0,"data_count":0}},
+{"replication":{"data_hash":7404222244,"data_count":97}},
+{"replication":{"data_hash":94132837196,"data_count":4}},
+{"replication":{"data_hash":1896088071,"data_count":2}},
+{"replication":{"data_hash":0,"data_count":0}},
+{"replication":{"data_hash":-672103903,"data_count":33538}},
+{"replication":{"data_hash":0,"data_count":0}},
+{"replication":{"data_hash":6833710705,"data_count":1}},
+{"replication":{"data_hash":5858155773472,"data_count":4500}},
+{"replication":{"data_hash":-473221280268823592,"data_count":65448}},
+{"replication":{"data_hash":0,"data_count":0}},
+{"replication":{"data_hash":8288213744,"data_count":3}},
+{"replication":{"data_hash":0,"data_count":0}},
+{"replication":{"data_hash":0,"data_count":0}},
+{"replication":{"data_hash":354171024786967,"data_count":3941}},
+{"replication":{"data_hash":-6520334670,"data_count":35886}},
+{"replication":{"data_hash":112772074632,"data_count":281}},
+{"replication":{"data_hash":-12679568198538,"data_count":1623116}}
+]
+Returned 27 rows'
+
+echo "##### Forward compatibility test #####"
+
+DB_PATH=$(pwd)"/rx_db"
+
+echo "Database: "$DB_PATH
+
+echo "## installing latest release: $LATEST_RELEASE"
+yum install -y $LATEST_RELEASE > /dev/null;
+# run RX server with disabled logging
+reindexer_server -l warning --httplog=none --rpclog=none --db $DB_PATH &
+server_pid=$!
+sleep 2;
+
+reindexer_tool --dsn $ADDRESS$DB_NAME -f mydb_big.rxdump --createdb;
+sleep 1;
+
+namespaces_1=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list');
+echo $namespaces_1;
+CompareNamespacesLists "${namespaces_1[@]}" "${namespaces_list_expected[@]}" $server_pid;
+
+memstats_1=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats');
+CompareMemstats "${memstats_1[@]}" "${memstats_expected[@]}" $server_pid;
+
+KillAndRemoveServer $server_pid;
+
+echo "## installing current version: $RX_SERVER_CURRENT_VERSION_RPM"
+yum install -y build/*.rpm > /dev/null;
+reindexer_server -l0  --corelog=none --httplog=none --rpclog=none --db $DB_PATH &
+server_pid=$!
+sleep 2;
+
+WaitForDB
+
+namespaces_2=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list');
+echo $namespaces_2;
+CompareNamespacesLists "${namespaces_2[@]}" "${namespaces_1[@]}" $server_pid;
+
+memstats_2=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats');
+CompareMemstats "${memstats_2[@]}" "${memstats_1[@]}" $server_pid;
+
+KillAndRemoveServer $server_pid;
+rm -rf $DB_PATH;
+sleep 1;
+
+echo "##### Backward compatibility test #####"
+
+echo "## installing current version: $RX_SERVER_CURRENT_VERSION_RPM"
+yum install -y build/*.rpm > /dev/null;
+reindexer_server -l warning --httplog=none --rpclog=none --db $DB_PATH &
+server_pid=$!
+sleep 2;
+
+reindexer_tool --dsn $ADDRESS$DB_NAME -f mydb_big.rxdump --createdb;
+sleep 1;
+
+namespaces_3=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list');
+echo $namespaces_3;
+CompareNamespacesLists "${namespaces_3[@]}" "${namespaces_list_expected[@]}" $server_pid;
+
+memstats_3=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats');
+CompareMemstats "${memstats_3[@]}" "${memstats_expected[@]}" $server_pid;
+
+KillAndRemoveServer $server_pid;
+
+echo "## installing latest release: $LATEST_RELEASE"
+yum install -y $LATEST_RELEASE > /dev/null;
+reindexer_server -l warning --httplog=none --rpclog=none --db $DB_PATH &
+server_pid=$!
+sleep 2;
+
+WaitForDB
+
+namespaces_4=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list');
+echo $namespaces_4;
+CompareNamespacesLists "${namespaces_4[@]}" "${namespaces_3[@]}" $server_pid;
+
+memstats_4=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats');
+CompareMemstats "${memstats_4[@]}" "${memstats_3[@]}" $server_pid;
+
+KillAndRemoveServer $server_pid;
+rm -rf $DB_PATH;
diff --git a/cpp_src/cmd/reindexer_tool/commandsexecutor.cc b/cpp_src/cmd/reindexer_tool/commandsexecutor.cc
@@ -917,8 +917,8 @@ Error CommandsExecutor<DBInterface>::commandBench(const std::string& command) {
 	LineParser parser(command);
 	parser.NextToken();
 
-	int benchTime = reindexer::stoi(parser.NextToken());
-	if (benchTime == 0) benchTime = kBenchDefaultTime;
+	const std::string_view benchTimeToken = parser.NextToken();
+	const int benchTime = benchTimeToken.empty() ? kBenchDefaultTime : reindexer::stoi(benchTimeToken);
 
 	db().DropNamespace(kBenchNamespace);
 

diff --git a/cpp_src/core/cjson/fieldextractor.h b/cpp_src/core/cjson/fieldextractor.h
@@ -16,7 +16,7 @@ class FieldsExtractor {
 	};
 
 	FieldsExtractor() = default;
-	FieldsExtractor(VariantArray *va, KeyValueType expectedType, int expectedPathDepth, FieldsSet *filter = nullptr,
+	FieldsExtractor(VariantArray *va, KeyValueType expectedType, int expectedPathDepth, const FieldsSet *filter,
 					FieldParams *params = nullptr) noexcept
 		: values_(va), expectedType_(expectedType), expectedPathDepth_(expectedPathDepth), filter_(filter), params_(params) {}
 	FieldsExtractor(FieldsExtractor &&other) = default;
@@ -165,7 +165,7 @@ class FieldsExtractor {
 	VariantArray *values_ = nullptr;
 	KeyValueType expectedType_{KeyValueType::Undefined{}};
 	int expectedPathDepth_ = 0;
-	FieldsSet *filter_;
+	const FieldsSet *filter_;
 	FieldParams *params_;
 };
 

diff --git a/cpp_src/core/cjson/tagsmatcherimpl.h b/cpp_src/core/cjson/tagsmatcherimpl.h
@@ -75,20 +75,26 @@ class TagsMatcherImpl {
 					if (content == "*"sv) {
 						node.MarkAllItems(true);
 					} else {
-						int index = stoi(content);
-						if (index == 0 && content != "0"sv && ev) {
-							VariantArray values = ev(content);
-							if (values.size() != 1) {
-								throw Error(errParams, "Index expression_ has wrong syntax: '%s'", content);
+						int index;
+						try {
+							index = stoi(content);
+						} catch (const Error &) {
+							if (ev) {
+								VariantArray values = ev(content);
+								if (values.size() != 1) {
+									throw Error(errParams, "Index expression_ has wrong syntax: '%s'", content);
+								}
+								values.front().Type().EvaluateOneOf(
+									[](OneOf<KeyValueType::Double, KeyValueType::Int, KeyValueType::Int64>) noexcept {},
+									[&](OneOf<KeyValueType::Bool, KeyValueType::String, KeyValueType::Tuple, KeyValueType::Composite,
+											  KeyValueType::Null, KeyValueType::Undefined, KeyValueType::Uuid>) {
+										throw Error(errParams, "Wrong type of index: '%s'", content);
+									});
+								node.SetExpression(content);
+								index = values.front().As<int>();
+							} else {
+								throw;
 							}
-							values.front().Type().EvaluateOneOf(
-								[](OneOf<KeyValueType::Double, KeyValueType::Int, KeyValueType::Int64>) noexcept {},
-								[&](OneOf<KeyValueType::Bool, KeyValueType::String, KeyValueType::Tuple, KeyValueType::Composite,
-										  KeyValueType::Null, KeyValueType::Undefined, KeyValueType::Uuid>) {
-									throw Error(errParams, "Wrong type of index: '%s'", content);
-								});
-							node.SetExpression(content);
-							index = values.front().As<int>();
 						}
 						if (index < 0) {
 							throw Error(errLogic, "Array index value cannot be negative");

diff --git a/cpp_src/core/comparator.cc b/cpp_src/core/comparator.cc
@@ -82,12 +82,6 @@ void Comparator::Bind(const PayloadType &type, int field) {
 	}
 }
 
-void Comparator::BindEqualPosition(int field, const VariantArray &val, CondType cond) { cmpEqualPosition.BindField(field, val, cond); }
-
-void Comparator::BindEqualPosition(const TagsPath &tagsPath, const VariantArray &val, CondType cond) {
-	cmpEqualPosition.BindField(tagsPath, val, cond);
-}
-
 bool Comparator::isNumericComparison(const VariantArray &values) const {
 	if (valuesType_.Is<KeyValueType::Undefined>() || values.empty()) return false;
 	const KeyValueType keyType{values.front().Type()};

diff --git a/cpp_src/core/comparator.h b/cpp_src/core/comparator.h
@@ -15,8 +15,10 @@ class Comparator : public ComparatorVars {
 	bool Compare(const PayloadValue &lhs, int rowId);
 	void ExcludeDistinct(const PayloadValue &, int rowId);
 	void Bind(const PayloadType &type, int field);
-	void BindEqualPosition(int field, const VariantArray &val, CondType cond);
-	void BindEqualPosition(const TagsPath &tagsPath, const VariantArray &val, CondType cond);
+	template <typename F>
+	void BindEqualPosition(F &&field, const VariantArray &val, CondType cond) {
+		cmpEqualPosition.BindField(std::forward<F>(field), val, cond);
+	}
 	void ClearDistinct() {
 		cmpInt.ClearDistinct();
 		cmpBool.ClearDistinct();
@@ -116,7 +118,7 @@ class Comparator : public ComparatorVars {
 	ComparatorImpl<PayloadValue> cmpComposite;
 	ComparatorImpl<Point> cmpGeom;
 	ComparatorImpl<Uuid> cmpUuid;
-	CompositeArrayComparator cmpEqualPosition;
+	EqualPositionComparator cmpEqualPosition;
 	KeyValueType valuesType_{KeyValueType::Undefined{}};
 };
 

diff --git a/cpp_src/core/compositearraycomparator.cc b/cpp_src/core/compositearraycomparator.cc
@@ -2,37 +2,29 @@
 #include <limits.h>
 namespace reindexer {
 
-CompositeArrayComparator::CompositeArrayComparator() {}
+void EqualPositionComparator::BindField(int field, const VariantArray &values, CondType cond) { bindField(field, values, cond); }
 
-void CompositeArrayComparator::BindField(int field, const VariantArray &values, CondType condType) {
+void EqualPositionComparator::BindField(const FieldsPath &fieldPath, const VariantArray &values, CondType cond) {
+	bindField(fieldPath, values, cond);
+}
+
+template <typename F>
+void EqualPositionComparator::bindField(F field, const VariantArray &values, CondType cond) {
 	fields_.push_back(field);
 	Context &ctx = ctx_.emplace_back();
 
-	ctx.cond = condType;
-	ctx.cmpBool.SetValues(condType, values);
-	ctx.cmpInt.SetValues(condType, values);
-	ctx.cmpInt64.SetValues(condType, values);
-	ctx.cmpString.SetValues(condType, values, CollateOpts());
-	ctx.cmpDouble.SetValues(condType, values);
-	ctx.cmpUuid.SetValues(condType, values);
+	ctx.cond = cond;
+	ctx.cmpBool.SetValues(cond, values);
+	ctx.cmpInt.SetValues(cond, values);
+	ctx.cmpInt64.SetValues(cond, values);
+	ctx.cmpString.SetValues(cond, values, CollateOpts());
+	ctx.cmpDouble.SetValues(cond, values);
+	ctx.cmpUuid.SetValues(cond, values);
 
 	assertrx(ctx_.size() == fields_.size());
 }
 
-void CompositeArrayComparator::BindField(const TagsPath &tagsPath, const VariantArray &values, CondType condType) {
-	fields_.push_back(tagsPath);
-	Context &ctx = ctx_.emplace_back();
-
-	ctx.cond = condType;
-	ctx.cmpBool.SetValues(condType, values);
-	ctx.cmpInt.SetValues(condType, values);
-	ctx.cmpInt64.SetValues(condType, values);
-	ctx.cmpString.SetValues(condType, values, CollateOpts());
-	ctx.cmpDouble.SetValues(condType, values);
-	ctx.cmpUuid.SetValues(condType, values);
-}
-
-bool CompositeArrayComparator::Compare(const PayloadValue &pv, const ComparatorVars &vars) {
+bool EqualPositionComparator::Compare(const PayloadValue &pv, const ComparatorVars &vars) {
 	ConstPayload pl(vars.payloadType_, pv);
 	size_t len = INT_MAX;
 
@@ -64,7 +56,7 @@ bool CompositeArrayComparator::Compare(const PayloadValue &pv, const ComparatorV
 	return false;
 }
 
-bool CompositeArrayComparator::compareField(size_t field, const Variant &v, const ComparatorVars &vars) {
+bool EqualPositionComparator::compareField(size_t field, const Variant &v, const ComparatorVars &vars) {
 	return v.Type().EvaluateOneOf(
 		[&](KeyValueType::Bool) { return ctx_[field].cmpBool.Compare(ctx_[field].cond, static_cast<bool>(v)); },
 		[&](KeyValueType::Int) { return ctx_[field].cmpInt.Compare(ctx_[field].cond, static_cast<int>(v)); },

diff --git a/cpp_src/core/compositearraycomparator.h b/cpp_src/core/compositearraycomparator.h
@@ -5,17 +5,19 @@
 
 namespace reindexer {
 
-class CompositeArrayComparator {
+class EqualPositionComparator {
 public:
-	CompositeArrayComparator();
+	EqualPositionComparator() noexcept = default;
 
-	void BindField(int field, const VariantArray &values, CondType condType);
-	void BindField(const TagsPath &tagsPath, const VariantArray &values, CondType condType);
-	bool Compare(const PayloadValue &pv, const ComparatorVars &vars);
+	void BindField(int field, const VariantArray &, CondType);
+	void BindField(const FieldsPath &, const VariantArray &, CondType);
+	bool Compare(const PayloadValue &, const ComparatorVars &);
 	bool IsBinded() { return !ctx_.empty(); }
 
 private:
-	bool compareField(size_t field, const Variant &v, const ComparatorVars &vars);
+	bool compareField(size_t field, const Variant &, const ComparatorVars &);
+	template <typename F>
+	void bindField(F field, const VariantArray &, CondType);
 
 	struct Context {
 		CondType cond;