Skip to content

Commit

Permalink
Merge branch '1398-explain-upgrade' into 'develop'
Browse files Browse the repository at this point in the history 
[#1398] "Explain for non-indexed fields"

See merge request itv-backend/reindexer!1358
  • Loading branch information
reindexer-bot committed Aug 17, 2023
1 parent 6d4504f commit d4db6eb
Show file tree
Hide file tree
Showing 32 changed files with 1,440 additions and 442 deletions.
194 changes: 194 additions & 0 deletions cpp_src/cmd/reindexer_server/test/test_storage_compatibility.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,194 @@
#!/bin/bash
# Task: https://github.com/restream/reindexer/-/issues/1188
set -e

function KillAndRemoveServer {
local pid=$1
kill $pid
wait $pid
yum remove -y 'reindexer*' > /dev/null
}

function WaitForDB {
# wait until DB is loaded
set +e # disable "exit on error" so the script won't stop when DB's not loaded yet
is_connected=$(reindexer_tool --dsn $ADDRESS --command '\databases list');
while [[ $is_connected != "test" ]]
do
sleep 2
is_connected=$(reindexer_tool --dsn $ADDRESS --command '\databases list');
done
set -e
}

function CompareNamespacesLists {
local ns_list_actual=$1
local ns_list_expected=$2
local pid=$3

diff=$(echo ${ns_list_actual[@]} ${ns_list_expected[@]} | tr ' ' '\n' | sort | uniq -u) # compare in any order
if [ "$diff" == "" ]; then
echo "## PASS: namespaces list not changed"
else
echo "##### FAIL: namespaces list was changed"
echo "expected: $ns_list_expected"
echo "actual: $ns_list_actual"
KillAndRemoveServer $pid;
exit 1
fi
}

function CompareMemstats {
local actual=$1
local expected=$2
local pid=$3
diff=$(echo ${actual[@]} ${expected[@]} | tr ' ' '\n' | sed 's/\(.*\),$/\1/' | sort | uniq -u) # compare in any order
if [ "$diff" == "" ]; then
echo "## PASS: memstats not changed"
else
echo "##### FAIL: memstats was changed"
echo "expected: $expected"
echo "actual: $actual"
KillAndRemoveServer $pid;
exit 1
fi
}


RX_SERVER_CURRENT_VERSION_RPM="$(basename build/reindexer-*server*.rpm)"
VERSION_FROM_RPM=$(echo "$RX_SERVER_CURRENT_VERSION_RPM" | grep -o '.*server-..')
VERSION=$(echo ${VERSION_FROM_RPM: -2:1}) # one-digit version

if [ $VERSION == 3 ]; then
LATEST_RELEASE=$(curl -s http://repo.restream.ru/itv-api-ng/7/x86_64/ | grep -o '>reindexer-server-.*.rpm' | tail -n1 | cut -c 2-)
namespaces_list_expected=$'purchase_options_ext_dict\nchild_account_recommendations\n#config\n#activitystats\nradio_channels\ncollections\n#namespaces\nwp_imports_tasks\nepg_genres\nrecom_media_items_personal\nrecom_epg_archive_default\n#perfstats\nrecom_epg_live_default\nmedia_view_templates\nasset_video_servers\nwp_tasks_schedule\nadmin_roles\n#clientsstats\nrecom_epg_archive_personal\nrecom_media_items_similars\nmenu_items\naccount_recommendations\nkaraoke_items\nmedia_items\nbanners\n#queriesperfstats\nrecom_media_items_default\nrecom_epg_live_personal\nservices\n#memstats\nchannels\nmedia_item_recommendations\nwp_tasks_tasks\nepg'
elif [ $VERSION == 4 ]; then
LATEST_RELEASE=$(curl -s http://repo.restream.ru/itv-api-ng/7/x86_64/ | grep -o '>reindexer-4-server-.*.rpm' | tail -n1 | cut -c 2-)
# replicationstats ns added for v4
namespaces_list_expected=$'purchase_options_ext_dict\nchild_account_recommendations\n#config\n#activitystats\n#replicationstats\nradio_channels\ncollections\n#namespaces\nwp_imports_tasks\nepg_genres\nrecom_media_items_personal\nrecom_epg_archive_default\n#perfstats\nrecom_epg_live_default\nmedia_view_templates\nasset_video_servers\nwp_tasks_schedule\nadmin_roles\n#clientsstats\nrecom_epg_archive_personal\nrecom_media_items_similars\nmenu_items\naccount_recommendations\nkaraoke_items\nmedia_items\nbanners\n#queriesperfstats\nrecom_media_items_default\nrecom_epg_live_personal\nservices\n#memstats\nchannels\nmedia_item_recommendations\nwp_tasks_tasks\nepg'
else
echo "Unknown version"
exit 1
fi

echo "## downloading latest release rpm file: $LATEST_RELEASE"
curl "http://repo.itv.restr.im/itv-api-ng/7/x86_64/$LATEST_RELEASE" --output $LATEST_RELEASE;
echo "## downloading example DB"
curl "https://git.restream.ru/MaksimKravchuk/reindexer_testdata/-/raw/master/big.zip" --output big.zip;
unzip -o big.zip # unzips into mydb_big.rxdump;

ADDRESS="cproto://127.0.0.1:6534/"
DB_NAME="test"

memstats_expected=$'[
{"replication":{"data_hash":24651210926,"data_count":3}},
{"replication":{"data_hash":6252344969,"data_count":1}},
{"replication":{"data_hash":37734732881,"data_count":28}},
{"replication":{"data_hash":0,"data_count":0}},
{"replication":{"data_hash":1024095024522,"data_count":1145}},
{"replication":{"data_hash":8373644068,"data_count":1315}},
{"replication":{"data_hash":0,"data_count":0}},
{"replication":{"data_hash":0,"data_count":0}},
{"replication":{"data_hash":0,"data_count":0}},
{"replication":{"data_hash":0,"data_count":0}},
{"replication":{"data_hash":7404222244,"data_count":97}},
{"replication":{"data_hash":94132837196,"data_count":4}},
{"replication":{"data_hash":1896088071,"data_count":2}},
{"replication":{"data_hash":0,"data_count":0}},
{"replication":{"data_hash":-672103903,"data_count":33538}},
{"replication":{"data_hash":0,"data_count":0}},
{"replication":{"data_hash":6833710705,"data_count":1}},
{"replication":{"data_hash":5858155773472,"data_count":4500}},
{"replication":{"data_hash":-473221280268823592,"data_count":65448}},
{"replication":{"data_hash":0,"data_count":0}},
{"replication":{"data_hash":8288213744,"data_count":3}},
{"replication":{"data_hash":0,"data_count":0}},
{"replication":{"data_hash":0,"data_count":0}},
{"replication":{"data_hash":354171024786967,"data_count":3941}},
{"replication":{"data_hash":-6520334670,"data_count":35886}},
{"replication":{"data_hash":112772074632,"data_count":281}},
{"replication":{"data_hash":-12679568198538,"data_count":1623116}}
]
Returned 27 rows'

echo "##### Forward compatibility test #####"

DB_PATH=$(pwd)"/rx_db"

echo "Database: "$DB_PATH

echo "## installing latest release: $LATEST_RELEASE"
yum install -y $LATEST_RELEASE > /dev/null;
# run RX server with disabled logging
reindexer_server -l warning --httplog=none --rpclog=none --db $DB_PATH &
server_pid=$!
sleep 2;

reindexer_tool --dsn $ADDRESS$DB_NAME -f mydb_big.rxdump --createdb;
sleep 1;

namespaces_1=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list');
echo $namespaces_1;
CompareNamespacesLists "${namespaces_1[@]}" "${namespaces_list_expected[@]}" $server_pid;

memstats_1=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats');
CompareMemstats "${memstats_1[@]}" "${memstats_expected[@]}" $server_pid;

KillAndRemoveServer $server_pid;

echo "## installing current version: $RX_SERVER_CURRENT_VERSION_RPM"
yum install -y build/*.rpm > /dev/null;
reindexer_server -l0 --corelog=none --httplog=none --rpclog=none --db $DB_PATH &
server_pid=$!
sleep 2;

WaitForDB

namespaces_2=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list');
echo $namespaces_2;
CompareNamespacesLists "${namespaces_2[@]}" "${namespaces_1[@]}" $server_pid;

memstats_2=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats');
CompareMemstats "${memstats_2[@]}" "${memstats_1[@]}" $server_pid;

KillAndRemoveServer $server_pid;
rm -rf $DB_PATH;
sleep 1;

echo "##### Backward compatibility test #####"

echo "## installing current version: $RX_SERVER_CURRENT_VERSION_RPM"
yum install -y build/*.rpm > /dev/null;
reindexer_server -l warning --httplog=none --rpclog=none --db $DB_PATH &
server_pid=$!
sleep 2;

reindexer_tool --dsn $ADDRESS$DB_NAME -f mydb_big.rxdump --createdb;
sleep 1;

namespaces_3=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list');
echo $namespaces_3;
CompareNamespacesLists "${namespaces_3[@]}" "${namespaces_list_expected[@]}" $server_pid;

memstats_3=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats');
CompareMemstats "${memstats_3[@]}" "${memstats_expected[@]}" $server_pid;

KillAndRemoveServer $server_pid;

echo "## installing latest release: $LATEST_RELEASE"
yum install -y $LATEST_RELEASE > /dev/null;
reindexer_server -l warning --httplog=none --rpclog=none --db $DB_PATH &
server_pid=$!
sleep 2;

WaitForDB

namespaces_4=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list');
echo $namespaces_4;
CompareNamespacesLists "${namespaces_4[@]}" "${namespaces_3[@]}" $server_pid;

memstats_4=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats');
CompareMemstats "${memstats_4[@]}" "${memstats_3[@]}" $server_pid;

KillAndRemoveServer $server_pid;
rm -rf $DB_PATH;
31 changes: 18 additions & 13 deletions cpp_src/core/expressiontree.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,12 @@ class ExpressionTree;

`ExpressionTree` does not support operator precedence.
You can support it manually as it done in `QueryEntries` and `SelectIteratorContainer`, or by enclosing higher priority operators in brackets as it done in `SortExpression`.
Here do not used traditional way for constructing of trees with inheritance of nodes, allocations of separate nodes and holding of pointers to they.
Here is not used the traditional way for constructing trees with inheritance of nodes, allocations of separate nodes and holding pointers to them.
`ExpressionTree` holds all nodes by value in a vector (`container_`) sequentially in type `Node` based on `variant`.
In order to support lazy copying `Node` can hold a reference to payload of another `Node` by using `ExpressionTree::Ref<T>` type. !Warning! lazy copy should not live over the original one.
In order to support lazy copying `Node` can hold a reference to payload of another `Node` by using `ExpressionTree::Ref<T>` type.

**Warning**: The lazy copy node shall not live longer than the original one.

Subtree is stored in `container_` just behind its head (`SubTree`) which holds occupied space. For details see examples.
This architecture allows to reduce count of allocations and virtual functions calls.

Expand Down Expand Up @@ -187,21 +190,23 @@ It contains operation (value of `OperationType`) and a value of one of the types
- `void Node::Append()` increments size of subexpression if it is head of subexpression, fails otherwise.
- `void Node::Erase(size_t)` reduces size of subexpression if it is head of subexpression, fails otherwise.
- ```c++
template <typename... Args>
void Node::ExecuteAppropriate(const std::function<void(Args&)>&... funcs);
template <typename... Args>
void Node::ExecuteAppropriate(const std::function<void(const Args&)>&... funcs) const;
```
invoke appropriate functor if the `Node` holds value of one of `Args...` types or `Ref<T>` where `T` is one of `Args...` types, no functor will be invoked otherwise.
template <typename... Args>
void Node::ExecuteAppropriate(const std::function<void(Args&)>&... funcs);
template <typename... Args>
void Node::ExecuteAppropriate(const std::function<void(const Args&)>&... funcs) const;
```
invokes appropriate functor if the `Node` holds value of one of `Args...` types or `Ref<T>`, where `T` is one of `Args...` types, no functor will be invoked otherwise.
- ```c++
template <typename R>
R Node::CalculateAppropriate(const std::function<R(const SubTree&)>& f, const std::function<R(const Ts&)>&... funcs) const;
```
invokes appropriate functor depending on type of value is holded by `Node` and provides returned value.
template <typename R>
R Node::CalculateAppropriate(const std::function<R(const SubTree&)>& f, const std::function<R(const Ts&)>&... funcs) const;
```
invokes appropriate functor depending on type of value is held by `Node` and provides returned value.
- `Node Node::MakeLazyCopy()&`
!Warning! the copy should not live over the origin.
* returns copy of origin one if it is head of subexpression or holds value of `Ref<T>` type.
* returns new `Node` that holds `Ref<T>` which references to payload of origin one if it holds `T` (one of `Ts...`).
> **Warning** the copy shall not live longer than the origin.
- `Node Node::MakeDeepCopy() const &`
* returns copy of origin one if it is head of subexpression or holds value of one of `Ts...` types.
* returns new `Node` which holds copy of value that `Ref<T>` references to if origin one holds value of `Ref<T>` type.
Expand Down
35 changes: 35 additions & 0 deletions cpp_src/core/index/indexstore.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,41 @@ SelectKeyResults IndexStore<T>::SelectKey(const VariantArray &keys, CondType con
if (condition == CondAny && !this->opts_.IsArray() && !this->opts_.IsSparse() && !sopts.distinct)
throw Error(errParams, "The 'NOT NULL' condition is suported only by 'sparse' or 'array' indexes");

// TODO: it may be necessary to remove or change this switch after QueryEntry refactoring
switch (condition) {
case CondAny:
if (!this->opts_.IsArray() && !this->opts_.IsSparse() && !sopts.distinct) {
throw Error(errParams, "The 'NOT NULL' condition is suported only by 'sparse' or 'array' indexes");
}
break;
case CondEmpty:
if (!this->opts_.IsArray() && !this->opts_.IsSparse()) {
throw Error(errParams, "The 'is NULL' condition is suported only by 'sparse' or 'array' indexes");
}
break;
case CondAllSet:
case CondSet:
case CondEq:
break;
case CondRange:
case CondDWithin:
if (keys.size() != 2) {
throw Error(errParams, "For condition %s required exactly 2 arguments, but provided %d", CondTypeToStr(condition),
keys.size());
}
break;
case CondLt:
case CondLe:
case CondGt:
case CondGe:
case CondLike:
if (keys.size() != 1) {
throw Error(errParams, "For condition %s required exactly 1 argument, but provided %d", CondTypeToStr(condition),
keys.size());
}
break;
}

res.comparators_.push_back(Comparator(condition, KeyType(), keys, opts_.IsArray(), sopts.distinct, payloadType_, fields_,
idx_data.size() ? idx_data.data() : nullptr, opts_.collateOpts_));
return SelectKeyResults(std::move(res));
Expand Down
Loading

0 comments on commit d4db6eb

Please sign in to comment.