diff --git a/src/sumo.erl b/src/sumo.erl index 2210c98..5120e48 100644 --- a/src/sumo.erl +++ b/src/sumo.erl @@ -47,7 +47,7 @@ integer | string | binary | text | float | date | datetime. -type field_name() :: atom(). -type field_value() :: term(). --type operator() :: '<' | '>' | '==' | '<=' | '>=' | '/=' | 'like'. +-type operator() :: '<' | '>' | '==' | '=<' | '>=' | '/=' | 'like'. -type doc() :: #{field_name() => field_value()}. -type conditions() :: condition() | [condition()]. -type condition() :: {'and', [condition()]} diff --git a/src/sumo_store_riak.erl b/src/sumo_store_riak.erl index ea7e19f..5217182 100644 --- a/src/sumo_store_riak.erl +++ b/src/sumo_store_riak.erl @@ -124,16 +124,15 @@ persist(Doc, -spec delete_by( sumo:schema_name(), sumo:conditions(), state() ) -> sumo_store:result(sumo_store:affected_rows(), state()). -delete_by(DocName, - Conditions, +delete_by(DocName, Conditions, #state{conn = Conn, bucket = Bucket, index = Index, - del_opts = Opts} = State) -> + del_opts = Opts} = State) when is_list(Conditions) -> IdField = sumo_internal:id_field_name(DocName), case lists:keyfind(IdField, 1, Conditions) of {_K, Key} -> - case delete_map(Conn, Bucket, iolist_to_binary(Key), Opts) of + case delete_map(Conn, Bucket, to_bin(Key), Opts) of ok -> {ok, 1, State}; {error, Error} -> @@ -148,6 +147,19 @@ delete_by(DocName, {error, Error} -> {error, Error, State} end + end; +delete_by(DocName, Conditions, + #state{conn = Conn, + bucket = Bucket, + index = Index, + del_opts = Opts} = State) -> + Query = build_query(Conditions), + case search_docs_by(DocName, Conn, Index, Query, 0, 0) of + {ok, {Total, Res}} -> + delete_docs(Conn, Bucket, Res, Opts), + {ok, Total, State}; + {error, Error} -> + {error, Error, State} end. -spec delete_all( @@ -200,18 +212,15 @@ find_by(DocName, Conditions, State) -> non_neg_integer(), state() ) -> sumo_store:result([sumo_internal:doc()], state()). -find_by(DocName, - Conditions, - Limit, - Offset, +find_by(DocName, Conditions, Limit, Offset, #state{conn = Conn, bucket = Bucket, index = Index, - get_opts = Opts} = State) -> + get_opts = Opts} = State) when is_list(Conditions) -> IdField = sumo_internal:id_field_name(DocName), case lists:keyfind(IdField, 1, Conditions) of {_K, Key} -> - case fetch_map(Conn, Bucket, iolist_to_binary(Key), Opts) of + case fetch_map(Conn, Bucket, to_bin(Key), Opts) of {ok, RMap} -> Val = rmap_to_doc(DocName, RMap), {ok, [Val], State}; @@ -226,6 +235,13 @@ find_by(DocName, {ok, {_, Res}} -> {ok, Res, State}; {error, Error} -> {error, Error, State} end + end; +find_by(DocName, Conditions, Limit, Offset, + #state{conn = Conn, index = Index} = State) -> + Query = build_query(Conditions), + case search_docs_by(DocName, Conn, Index, Query, Limit, Offset) of + {ok, {_, Res}} -> {ok, Res, State}; + {error, Error} -> {error, Error, State} end. -spec find_by( @@ -314,12 +330,8 @@ search(Conn, Index, Query, Limit, Offset) -> riakc_pb_socket:search(Conn, Index, Query, [{start, Offset}, {rows, Limit}]). -spec build_query(sumo:conditions()) -> binary(). -build_query([]) -> - <<"*:*">>; -build_query(PL) when is_list(PL) -> - build_query1(PL, <<"">>); -build_query(_) -> - <<"*:*">>. +build_query(Conditions) -> + build_query1(Conditions, fun escape/1, fun quote/1). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Private API. @@ -415,7 +427,7 @@ search_docs_by(DocName, Conn, Index, Query, Limit, Offset) -> case search(Conn, Index, Query, Limit, Offset) of {ok, {search_results, Results, _, Total}} -> F = fun({_, KV}, Acc) -> [kv_to_doc(DocName, KV) | Acc] end, - NewRes = lists:foldl(F, [], Results), + NewRes = lists:reverse(lists:foldl(F, [], Results)), {ok, {Total, NewRes}}; {error, Error} -> {error, Error} @@ -456,39 +468,98 @@ to_atom(Data) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% @private -build_query1([], Acc) -> - Acc; -build_query1([{_, [{_, _} | _T0]} = KV | T], <<"">>) -> - build_query1(T, - <<(<<"(">>)/binary, (build_query2(KV))/binary, (<<")">>)/binary>>); -build_query1([{_, [{_, _} | _T0]} = KV | T], Acc) -> - build_query1(T, - <>)/binary, - (build_query2(KV))/binary, (<<")">>)/binary>>); -build_query1([{K, V} | T], <<"">>) -> - build_query1(T, <<(query_eq(K, V))/binary>>); -build_query1([{K, V} | T], Acc) -> - build_query1(T, - <>)/binary, (query_eq(K, V))/binary>>). +build_query1([], _EscapeFun, _QuoteFun) -> + <<"*:*">>; +build_query1(Exprs, EscapeFun, QuoteFun) when is_list(Exprs) -> + Clauses = [build_query1(Expr, EscapeFun, QuoteFun) || Expr <- Exprs], + binary:list_to_bin(["(", interpose(" AND ", Clauses), ")"]); +build_query1({'and', Exprs}, EscapeFun, QuoteFun) -> + build_query1(Exprs, EscapeFun, QuoteFun); +build_query1({'or', Exprs}, EscapeFun, QuoteFun) -> + Clauses = [build_query1(Expr, EscapeFun, QuoteFun) || Expr <- Exprs], + binary:list_to_bin(["(", interpose(" OR ", Clauses), ")"]); +build_query1({'not', Expr}, EscapeFun, QuoteFun) -> + binary:list_to_bin(["(NOT ", build_query1(Expr, EscapeFun, QuoteFun), ")"]); +build_query1({Name, '<', Value}, EscapeFun, _QuoteFun) -> + NewVal = binary:list_to_bin(["{* TO ", EscapeFun(Value), "}"]), + query_eq(Name, NewVal); +build_query1({Name, '=<', Value}, EscapeFun, _QuoteFun) -> + NewVal = binary:list_to_bin(["[* TO ", EscapeFun(Value), "]"]), + query_eq(Name, NewVal); +build_query1({Name, '>', Value}, EscapeFun, _QuoteFun) -> + NewVal = binary:list_to_bin(["{", EscapeFun(Value), " TO *}"]), + query_eq(Name, NewVal); +build_query1({Name, '>=', Value}, EscapeFun, _QuoteFun) -> + NewVal = binary:list_to_bin(["[", EscapeFun(Value), " TO *]"]), + query_eq(Name, NewVal); +build_query1({Name, '==', Value}, EscapeFun, QuoteFun) -> + build_query1({Name, Value}, EscapeFun, QuoteFun); +build_query1({Name, '/=', Value}, EscapeFun, QuoteFun) -> + build_query1({negative_field(Name), Value}, EscapeFun, QuoteFun); +build_query1({Name, 'like', Value}, _EscapeFun, _QuoteFun) -> + NewVal = like_to_wildcard_search(Value), + Bypass = fun(X) -> X end, + build_query1({Name, NewVal}, Bypass, Bypass); +build_query1({Name, 'null'}, _EscapeFun, _QuoteFun) -> + %% null: (Field:undefined OR (NOT Field:[* TO *])) + Val = {'or', [{Name, <<"undefined">>}, {'not', {Name, <<"[* TO *]">>}}]}, + Bypass = fun(X) -> X end, + build_query1(Val, Bypass, Bypass); +build_query1({Name, 'not_null'}, _EscapeFun, _QuoteFun) -> + %% not_null: (Field:[* TO *] AND -Field:undefined) + Val = {'and', [{Name, <<"[* TO *]">>}, {Name, '/=', <<"undefined">>}]}, + Bypass = fun(X) -> X end, + build_query1(Val, Bypass, Bypass); +build_query1({Name, Value}, EscapeFun, QuoteFun) -> + query_eq(Name, QuoteFun(EscapeFun(Value))). %% @private -build_query2({K, [{_, _} | _T] = V}) -> - F = fun({K_, V_}, Acc) -> - Eq = <<(atom_to_binary(K_, utf8))/binary, - (<<"_register:">>)/binary, - (to_bin(V_))/binary>>, - case Acc of - <<"">> -> - Eq; - _ -> - <>)/binary, (to_bin(K))/binary, - (<<" ">>)/binary, Eq/binary>> - end - end, - lists:foldl(F, <<"">>, V). +query_eq(K, V) -> + binary:list_to_bin([build_key(K), V]). %% @private -query_eq(K, V) -> - <<(atom_to_binary(K, utf8))/binary, - (<<"_register:">>)/binary, - (to_bin(V))/binary>>. +build_key(K) -> + build_key(binary:split(to_bin(K), <<".">>, [global]), <<"">>). + +%% @private +build_key([K], <<"">>) -> + binary:list_to_bin([K, "_register:"]); +build_key([K], Acc) -> + binary:list_to_bin([Acc, ".", K, "_register:"]); +build_key([K | T], <<"">>) -> + build_key(T, binary:list_to_bin([K, "_map"])); +build_key([K | T], Acc) -> + build_key(T, binary:list_to_bin([Acc, ".", K, "_map"])). + +%% @private +interpose(Sep, List) -> + interpose(Sep, List, []). + +%% @private +interpose(_Sep, [], Result) -> + lists:reverse(Result); +interpose(Sep, [Item | []], Result) -> + interpose(Sep, [], [Item | Result]); +interpose(Sep, [Item | Rest], Result) -> + interpose(Sep, Rest, [Sep, Item | Result]). + +%% @private +negative_field(Name) -> + binary:list_to_bin([<<"-">>, to_bin(Name)]). + +%% @private +quote(Value) -> + [$\", re:replace(to_bin(Value), "[\\\"\\\\]", "\\\\&", [global]), $\"]. + +%% @private +escape(Value) -> + Escape = "[\\+\\-\\&\\|\\!\\(\\)\\{\\}\\[\\]\\^\\\"\\~\\*\\?\\:\\\\]", + re:replace(to_bin(Value), Escape, "\\\\&", [global, {return, binary}]). + +%% @private +whitespace(Value) -> + re:replace(Value, "[\\\s\\\\]", "\\\\&", [global, {return, binary}]). + +%% @private +like_to_wildcard_search(Like) -> + whitespace(binary:replace(to_bin(Like), <<"%">>, <<"*">>, [global])). diff --git a/test/conditional_logic_SUITE.erl b/test/conditional_logic_SUITE.erl index 3a448e3..bcfc1c6 100644 --- a/test/conditional_logic_SUITE.erl +++ b/test/conditional_logic_SUITE.erl @@ -47,7 +47,9 @@ init_per_suite(Config) -> sumo:persist(Module, Module:new("John", "Doe", 30)), sumo:persist(Module, Module:new("Jane Jr.", "Doe", 5)), sumo:persist(Module, Module:new("Joe", "Armstrong")), - sumo:persist(Module, Module:new("Alan", "Turing", 102, "Computer St.")) + sumo:persist(Module, Module:new("Alan", "Turing", 102, "Computer St.")), + + sumo_test_utils:sleep_if_required(Module) end, lists:foreach(Fun, sumo_test_utils:people_with_conditional_logic()), @@ -61,6 +63,7 @@ end_per_suite(Config) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%% Tests cases %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + backward_compatibility(_Config) -> lists:foreach( fun do_backward_compatibility/1, @@ -144,7 +147,7 @@ do_null_conditional(Module) -> operators(_Config) -> lists:foreach( fun do_operators/1, - sumo_test_utils:people_with_conditional_logic()). + sumo_test_utils:people_with_numeric_sort()). do_operators(Module) -> [_, _] = sumo:find_by(Module, diff --git a/test/sumo_basic_SUITE.erl b/test/sumo_basic_SUITE.erl index 814c076..01ba617 100644 --- a/test/sumo_basic_SUITE.erl +++ b/test/sumo_basic_SUITE.erl @@ -103,8 +103,6 @@ find_by_module(Module) -> SortFun = fun(A, B) -> Module:name(A) < Module:name(B) end, [First, Second | _] = lists:sort(SortFun, Results), - true = is_integer(Module:age(First)), - "B" = to_str(Module:name(First)), "D" = to_str(Module:name(Second)), diff --git a/test/sumo_test_people.erl b/test/sumo_test_people.erl index 6b5e7a5..00a68f6 100644 --- a/test/sumo_test_people.erl +++ b/test/sumo_test_people.erl @@ -24,21 +24,21 @@ -spec sumo_sleep(person()) -> sumo:doc(). sumo_sleep(Person) -> - #{id => Person#person.id, - name => Person#person.name, - last_name => Person#person.last_name, - age => Person#person.age, - address => Person#person.address}. + #{id => Person#person.id, + name => Person#person.name, + last_name => Person#person.last_name, + age => Person#person.age, + address => Person#person.address}. -spec sumo_wakeup(sumo:doc()) -> person(). sumo_wakeup(Person) -> - #person{ - id = maps:get(id, Person), - name = maps:get(name, Person), - last_name = maps:get(last_name, Person), - age = from_bin(maps:get(age, Person), integer), - address = maps:get(address, Person) - }. + #person{ + id = maps:get(id, Person), + name = maps:get(name, Person), + last_name = maps:get(last_name, Person), + age = maps:get(age, Person), + address = maps:get(address, Person) + }. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%% Exported @@ -67,24 +67,3 @@ id(Person) -> age(Person) -> Person#person.age. - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%%% Internals -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -%% @private -%% @doc This helper function is needed by Riak, because data in Riak is stored -%% as Riak Maps, and values in the them must be binary, so when a data is -%% stored in Riak, all values in the map are converted to binary, because -%% of that, is necessary convert values to original types when data is -%% returned. -from_bin(Bin, integer) when is_binary(Bin) -> - binary_to_integer(Bin); -from_bin(Bin, float) when is_binary(Bin) -> - binary_to_float(Bin); -from_bin(Bin, atom) when is_binary(Bin) -> - binary_to_atom(Bin, utf8); -from_bin(Bin, string) when is_binary(Bin) -> - binary_to_list(Bin); -from_bin(Bin, _) -> - Bin. diff --git a/test/sumo_test_utils.erl b/test/sumo_test_utils.erl index cadc8b3..5cf6da7 100644 --- a/test/sumo_test_utils.erl +++ b/test/sumo_test_utils.erl @@ -7,6 +7,8 @@ , people_with_sort/0 , people_with_conditional_logic/0 , people_with_like/0 + , people_with_numeric_sort/0 + , sleep_if_required/1 ]). -spec start_apps() -> ok. @@ -25,7 +27,7 @@ all_people() -> -spec people_with_conditional_logic() -> [atom()]. people_with_conditional_logic() -> - all_people() -- [sumo_test_people_elasticsearch, sumo_test_people_riak]. + all_people() -- [sumo_test_people_elasticsearch]. -spec people_with_sort() -> [atom()]. people_with_sort() -> @@ -39,6 +41,16 @@ people_with_sort() -> people_with_like() -> all_people() -- [ sumo_test_people_elasticsearch - , sumo_test_people_riak , sumo_test_people_mnesia ]. + +-spec people_with_numeric_sort() -> [atom()]. +people_with_numeric_sort() -> + people_with_conditional_logic() -- [sumo_test_people_riak]. + +-spec sleep_if_required(atom()) -> ok. +sleep_if_required(Module) -> + case Module of + sumo_test_people_riak -> timer:sleep(5000); + _ -> ok + end.