From ba7ad190537fa59de87e45671279dbb696475185 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Fri, 9 Apr 2021 14:36:26 +0200 Subject: [PATCH 01/28] emqx: add 'emqx' to beam boot log --- erts/emulator/beam/erl_bif_info.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 6ae5f3ce5750..9fed812a005a 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -87,9 +87,9 @@ static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE " [erts-" ERLANG_VERSION "]" #ifndef OTP_RELEASE #ifdef ERLANG_GIT_VERSION - " [source-" ERLANG_GIT_VERSION "]" + " [emqx-" ERLANG_GIT_VERSION "]" #else - " [source]" + " [emqx]" #endif #endif #if defined(ARCH_64) From 8cf0a0bbb3eb8f6ee852bf9a5f9d9730d484b06a Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Fri, 9 Apr 2021 10:47:53 +0200 Subject: [PATCH 02/28] emqx: dynmaic ipv6 resolution for gen_tcp In this commit, a new option `ipv6_probe` is added for gen_tcp:connect API. The option can either be a proplist boolean flag or a timeout non-neg-integer or `infinity` When this option is provided, gen_tcp will try to connect the target with ipv6, then fallback to default options if failed to connect. --- lib/kernel/src/gen_tcp.erl | 89 +++++++++++++++++++++++++++++++++----- 1 file changed, 78 insertions(+), 11 deletions(-) diff --git a/lib/kernel/src/gen_tcp.erl b/lib/kernel/src/gen_tcp.erl index df0e8b1b13bc..550cc6021387 100644 --- a/lib/kernel/src/gen_tcp.erl +++ b/lib/kernel/src/gen_tcp.erl @@ -114,7 +114,8 @@ recvtclass | recvttl | pktoptions | - ipv6_v6only. + ipv6_v6only | + ipv6_probe. -type connect_option() :: {fd, Fd :: non_neg_integer()} | inet:address_family() | @@ -125,6 +126,7 @@ {tcp_module, module()} | {netns, file:filename_all()} | {bind_to_device, binary()} | + {ipv6_probe, boolean() | timeout()} | option(). -type listen_option() :: {fd, Fd :: non_neg_integer()} | @@ -210,21 +212,86 @@ connect(#{family := Fam} = SockAddr, Opts, Timeout) Reason :: timeout | inet:posix(). connect(Address, Port, Opts0, Timeout) -> - case inet:gen_tcp_module(Opts0) of + %% When neither `inet` nor `inet6` is provided in Opts0, + %% and if `ipv6_probe` option is given, try to connect ipv6 first. + {TryIpv6, Ipv6T} = + case proplists:get_value(ipv6_probe, Opts0) of + true -> {true, 2000}; %% default 2 seconds + false -> {false, 0}; + undefined -> {false, 0}; + T -> {true, T} + end, + %% delete it to avoid interference + Opts1 = proplists:delete(ipv6_probe, Opts0), + case inet:gen_tcp_module(Opts1) of {?MODULE, Opts} -> - Timer = inet:start_timer(Timeout), - Res = (catch connect1(Address,Port,Opts,Timer)), - _ = inet:stop_timer(Timer), - case Res of - {ok,S} -> {ok,S}; - {error, einval} -> exit(badarg); - {'EXIT',Reason} -> exit(Reason); - Error -> Error - end; + connect_maybe_ipv6(Address, Port, Opts, Timeout, TryIpv6, Ipv6T); {GenTcpMod, Opts} -> GenTcpMod:connect(Address, Port, Opts, Timeout) end. +connect_maybe_ipv6(Address, Port, Opts, Timeout, TryIpv6, Ipv6T) -> + case maybe_ipv6(Address, Opts, TryIpv6) of + {maybe, NewOpts} when TryIpv6 -> + try + {ok, _} = connect_0(Address, Port, NewOpts, Ipv6T) + catch + _ : _ -> + %% fallback + connect_0(Address, Port, Opts, Timeout) + end; + NewOpts -> + connect_0(Address, Port, NewOpts, Timeout) + end. + +connect_0(Address, Port, Opts, Timeout) -> + Timer = inet:start_timer(Timeout), + Res = (catch connect1(Address,Port,Opts,Timer)), + _ = inet:stop_timer(Timer), + case Res of + {ok,S} -> {ok,S}; + {error, einval} -> exit(badarg); + {'EXIT',Reason} -> exit(Reason); + Error -> Error + end. + +maybe_ipv6(Host, Opts, TryIpv6) -> + case lists:member(inet, Opts) orelse lists:member(inet6, Opts) of + true -> + Opts; %% caller has made the decision + false when is_tuple(Host) -> + %% ip tuple provided + maybe_ipv6_1(Host, Opts); + false when TryIpv6 -> + %% string host + maybe_ipv6_2(Host, Opts); + false -> + Opts + end. + +maybe_ipv6_1(Host, Opts) when tuple_size(Host) =:= 4 -> Opts; +maybe_ipv6_1(Host, Opts) when tuple_size(Host) =:= 8 -> [inet6 | Opts]. + +maybe_ipv6_2(Host, Opts) -> + case inet:parse_address(Host) of + {ok, Ip} when is_tuple(Ip) -> + %% ip string provided, parsed into tuple + maybe_ipv6_1(Ip, Opts); + _ -> + maybe_ipv6_3(Host, Opts) + end. + +maybe_ipv6_3(Host, Opts) -> + case inet:getaddr(Host, inet6) of + {ok, _} -> + %% the target has a resolvable v6 IP + %% maybe try to connect + {maybe, [inet6 | Opts]}; + _ -> + %% the target has no resolvable v6 IP + Opts + end. + connect1(Address, Port, Opts0, Timer) -> {Mod, Opts} = inet:tcp_module(Opts0, Address), case Mod:getaddrs(Address, Timer) of From be79756a9ac268ba30733b62d92dd63cabc257c1 Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 30 Mar 2021 11:14:38 +0200 Subject: [PATCH 03/28] Configurable send table batch size. Acked-by: William Yang --- lib/mnesia/src/mnesia_loader.erl | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/mnesia/src/mnesia_loader.erl b/lib/mnesia/src/mnesia_loader.erl index 4dea36696429..738702f4fa89 100644 --- a/lib/mnesia/src/mnesia_loader.erl +++ b/lib/mnesia/src/mnesia_loader.erl @@ -909,7 +909,16 @@ get_chunk_func(Pid, Tab, {ext, Alias, Mod}, RemoteS) -> get_chunk_func(Pid, Tab, Storage, RemoteS) -> try TabSize = mnesia:table_info(Tab, size), - KeysPerTransfer = calc_nokeys(Storage, Tab), + KeysPerTransfer = + case ?catch_val(send_table_batch_size) of + {'EXIT', _} -> + mnesia_lib:set(send_table_batch_size, 0), + calc_nokeys(Storage, Tab); + 0 -> + calc_nokeys(Storage, Tab); + Val when is_integer(Val) -> + Val + end, ChunkData = dets:info(Tab, bchunk_format), UseDetsChunk = Storage == RemoteS andalso From 6c3d5120536e2373e3cedca28061c2f9e8ee9243 Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 30 Mar 2021 11:29:39 +0200 Subject: [PATCH 04/28] mnesia send_table_batch_size as app env. --- lib/mnesia/src/mnesia_monitor.erl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/mnesia/src/mnesia_monitor.erl b/lib/mnesia/src/mnesia_monitor.erl index dbd4a9c42f5d..87034ade69ec 100644 --- a/lib/mnesia/src/mnesia_monitor.erl +++ b/lib/mnesia/src/mnesia_monitor.erl @@ -692,6 +692,7 @@ env() -> dc_dump_limit, send_compressed, max_transfer_size, + send_table_batch_size, schema ]. @@ -744,6 +745,8 @@ default_env(send_compressed) -> 0; default_env(max_transfer_size) -> 64000; +default_env(send_table_batch_size) -> + 0; default_env(schema) -> []. @@ -794,6 +797,7 @@ do_check_type(no_table_loaders, N) when is_integer(N), N > 0 -> N; do_check_type(dc_dump_limit,N) when is_number(N), N > 0 -> N; do_check_type(send_compressed, L) when is_integer(L), L >= 0, L =< 9 -> L; do_check_type(max_transfer_size, N) when is_integer(N), N > 0 -> N; +do_check_type(send_table_batch_size, L) when is_integer(L), L >= 0 -> L; do_check_type(schema, L) when is_list(L) -> L. bool(true) -> true; From 95d408cd2b4374e53ef87f1245e5875a480a2cc2 Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 31 Mar 2021 15:45:46 +0200 Subject: [PATCH 05/28] mnesia: copy table from a specified node When a new node is added to the mnesia cluster, it selects a random node to copy the table from if no 'master' node is defined (set master node is not mandatory). by setting arg: 'copy_from_node', operator is now free to choice which node to copy from. default is 'undefined', means disable. note, if the specified node is not in the candidates list, it will fallback to default behavior. --- lib/mnesia/src/mnesia_loader.erl | 14 +++++++++++++- lib/mnesia/src/mnesia_monitor.erl | 4 ++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/mnesia/src/mnesia_loader.erl b/lib/mnesia/src/mnesia_loader.erl index 738702f4fa89..c522f2b53aa8 100644 --- a/lib/mnesia/src/mnesia_loader.erl +++ b/lib/mnesia/src/mnesia_loader.erl @@ -210,7 +210,19 @@ do_get_network_copy(Tab, _Reason, _Ns, unknown, _Cs) -> verbose("Local table copy of ~tp has recently been deleted, ignored.~n", [Tab]), {not_loaded, storage_unknown}; do_get_network_copy(Tab, Reason, Ns, Storage, Cs) -> - [Node | Tail] = Ns, + [Node | Tail] = + case ?catch_val(copy_from_node) of + undefined -> Ns; + CPNode when is_atom(CPNode) -> + case lists:member(CPNode, Ns) of + true -> + [CPNode | Ns -- [CPNode]]; + false -> + Ns + end; + _ -> + Ns + end, case lists:member(Node,val({current, db_nodes})) of true -> dbg_out("Getting table ~tp (~p) from node ~p: ~tp~n", diff --git a/lib/mnesia/src/mnesia_monitor.erl b/lib/mnesia/src/mnesia_monitor.erl index 87034ade69ec..9b06a41108ec 100644 --- a/lib/mnesia/src/mnesia_monitor.erl +++ b/lib/mnesia/src/mnesia_monitor.erl @@ -690,6 +690,7 @@ env() -> pid_sort_order, no_table_loaders, dc_dump_limit, + copy_from_node, send_compressed, max_transfer_size, send_table_batch_size, @@ -741,6 +742,8 @@ default_env(no_table_loaders) -> 2; default_env(dc_dump_limit) -> 4; +default_env(copy_from_node) -> + undefined; default_env(send_compressed) -> 0; default_env(max_transfer_size) -> @@ -795,6 +798,7 @@ do_check_type(pid_sort_order, "standard") -> standard; do_check_type(pid_sort_order, _) -> false; do_check_type(no_table_loaders, N) when is_integer(N), N > 0 -> N; do_check_type(dc_dump_limit,N) when is_number(N), N > 0 -> N; +do_check_type(copy_from_node, L) when is_atom(L) -> L; do_check_type(send_compressed, L) when is_integer(L), L >= 0, L =< 9 -> L; do_check_type(max_transfer_size, N) when is_integer(N), N > 0 -> N; do_check_type(send_table_batch_size, L) when is_integer(L), L >= 0 -> L; From 7887d0ed6d4e861f53738956a961b16ad28bca36 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 16 Nov 2021 01:21:47 +0100 Subject: [PATCH 06/28] chore: export gen_tcp:ipv6_probe/0 as emqx's fork identifier --- lib/kernel/src/gen_tcp.erl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/kernel/src/gen_tcp.erl b/lib/kernel/src/gen_tcp.erl index 550cc6021387..23d0abc09fc1 100644 --- a/lib/kernel/src/gen_tcp.erl +++ b/lib/kernel/src/gen_tcp.erl @@ -28,6 +28,7 @@ -export([send/2, recv/2, recv/3, unrecv/2]). -export([controlling_process/2]). -export([fdopen/2]). +-export([ipv6_probe/0]). -include("inet_int.hrl"). -include("file.hrl"). @@ -153,6 +154,8 @@ %% Connect a socket %% +ipv6_probe() -> true. + -spec connect(SockAddr, Opts) -> {ok, Socket} | {error, Reason} when SockAddr :: socket:sockaddr_in() | socket:sockaddr_in6(), Opts :: [inet:inet_backend() | connect_option()], From 2a0e1d5cae6dd002d6dab3353bbb6d850a92c85f Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 24 Jan 2022 09:19:55 -0300 Subject: [PATCH 07/28] mnesia: Add post-commit hook Merge pull request #20 from emqx/mnesia-post-commit-hook-emqx-OTP-24.1.5 --- lib/mnesia/src/Makefile | 1 + lib/mnesia/src/mnesia.app.src | 1 + lib/mnesia/src/mnesia_hook.erl | 74 ++++++++++++++++++++++++++++++++++ lib/mnesia/src/mnesia_tm.erl | 1 + 4 files changed, 77 insertions(+) create mode 100644 lib/mnesia/src/mnesia_hook.erl diff --git a/lib/mnesia/src/Makefile b/lib/mnesia/src/Makefile index 72aa054fb326..ec540ce4e482 100644 --- a/lib/mnesia/src/Makefile +++ b/lib/mnesia/src/Makefile @@ -55,6 +55,7 @@ MODULES= \ mnesia_ext_sup \ mnesia_frag \ mnesia_frag_hash \ + mnesia_hook \ mnesia_index \ mnesia_kernel_sup \ mnesia_late_loader \ diff --git a/lib/mnesia/src/mnesia.app.src b/lib/mnesia/src/mnesia.app.src index 77bd1a78166f..14117c015156 100644 --- a/lib/mnesia/src/mnesia.app.src +++ b/lib/mnesia/src/mnesia.app.src @@ -15,6 +15,7 @@ mnesia_ext_sup, mnesia_frag, mnesia_frag_hash, + mnesia_hook, mnesia_index, mnesia_kernel_sup, mnesia_late_loader, diff --git a/lib/mnesia/src/mnesia_hook.erl b/lib/mnesia/src/mnesia_hook.erl new file mode 100644 index 000000000000..f9bada98c511 --- /dev/null +++ b/lib/mnesia/src/mnesia_hook.erl @@ -0,0 +1,74 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1996-2021. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(mnesia_hook). + +-include("mnesia.hrl"). + +-export([ + register_hook/2, + do_post_commit/2 + ]). + +-define(hook(NAME), {mnesia_hook, NAME}). + +-type post_commit_hook_data() :: + #{ node => node() + , ram_copies => list() + , disc_copies => list() + , disc_only_copies => list() + , ext => list() + , schema_ops => list() + }. + +-type post_commit_hook() :: fun((_Tid, post_commit_hook_data()) -> ok). + +-spec register_hook(post_commit, post_commit_hook()) -> ok | {error, term()}. +register_hook(post_commit, Hook) when is_function(Hook, 2) -> + persistent_term:put(?hook(post_commit), Hook); +register_hook(_, _) -> + {error, bad_type}. + +-spec do_post_commit(_Tid, #commit{}) -> ok. +do_post_commit(Tid, Commit) -> + case persistent_term:get(?hook(post_commit), undefined) of + undefined -> + ok; + Fun -> + #commit{ node = Node + , ram_copies = Ram + , disc_copies = Disc + , disc_only_copies = DiscOnly + , ext = Ext + , schema_ops = SchemaOps + } = Commit, + CommitData = #{ node => Node + , ram_copies => Ram + , disc_copies => Disc + , disc_only_copies => DiscOnly + , ext => Ext + , schema_ops => SchemaOps + }, + try Fun(Tid, CommitData) + catch EC:Err:Stack -> + mnesia_lib:error("Mnesia post_commit hook failed: ~p:~p~nStacktrace:~p~n", [EC, Err, Stack]) + end, + ok + end. diff --git a/lib/mnesia/src/mnesia_tm.erl b/lib/mnesia/src/mnesia_tm.erl index 847cd0074b81..cb01ad5e5af8 100644 --- a/lib/mnesia/src/mnesia_tm.erl +++ b/lib/mnesia/src/mnesia_tm.erl @@ -1785,6 +1785,7 @@ do_commit(Tid, C, DumperMode) -> R4 = do_update(Tid, disc_only_copies, C#commit.disc_only_copies, R3), R5 = do_update_ext(Tid, C#commit.ext, R4), mnesia_subscr:report_activity(Tid), + mnesia_hook:do_post_commit(Tid, C), R5. %% This could/should be optimized From 44379b83caf7c9ce102771da20db017a675df9b8 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 31 May 2022 14:09:10 -0300 Subject: [PATCH 08/28] feat: add minimal/experimental OCSP server support (TLS 1.2 and 1.3) (OTP 25.1.2) Merge pull request #25 from thalesmg/ocsp-stapling-tmg-otp24 --- lib/ssl/src/ssl.erl | 4 ++ lib/ssl/src/ssl_handshake.erl | 32 ++++++++++++- lib/ssl/src/ssl_internal.hrl | 1 + lib/ssl/src/tls_dtls_connection.erl | 21 ++++++-- lib/ssl/src/tls_handshake_1_3.erl | 51 ++++++++++++++++---- lib/ssl/test/make_certs.erl | 16 +++++++ lib/ssl/test/openssl_ocsp_SUITE.erl | 74 +++++++++++++++++++++++++++-- lib/ssl/test/ssl_test_lib.erl | 11 +++++ 8 files changed, 192 insertions(+), 18 deletions(-) diff --git a/lib/ssl/src/ssl.erl b/lib/ssl/src/ssl.erl index 9835d3d9db39..a88269324b18 100644 --- a/lib/ssl/src/ssl.erl +++ b/lib/ssl/src/ssl.erl @@ -2139,6 +2139,10 @@ validate_option(certfile, Value, _) validate_option(certfile, Value, _) when is_list(Value) -> binary_filename(Value); +validate_option(certificate_status, Value = #certificate_status{}, _) -> + Value; +validate_option(certificate_status, Value = undefined, _) -> + Value; validate_option(client_preferred_next_protocols, {Precedence, PreferredProtocols}, _) when is_list(PreferredProtocols) -> validate_binary_list(client_preferred_next_protocols, PreferredProtocols), diff --git a/lib/ssl/src/ssl_handshake.erl b/lib/ssl/src/ssl_handshake.erl index 3b3e2c50e767..b6942e40a657 100644 --- a/lib/ssl/src/ssl_handshake.erl +++ b/lib/ssl/src/ssl_handshake.erl @@ -583,6 +583,9 @@ encode_handshake(#certificate_request{certificate_types = CertTypes, <> }; +encode_handshake(#certificate_status{status_type = StatusType, response = Response}, _Version) -> + Size = byte_size(Response), + {?CERTIFICATE_STATUS, <>}; encode_handshake(#server_hello_done{}, _Version) -> {?SERVER_HELLO_DONE, <<>>}; encode_handshake(#client_key_exchange{exchange_keys = ExchangeKeys}, Version) -> @@ -779,7 +782,13 @@ encode_cert_status_req( request_extensions = ReqExtns}) -> ResponderIDListBin = encode_responderID_list(ResponderIDList), ReqExtnsBin = encode_request_extensions(ReqExtns), - <>. + <>; +encode_cert_status_req(_StatusType, #certificate_status{} = Status) -> + Version = {3, 4}, + {_, EncStatus} = encode_handshake(Status, Version), + EncStatus; +encode_cert_status_req(_StatusType, _Value) -> + <<>>. encode_responderID_list([]) -> <>; @@ -1493,7 +1502,8 @@ handle_client_hello_extensions(RecordCB, Random, ClientCipherSuites, ConnectionStates, Renegotiation), ec_point_formats => server_ecc_extension(Version, maps:get(ec_point_formats, Exts, undefined)), - max_frag_enum => ServerMaxFragEnum + max_frag_enum => ServerMaxFragEnum, + status_request => handle_status_request(Opts, Exts) }, %% If we receive an ALPN extension and have ALPN configured for this connection, @@ -1566,6 +1576,19 @@ handle_server_hello_extensions(RecordCB, Random, CipherSuite, Compression, end end. +handle_status_request(SSLOptions, ClientExtensions) -> + case {SSLOptions, ClientExtensions} of + { #{certificate_status := #certificate_status{}} + , #{status_request := #certificate_status_request{}} + } -> + #certificate_status_request{ + status_type = ?CERTIFICATE_STATUS_TYPE_OCSP, + request = <<>> + }; + _ -> + undefined + end. + select_curve(Client, Server, Version) -> select_curve(Client, Server, Version, false). @@ -3069,6 +3092,11 @@ decode_extensions(<> -> decode_extensions(Rest, Version, MessageType, Acc#{status_request => #certificate_status{response = ASN1OCSPResponse}}); + <> -> + decode_extensions(Rest, Version, MessageType, + Acc#{status_request => #certificate_status_request{}}); _Other -> decode_extensions(Rest, Version, MessageType, Acc) end; diff --git a/lib/ssl/src/ssl_internal.hrl b/lib/ssl/src/ssl_internal.hrl index 6b1bd360f8f5..7f1c806d6faa 100644 --- a/lib/ssl/src/ssl_internal.hrl +++ b/lib/ssl/src/ssl_internal.hrl @@ -135,6 +135,7 @@ certs_keys => {undefined, [versions]}, certfile => {<<>>, [versions]}, certificate_authorities => {false, [versions]}, + certificate_status => {undefined, []}, ciphers => {[], [versions]}, client_renegotiation => {undefined, [versions]}, cookie => {true, [versions]}, diff --git a/lib/ssl/src/tls_dtls_connection.erl b/lib/ssl/src/tls_dtls_connection.erl index 1b1c724a1f46..ae079aaf8b74 100644 --- a/lib/ssl/src/tls_dtls_connection.erl +++ b/lib/ssl/src/tls_dtls_connection.erl @@ -738,12 +738,19 @@ do_server_hello(Type, #{next_protocol_negotiation := NextProtocols} = handshake_env = HsEnv, session = #session{session_id = SessId}, connection_states = ConnectionStates0, - ssl_options = #{versions := [HighestVersion|_]}} + ssl_options = #{versions := [HighestVersion|_]} = SSLOpts0} = State0) when is_atom(Type) -> %% TLS 1.3 - Section 4.1.3 %% Override server random values for TLS 1.3 downgrade protection mechanism. ConnectionStates1 = update_server_random(ConnectionStates0, Version, HighestVersion), - State1 = State0#state{connection_states = ConnectionStates1}, + SSLOpts1 = case {SSLOpts0, ServerHelloExt} of + { #{certificate_status := #certificate_status{}} + , #{status_request := #certificate_status_request{}} + } -> SSLOpts0; + _ -> SSLOpts0#{certificate_status => undefined} + end, + State1 = State0#state{connection_states = ConnectionStates1, + ssl_options = SSLOpts1}, ServerHello = ssl_handshake:server_hello(SessId, ssl:tls_version(Version), ConnectionStates1, ServerHelloExt), @@ -920,8 +927,14 @@ do_client_certify_and_key_exchange(State0, Connection) -> server_certify_and_key_exchange(State0, Connection) -> State1 = certify_server(State0, Connection), - State2 = key_exchange(State1, Connection), - request_client_cert(State2, Connection). + State2 = certificate_status(State1, Connection), + State3 = key_exchange(State2, Connection), + request_client_cert(State3, Connection). + +certificate_status(#state{ssl_options = #{certificate_status := #certificate_status{} = Status}} = State, Connection) -> + Connection:queue_handshake(Status, State); +certificate_status(State, _) -> + State. certify_client_key_exchange(#encrypted_premaster_secret{premaster_secret= EncPMS}, #state{session = #session{private_key = PrivateKey}, diff --git a/lib/ssl/src/tls_handshake_1_3.erl b/lib/ssl/src/tls_handshake_1_3.erl index e68b2f96f02c..a66036e9d9ed 100644 --- a/lib/ssl/src/tls_handshake_1_3.erl +++ b/lib/ssl/src/tls_handshake_1_3.erl @@ -273,10 +273,11 @@ certificate(undefined, _, _, _, client) -> {ok, #certificate_1_3{ certificate_request_context = <<>>, certificate_list = []}}; -certificate([OwnCert], CertDbHandle, CertDbRef, _CRContext, Role) -> +certificate([OwnCert], CertDbHandle, CertDbRef, CRContext, Role) -> case ssl_certificate:certificate_chain(OwnCert, CertDbHandle, CertDbRef) of {ok, _, Chain} -> - CertList = chain_to_cert_list(Chain), + CertList0 = chain_to_cert_list(Chain), + CertList = maybe_add_certificate_entry_extensions(CertList0, CRContext), %% If this message is in response to a CertificateRequest, the value of %% certificate_request_context in that message. Otherwise (in the case %%of server authentication), this field SHALL be zero length. @@ -296,8 +297,9 @@ certificate([OwnCert], CertDbHandle, CertDbRef, _CRContext, Role) -> certificate_request_context = <<>>, certificate_list = []}} end; -certificate([_,_| _] = Chain, _,_,_,_) -> - CertList = chain_to_cert_list(Chain), +certificate([_,_| _] = Chain, _,_,CRContext,_) -> + CertList0 = chain_to_cert_list(Chain), + CertList = maybe_add_certificate_entry_extensions(CertList0, CRContext), {ok, #certificate_1_3{ certificate_request_context = <<>>, certificate_list = CertList}}. @@ -702,6 +704,7 @@ do_start(#client_hello{cipher_suites = ClientCiphers, State2 end, + StatusRequest = maps:get(status_request, Extensions, undefined), State4 = update_start_state(State3, #{cipher => Cipher, key_share => KeyShare, @@ -709,7 +712,8 @@ do_start(#client_hello{cipher_suites = ClientCiphers, group => Group, sign_alg => SelectedSignAlg, peer_public_key => ClientPubKey, - alpn => ALPNProtocol}), + alpn => ALPNProtocol, + status_request => StatusRequest}), %% 4.1.4. Hello Retry Request %% @@ -1353,11 +1357,20 @@ maybe_send_certificate_request(#state{static_env = #static_env{protocol_cb = Con maybe_send_certificate(State, PSK) when PSK =/= undefined -> {ok, State}; maybe_send_certificate(#state{session = #session{own_certificates = OwnCerts}, + protocol_specific = ProtocolSpecific, + ssl_options = SslOpts, static_env = #static_env{ protocol_cb = Connection, cert_db = CertDbHandle, cert_db_ref = CertDbRef}} = State, _) -> - case certificate(OwnCerts, CertDbHandle, CertDbRef, <<>>, server) of + %% hack: apparently, CRContext is not used by the server (whatever + %% that may be...) + StatusRequest = maps:get(status_request, ProtocolSpecific, undefined), + CertificateStatus = maps:get(certificate_status, SslOpts, undefined), + CRContext = #{ status_request => StatusRequest + , certificate_status => CertificateStatus + }, + case certificate(OwnCerts, CertDbHandle, CertDbRef, CRContext, server) of {ok, Certificate} -> {ok, Connection:queue_handshake(Certificate, State)}; Error -> @@ -1923,16 +1936,19 @@ update_start_state(State, Map) -> SelectedSignAlg = maps:get(sign_alg, Map, undefined), PeerPublicKey = maps:get(peer_public_key, Map, undefined), ALPNProtocol = maps:get(alpn, Map, undefined), + StatusRequest = maps:get(status_request, Map, undefined), update_start_state(State, Cipher, KeyShare, SessionId, Group, SelectedSignAlg, PeerPublicKey, - ALPNProtocol). + ALPNProtocol, StatusRequest). %% update_start_state(#state{connection_states = ConnectionStates0, handshake_env = #handshake_env{} = HsEnv, + protocol_specific = ProtocolSpecific0, connection_env = CEnv, session = Session} = State, Cipher, KeyShare, SessionId, - Group, SelectedSignAlg, PeerPublicKey, ALPNProtocol) -> + Group, SelectedSignAlg, PeerPublicKey, ALPNProtocol, + StatusRequest) -> #{security_parameters := SecParamsR0} = PendingRead = maps:get(pending_read, ConnectionStates0), #{security_parameters := SecParamsW0} = PendingWrite = @@ -1942,9 +1958,11 @@ update_start_state(#state{connection_states = ConnectionStates0, ConnectionStates = ConnectionStates0#{pending_read => PendingRead#{security_parameters => SecParamsR}, pending_write => PendingWrite#{security_parameters => SecParamsW}}, + ProtocolSpecific = ProtocolSpecific0#{status_request => StatusRequest}, State#state{connection_states = ConnectionStates, handshake_env = HsEnv#handshake_env{alpn = ALPNProtocol}, key_share = KeyShare, + protocol_specific = ProtocolSpecific, session = Session#session{session_id = SessionId, ecc = Group, sign_alg = SelectedSignAlg, @@ -3081,3 +3099,20 @@ plausible_missing_chain([_] = EncodedChain, undefined, SignAlg, Key, Session0) - plausible_missing_chain(_,Plausible,_,_,_) -> Plausible. +maybe_add_certificate_entry_extensions( + [ServerCertEntry = #certificate_entry{} | Rest], + #{ status_request := #certificate_status_request{} = Req + , certificate_status := #certificate_status{} = Status + }) -> + [ ServerCertEntry#certificate_entry{ + extensions = + #{ status_request => + Req#certificate_status_request{ + status_type = ?CERTIFICATE_STATUS_TYPE_OCSP, + request = Status + } + } + } + | Rest]; +maybe_add_certificate_entry_extensions(CertList, _CRContext) -> + CertList. diff --git a/lib/ssl/test/make_certs.erl b/lib/ssl/test/make_certs.erl index e786cddeb642..0640c3dc5f4f 100644 --- a/lib/ssl/test/make_certs.erl +++ b/lib/ssl/test/make_certs.erl @@ -183,6 +183,22 @@ revoke(Root, CA, User, C) -> cmd(Cmd, Env), gencrl(Root, CA, C). +make_ocsp_response(Port, Root, CA, User, Issuer, C) -> + UsrCert = filename:join([Root, User, "cert.pem"]), + IssuerCert = filename:join([Root, Issuer, "cert.pem"]), + CACertFile = filename:join([Root, CA, "cert.pem"]), + OCSPResp = filename:join([Root, User, "ocsp.resp"]), + Cmd = [C#config.openssl_cmd, " ocsp" + " -CAfile ", CACertFile, + " -url ", "http://localhost:" ++ integer_to_list(Port), + " -no_nonce", + " -respout ", OCSPResp, + " -issuer ", IssuerCert, + " -cert ", UsrCert], + Env = [{"ROOTDIR", filename:absname(Root)}], + cmd(Cmd, Env), + OCSPResp. + %% Remove the certificate's entry from the database. The OCSP responder %% will consider the certificate to be unknown. remove_entry(Root, CA, User, C) -> diff --git a/lib/ssl/test/openssl_ocsp_SUITE.erl b/lib/ssl/test/openssl_ocsp_SUITE.erl index 888a0ab3c44e..054cbed2f57f 100644 --- a/lib/ssl/test/openssl_ocsp_SUITE.erl +++ b/lib/ssl/test/openssl_ocsp_SUITE.erl @@ -23,6 +23,7 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("public_key/include/public_key.hrl"). -include("ssl_test_lib.hrl"). +-include("ssl_handshake.hrl"). %% Callback functions -export([all/0, @@ -40,7 +41,8 @@ ocsp_stapling_with_responder_cert/0,ocsp_stapling_with_responder_cert/1, ocsp_stapling_revoked/0, ocsp_stapling_revoked/1, ocsp_stapling_undetermined/0, ocsp_stapling_undetermined/1, - ocsp_stapling_no_staple/0, ocsp_stapling_no_staple/1 + ocsp_stapling_no_staple/0, ocsp_stapling_no_staple/1, + ocsp_stapling_server/0, ocsp_stapling_server/1 ]). %% spawn export @@ -55,9 +57,9 @@ all() -> {group, 'tlsv1.2'}, {group, 'dtlsv1.2'}]. -groups() -> - [{'tlsv1.3', [], ocsp_tests()}, - {'tlsv1.2', [], ocsp_tests()}, +groups() -> + [{'tlsv1.3', [], ocsp_tests() ++ ocsp_server_tests()}, + {'tlsv1.2', [], ocsp_tests() ++ ocsp_server_tests()}, {'dtlsv1.2', [], ocsp_tests()}]. ocsp_tests() -> @@ -69,6 +71,9 @@ ocsp_tests() -> ocsp_stapling_no_staple ]. +ocsp_server_tests() -> + [ocsp_stapling_server]. + %%-------------------------------------------------------------------- init_per_suite(Config) -> case ssl_test_lib:openssl_ocsp_support() of @@ -178,6 +183,61 @@ ocsp_stapling_helper(Config, Opts) -> ssl_test_lib:close(Server), ssl_test_lib:close(Client). %%-------------------------------------------------------------------- + +ocsp_stapling_server() -> + [{doc, "Verify basic OCSP stapling works (server side)"}]. +ocsp_stapling_server(Config0) + when is_list(Config0) -> + PrivDir = proplists:get_value(priv_dir, Config0), + ResponderPort = proplists:get_value(responder_port, Config0), + OCSPRespPath = make_certs:make_ocsp_response(ResponderPort, PrivDir, "otpCA", + "server", "b.server", + make_certs:default_config()), + {ok, OCSPRespDer} = file:read_file(OCSPRespPath), + ServerOpts = proplists:get_value(server_opts, Config0, []), + Config = [ {server_opts, [ {sni_fun, + fun(SN) -> ocsp_sni_fun(SN, OCSPRespDer) end} + | ServerOpts]} + | Config0], + ocsp_stapling_server_helper(Config, []). + +ocsp_stapling_server_helper(Config, Opts) -> + Data = "ping", %% 4 bytes + GroupName = undefined, + ServerOpts = [{group, GroupName}], + Server = ssl_test_lib:start_server(erlang, + [{options, ServerOpts}], + Config), + Port = ssl_test_lib:inet_port(Server), + + ClientOpts = ssl_test_lib:ssl_options(Opts, Config), + Client = ssl_test_lib:start_client(openssl, + [{port, Port}, + {options, ClientOpts}, + {server_name_indication, "server"}, + {ocsp_stapling, true}, + {ocsp_nonce, false}, + {debug_openssl, false}], + Config), + true = is_pid(Client), + ct:sleep(1000), + {messages, ClientMsgs} = process_info(Client, messages), + [OCSPOutput] = [Output || + {_Port, {data, Output}} <- ClientMsgs, + case re:run(Output, "OCSP response") of + {match, _} -> true; + _ -> false + end], + {match, _} = re:run(OCSPOutput, "Response Status: successful"), + {match, _} = re:run(OCSPOutput, "Cert Status:"), + + ssl_test_lib:check_active_receive(Server, "Hello world"), + ssl_test_lib:send(Client, Data), + Data = ssl_test_lib:check_active_receive(Server, Data), + ssl_test_lib:close(Server), + ssl_test_lib:close(Client). + +%%-------------------------------------------------------------------- ocsp_stapling_revoked() -> [{doc, "Verify OCSP stapling works with revoked certificate."}]. ocsp_stapling_revoked(Config) @@ -276,3 +336,9 @@ get_free_port() -> {ok, Port} = inet:port(Listen), ok = gen_tcp:close(Listen), Port. + +ocsp_sni_fun(_Servername, OCSPRespDer) -> + [{certificate_status, #certificate_status{ + status_type = 1, + response = OCSPRespDer + }}]. diff --git a/lib/ssl/test/ssl_test_lib.erl b/lib/ssl/test/ssl_test_lib.erl index ba2493bd72a7..c4060e06015c 100644 --- a/lib/ssl/test/ssl_test_lib.erl +++ b/lib/ssl/test/ssl_test_lib.erl @@ -2212,6 +2212,7 @@ start_client(openssl, Port, ClientOpts, Config) -> HostName = proplists:get_value(hostname, ClientOpts, net_adm:localhost()), SNI = openssl_sni(proplists:get_value(server_name_indication, ClientOpts, undefined)), Debug = openssl_debug_options(DOpenssl), + OCSPStatus = openssl_ocsp_status(proplists:get_value(ocsp_stapling, ClientOpts, undefined)), Exe = "openssl", Args0 = case Groups0 of @@ -2230,6 +2231,7 @@ start_client(openssl, Port, ClientOpts, Config) -> Reconnect ++ MaxFragLen ++ SessionArgs ++ + OCSPStatus ++ Debug; Group -> ["s_client", @@ -2247,6 +2249,7 @@ start_client(openssl, Port, ClientOpts, Config) -> Reconnect ++ MaxFragLen ++ SessionArgs ++ + OCSPStatus ++ Debug end, Args = maybe_force_ipv4(Args0), @@ -2390,6 +2393,14 @@ openssl_debug_options(true) -> ["-msg", "-debug"]; openssl_debug_options(false) -> []. + +openssl_ocsp_status(undefined) -> + []; +openssl_ocsp_status(true) -> + ["-status"]; +openssl_ocsp_status(false) -> + []. + %% openssl_debug_options(PrivDir, true) -> case is_keylogfile_supported() of From e6dad3a57362a70c21ff8786792c00b8a76b97a2 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 5 Dec 2022 18:53:03 +0300 Subject: [PATCH 09/28] chore: bump OTP version to 25.1.2-1 --- OTP_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OTP_VERSION b/OTP_VERSION index 44c5306e2472..fba3f8593772 100644 --- a/OTP_VERSION +++ b/OTP_VERSION @@ -1 +1 @@ -25.1.2 +25.1.2-1 From 9a2cba889a7293850992f4d34f08228107b04708 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Wed, 1 Dec 2021 22:16:02 +0800 Subject: [PATCH 10/28] feat(ssl): no warning log when using verify_none --- lib/ssl/src/ssl.erl | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/lib/ssl/src/ssl.erl b/lib/ssl/src/ssl.erl index a88269324b18..126fa3cb67ee 100644 --- a/lib/ssl/src/ssl.erl +++ b/lib/ssl/src/ssl.erl @@ -1539,9 +1539,9 @@ handle_options(Transport, Socket, Opts0, Role, Host) -> #{role => Role, host => Host, rules => ?RULES}), - - maybe_client_warn_no_verify(SslOpts2, Role), + SslOpts = maps:without([warn_verify_none], SslOpts2), + %% Handle special options {Sock, Emulated} = emulated_options(Transport, Socket, Protocol, SockOpts0), ConnetionCb = connection_cb(Protocol), @@ -2874,15 +2874,6 @@ add_filter(undefined, Filters) -> add_filter(Filter, Filters) -> [Filter | Filters]. -maybe_client_warn_no_verify(#{verify := verify_none, - warn_verify_none := true, - log_level := LogLevel}, client) -> - ssl_logger:log(warning, LogLevel, #{description => "Authenticity is not established by certificate path validation", - reason => "Option {verify, verify_peer} and cacertfile/cacerts is missing"}, ?LOCATION); -maybe_client_warn_no_verify(_,_) -> - %% Warning not needed. Note client certificate validation is optional in TLS - ok. - unambiguous_path(Value) -> AbsName = filename:absname(Value), case file:read_link(AbsName) of From 550f66aebb5950d578419b426caeeaf146ed974b Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 7 Dec 2022 15:05:02 +0300 Subject: [PATCH 11/28] ci: disable corresponding testcase --- lib/ssl/test/ssl_api_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ssl/test/ssl_api_SUITE.erl b/lib/ssl/test/ssl_api_SUITE.erl index 6058672c4bf4..887796943465 100644 --- a/lib/ssl/test/ssl_api_SUITE.erl +++ b/lib/ssl/test/ssl_api_SUITE.erl @@ -315,7 +315,7 @@ gen_api_tests() -> cb_info, log_alert, getstat, - warn_verify_none, + % warn_verify_none, suppress_warn_verify_none, check_random_nonce, cipher_listing From e7e56f6e125cedb977ea4aa3a49e5d6a5db90f6d Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 8 Dec 2022 10:47:12 +0300 Subject: [PATCH 12/28] fix(ipv6_probe): handle local sockets properly They are legal `Address`es to connect to. --- lib/kernel/src/gen_tcp.erl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/kernel/src/gen_tcp.erl b/lib/kernel/src/gen_tcp.erl index 23d0abc09fc1..e502d9aec496 100644 --- a/lib/kernel/src/gen_tcp.erl +++ b/lib/kernel/src/gen_tcp.erl @@ -258,6 +258,9 @@ connect_0(Address, Port, Opts, Timeout) -> Error -> Error end. +maybe_ipv6({local, _}, Opts, _TryIpv6) -> + %% unapplicable to local sockets + Opts; maybe_ipv6(Host, Opts, TryIpv6) -> case lists:member(inet, Opts) orelse lists:member(inet6, Opts) of true -> @@ -272,8 +275,8 @@ maybe_ipv6(Host, Opts, TryIpv6) -> Opts end. -maybe_ipv6_1(Host, Opts) when tuple_size(Host) =:= 4 -> Opts; -maybe_ipv6_1(Host, Opts) when tuple_size(Host) =:= 8 -> [inet6 | Opts]. +maybe_ipv6_1(Ip, Opts) when tuple_size(Ip) =:= 4 -> Opts; +maybe_ipv6_1(Ip, Opts) when tuple_size(Ip) =:= 8 -> [inet6 | Opts]. maybe_ipv6_2(Host, Opts) -> case inet:parse_address(Host) of From fdd8f9358baea5de7f817bc53bbcf36ab58ce05f Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 8 Dec 2022 16:50:52 +0300 Subject: [PATCH 13/28] fix(vsn): bump OTP version to 25.1.2-2 --- OTP_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OTP_VERSION b/OTP_VERSION index fba3f8593772..d2fcbc4324bd 100644 --- a/OTP_VERSION +++ b/OTP_VERSION @@ -1 +1 @@ -25.1.2-1 +25.1.2-2 From 56c54f720a44937b4ee51f6cae502487ad21b967 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Tue, 21 Mar 2023 18:02:50 +0200 Subject: [PATCH 14/28] feat(mnesia): implement unregister_hook/1 in mnesia_hook.erl --- lib/mnesia/src/mnesia_hook.erl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/mnesia/src/mnesia_hook.erl b/lib/mnesia/src/mnesia_hook.erl index f9bada98c511..e23c8a6471e1 100644 --- a/lib/mnesia/src/mnesia_hook.erl +++ b/lib/mnesia/src/mnesia_hook.erl @@ -24,6 +24,7 @@ -export([ register_hook/2, + unregister_hook/1, do_post_commit/2 ]). @@ -46,6 +47,12 @@ register_hook(post_commit, Hook) when is_function(Hook, 2) -> register_hook(_, _) -> {error, bad_type}. +-spec unregister_hook(post_commit) -> boolean() | {error, term()}. +unregister_hook(post_commit) -> + persistent_term:erase(?hook(post_commit)); +unregister_hook(_) -> + {error, bad_type}. + -spec do_post_commit(_Tid, #commit{}) -> ok. do_post_commit(Tid, Commit) -> case persistent_term:get(?hook(post_commit), undefined) of From 085dda3425725d045988a8f4a46dbd0bc08279a0 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Tue, 21 Mar 2023 19:24:58 +0200 Subject: [PATCH 15/28] chore: bump otp version to 25.1.2-3 --- OTP_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OTP_VERSION b/OTP_VERSION index d2fcbc4324bd..50fecf3dd821 100644 --- a/OTP_VERSION +++ b/OTP_VERSION @@ -1 +1 @@ -25.1.2-2 +25.1.2-3 From 30522728168fe74a469104f29fa18f79fd4e63f7 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Mon, 21 Aug 2023 19:32:25 +0300 Subject: [PATCH 16/28] fix(mnesia): exclude sensitive data from mnesia_hook log message Closes: EMQX-10390 --- lib/mnesia/src/mnesia_hook.erl | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/lib/mnesia/src/mnesia_hook.erl b/lib/mnesia/src/mnesia_hook.erl index e23c8a6471e1..dc5d4afbb704 100644 --- a/lib/mnesia/src/mnesia_hook.erl +++ b/lib/mnesia/src/mnesia_hook.erl @@ -74,8 +74,32 @@ do_post_commit(Tid, Commit) -> , schema_ops => SchemaOps }, try Fun(Tid, CommitData) - catch EC:Err:Stack -> - mnesia_lib:error("Mnesia post_commit hook failed: ~p:~p~nStacktrace:~p~n", [EC, Err, Stack]) + catch EC:Err:St -> + CommitTabs = commit_tabs(Ram, Disc, DiscOnly, Ext), + mnesia_lib:error("Mnesia post_commit hook failed: ~p:~p~nStacktrace:~p~nCommit tables:~p~n", + [EC, Err, stack_without_args(St), CommitTabs]) end, ok end. + +%% May be helpful for debugging +commit_tabs(Ram, Disc, DiscOnly, Ext) -> + Acc = tabs_from_ops(Ram, []), + Acc1 = tabs_from_ops(Disc, Acc), + Acc2 = tabs_from_ops(DiscOnly, Acc1), + lists:uniq(tabs_from_ops(Ext, Acc2)). + +tabs_from_ops([{{Tab, _K}, _Val, _Op} | T], Acc) -> + tabs_from_ops(T, [Tab | Acc]); +tabs_from_ops([_ | T], Acc) -> + tabs_from_ops(T, Acc); +tabs_from_ops([], Acc) -> + Acc. + +%% Args may contain sensitive data +stack_without_args([{M, F, Args, Info} | T]) when is_list(Args) -> + [{M, F, length(Args), Info} | stack_without_args(T)]; +stack_without_args([StItem | T] ) -> + [StItem | stack_without_args(T)]; +stack_without_args([]) -> + []. From 23e873fa79e140b5fbf1f33e5015afd46070d669 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Tue, 22 Aug 2023 15:23:59 +0300 Subject: [PATCH 17/28] fix(mnesia): report mnesia_hook post_commit failures at debug level --- lib/mnesia/src/mnesia_hook.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/mnesia/src/mnesia_hook.erl b/lib/mnesia/src/mnesia_hook.erl index dc5d4afbb704..cb23868db68d 100644 --- a/lib/mnesia/src/mnesia_hook.erl +++ b/lib/mnesia/src/mnesia_hook.erl @@ -76,8 +76,8 @@ do_post_commit(Tid, Commit) -> try Fun(Tid, CommitData) catch EC:Err:St -> CommitTabs = commit_tabs(Ram, Disc, DiscOnly, Ext), - mnesia_lib:error("Mnesia post_commit hook failed: ~p:~p~nStacktrace:~p~nCommit tables:~p~n", - [EC, Err, stack_without_args(St), CommitTabs]) + mnesia_lib:dbg_out("Mnesia post_commit hook failed: ~p:~p~nStacktrace:~p~nCommit tables:~p~n", + [EC, Err, stack_without_args(St), CommitTabs]) end, ok end. From dfc38ee3e5eb95b8bb6760884dafcefbcc6f702a Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Mon, 21 Aug 2023 19:36:59 +0300 Subject: [PATCH 18/28] chore: bump otp version to 25.3.2-2 --- OTP_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OTP_VERSION b/OTP_VERSION index 37b777424592..327376ab0ba1 100644 --- a/OTP_VERSION +++ b/OTP_VERSION @@ -1 +1 @@ -25.3.2-1 +25.3.2-2 From 51b1d8a2d38cfa9e4ee7394336114c29c3f8b27c Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 21 Nov 2023 14:01:01 -0300 Subject: [PATCH 19/28] test(ssl_api): remove references to non-existent test cases --- lib/ssl/test/ssl_api_SUITE.erl | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/ssl/test/ssl_api_SUITE.erl b/lib/ssl/test/ssl_api_SUITE.erl index 0a611225d467..aa93a021a02e 100644 --- a/lib/ssl/test/ssl_api_SUITE.erl +++ b/lib/ssl/test/ssl_api_SUITE.erl @@ -325,8 +325,6 @@ gen_api_tests() -> ssl_not_started, log_alert, getstat, - % warn_verify_none, - suppress_warn_verify_none, check_random_nonce, cipher_listing ]. From eed9fec23cc68d409845160aae8f2872c5fceeaf Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 21 Nov 2023 11:07:39 -0300 Subject: [PATCH 20/28] chore: prepare to tag 26.1.2-1 --- OTP_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OTP_VERSION b/OTP_VERSION index aef485e63578..d26da36ff815 100644 --- a/OTP_VERSION +++ b/OTP_VERSION @@ -1 +1 @@ -26.1.2 +26.1.2-1 From 2b642a1a54e50c595698efa59be2837e43162c96 Mon Sep 17 00:00:00 2001 From: frazze-jobb Date: Fri, 27 Oct 2023 20:14:00 +0200 Subject: [PATCH 21/28] stdlib: Prevent crash when writing a faulty attribute like '1> -hej.' --- lib/stdlib/src/shell.erl | 5 +++-- lib/stdlib/test/shell_SUITE.erl | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/stdlib/src/shell.erl b/lib/stdlib/src/shell.erl index af9aea26d651..e7fb5674d900 100644 --- a/lib/stdlib/src/shell.erl +++ b/lib/stdlib/src/shell.erl @@ -279,7 +279,7 @@ get_command(Prompt, Eval, Bs, RT, FT, Ds) -> [text,{reserved_word_fun,ResWordFun}]) of {ok,Toks,_EndPos} -> - %% NOTE: we can handle function definitions, records and soon type declarations + %% NOTE: we can handle function definitions, records and type declarations %% but this cannot be handled by the function which only expects erl_parse:abstract_expressions() %% for now just pattern match against those types and pass the string to shell local func. case Toks of @@ -300,7 +300,8 @@ get_command(Prompt, Eval, Bs, RT, FT, Ds) -> case Atom of record -> SpecialCase(rd); spec -> SpecialCase(ft); - type -> SpecialCase(td) + type -> SpecialCase(td); + _ -> erl_eval:extended_parse_exprs(Toks) end; [{atom, _, FunName}, {'(', _}|_] -> case erl_parse:parse_form(Toks) of diff --git a/lib/stdlib/test/shell_SUITE.erl b/lib/stdlib/test/shell_SUITE.erl index 142f8ad4450f..8996e4c039f7 100644 --- a/lib/stdlib/test/shell_SUITE.erl +++ b/lib/stdlib/test/shell_SUITE.erl @@ -332,6 +332,7 @@ types(Config) when is_list(Config) -> [ok] = scan(<<"-spec foo(Bar) -> Baz when Bar :: string(), Baz :: integer().">>), + "exception error"++_ = comm_err(<<"-hej.">>), shell_attribute_test(Config), ok. shell_attribute_test(Config) -> From 87f2728a7825e614fc69524e55fe171e0e21ce6e Mon Sep 17 00:00:00 2001 From: frazze-jobb Date: Fri, 27 Oct 2023 20:14:48 +0200 Subject: [PATCH 22/28] stdlib: Allow local func v(), in a restricted shell --- lib/stdlib/src/shell.erl | 4 +++- lib/stdlib/test/shell_SUITE.erl | 27 +++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/lib/stdlib/src/shell.erl b/lib/stdlib/src/shell.erl index e7fb5674d900..2662236ab636 100644 --- a/lib/stdlib/src/shell.erl +++ b/lib/stdlib/src/shell.erl @@ -974,7 +974,7 @@ not_restricted(exit, []) -> true; not_restricted(fl, []) -> true; -not_restricted(fd, [_]) -> +not_restricted(fd, [_,_]) -> true; not_restricted(ft, [_]) -> true; @@ -1000,6 +1000,8 @@ not_restricted(rr, [_,_]) -> true; not_restricted(rr, [_,_,_]) -> true; +not_restricted(v, [_]) -> + true; not_restricted(_, _) -> false. diff --git a/lib/stdlib/test/shell_SUITE.erl b/lib/stdlib/test/shell_SUITE.erl index 8996e4c039f7..ef2015b9d476 100644 --- a/lib/stdlib/test/shell_SUITE.erl +++ b/lib/stdlib/test/shell_SUITE.erl @@ -185,12 +185,39 @@ comm_err(<<"ugly().">>), "exception exit: " "restricted shell module returned bad value non_conforming_reply" = comm_err(<<"1 - 2.">>), +%% Make sure we test all local shell functions in a restricted shell. +LocalFuncs = shell:local_func(), +[] = lists:subtract(LocalFuncs, [v,h,b,f,fl,rd,rf,rl,rp,rr,history,results,catch_exception]), + +LocalFuncs2 = [ + <<"A = 1.\nv(1).">>, <<"h().">>, <<"b().">>, <<"f().">>, <<"f(A).">>, + <<"fl()">>, <<"rd(foo,{bar}).">>, <<"rf().">>, <<"rf(foo).">>, <<"rl().">>, <<"rl(foo).">>, <<"rp([hej]).">>, + <<"rr(shell).">>, <<"rr(shell, shell_state).">>, <<"rr(shell,shell_state,[]).">>, + <<"history(20).">>, <<"results(20).">>, <<"catch_exception(0).">>], +lists:foreach(fun(LocalFunc) -> + try + ("exception exit: restricted shell does not allow"++_Rest) = Error = local_func_error_t(LocalFunc), + error(Error) + catch + error:{badmatch, _} -> ok + end + end, + LocalFuncs2), "exception exit: restricted shell stopped"= comm_err(<<"begin shell:stop_restricted() end.">>), undefined = application:get_env(stdlib, restricted_shell), ok. +local_func_error_t(B) -> + Reply = t(B), + S0 = lists:flatten(string:replace(Reply, "1\n", "")), %% v(1) requires special treatment + S1 = string:left(S0, string:chr(S0, $\n)-1), + S2 = string:strip(S1, left, $*), + S3 = string:strip(S2, both, $ ), + S = string:strip(S3, both, $"), + string:strip(S, right, $.). + %% Check restricted shell when started from the command line. start_restricted_on_command_line(Config) when is_list(Config) -> {ok, Peer, Node} = ?CT_PEER(["-pa", proplists:get_value(priv_dir,Config), From dc2ae544fd91c84d07c3b0bb1006bee4c95c8b06 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 11 Dec 2023 14:30:00 +0100 Subject: [PATCH 23/28] chore: bump OTP_VERSION to 26.1.2-2 --- OTP_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OTP_VERSION b/OTP_VERSION index d26da36ff815..12bd8af6dac2 100644 --- a/OTP_VERSION +++ b/OTP_VERSION @@ -1 +1 @@ -26.1.2-1 +26.1.2-2 From d54de5f1ecd35a8e4a99c58662d61ec19f4c9e64 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Wed, 13 Dec 2023 19:41:06 +0200 Subject: [PATCH 24/28] feat: implement `mnesia:match_delete/2` function --- lib/mnesia/src/mnesia.erl | 15 +- lib/mnesia/src/mnesia_checkpoint.erl | 9 +- lib/mnesia/src/mnesia_dumper.erl | 8 +- lib/mnesia/src/mnesia_log.erl | 7 +- lib/mnesia/src/mnesia_subscr.erl | 11 +- lib/mnesia/src/mnesia_tm.erl | 2 +- lib/mnesia/test/Makefile | 3 +- lib/mnesia/test/mnesia_SUITE.erl | 3 +- lib/mnesia/test/mnesia_match_delete_test.erl | 217 +++++++++++++++++++ 9 files changed, 260 insertions(+), 15 deletions(-) create mode 100644 lib/mnesia/test/mnesia_match_delete_test.erl diff --git a/lib/mnesia/src/mnesia.erl b/lib/mnesia/src/mnesia.erl index bb5858b3f832..c4b547fac6a5 100644 --- a/lib/mnesia/src/mnesia.erl +++ b/lib/mnesia/src/mnesia.erl @@ -99,6 +99,7 @@ read_table_property/2, write_table_property/2, delete_table_property/2, change_table_frag/2, clear_table/1, clear_table/4, + match_delete/2, %% Table load dump_tables/1, wait_for_tables/2, force_load_table/1, @@ -2808,21 +2809,25 @@ change_table_copy_type(T, N, S) -> -spec clear_table(Tab::table()) -> t_result('ok'). clear_table(Tab) -> + match_delete(Tab, '_'). + +-spec match_delete(Tab::table(), ets:match_pattern()) -> t_result('ok'). +match_delete(Tab, Pattern) -> case get(mnesia_activity_state) of State = {Mod, Tid, _Ts} when element(1, Tid) =/= tid -> - transaction(State, fun() -> do_clear_table(Tab) end, [], infinity, Mod, sync); + transaction(State, fun() -> do_clear_table(Tab, Pattern) end, [], infinity, Mod, sync); undefined -> - transaction(undefined, fun() -> do_clear_table(Tab) end, [], infinity, ?DEFAULT_ACCESS, sync); + transaction(undefined, fun() -> do_clear_table(Tab, Pattern) end, [], infinity, ?DEFAULT_ACCESS, sync); _ -> %% Not allowed for clear_table mnesia:abort({aborted, nested_transaction}) end. -do_clear_table(Tab) -> +do_clear_table(Tab, Pattern) -> case get(mnesia_activity_state) of {?DEFAULT_ACCESS, Tid, Ts} -> - clear_table(Tid, Ts, Tab, '_'); + clear_table(Tid, Ts, Tab, Pattern); {Mod, Tid, Ts} -> - Mod:clear_table(Tid, Ts, Tab, '_'); + Mod:clear_table(Tid, Ts, Tab, Pattern); _ -> abort(no_transaction) end. diff --git a/lib/mnesia/src/mnesia_checkpoint.erl b/lib/mnesia/src/mnesia_checkpoint.erl index ed1c0df605e9..505c604e4ddd 100644 --- a/lib/mnesia/src/mnesia_checkpoint.erl +++ b/lib/mnesia/src/mnesia_checkpoint.erl @@ -30,6 +30,7 @@ tm_prepare/1, tm_retain/4, tm_retain/5, + tm_retain/6, tm_enter_pending/1, tm_enter_pending/3, tm_exit_pending/1 @@ -148,7 +149,6 @@ enter_still_pending([Tid | Tids], Tab) -> enter_still_pending([], _Tab) -> ok. - %% Looks up checkpoints for functions in mnesia_tm. tm_retain(Tid, Tab, Key, Op) -> case val({Tab, commit_work}) of @@ -157,11 +157,14 @@ tm_retain(Tid, Tab, Key, Op) -> _ -> undefined end. - + tm_retain(Tid, Tab, Key, Op, Checkpoints) -> + tm_retain(Tid, Tab, Key, Op, Checkpoints, '_'). + +tm_retain(Tid, Tab, Key, Op, Checkpoints, Obj) -> case Op of clear_table -> - OldRecs = mnesia_lib:db_match_object(Tab, '_'), + OldRecs = mnesia_lib:db_match_object(Tab, Obj), send_group_retain(OldRecs, Checkpoints, Tid, Tab, []), OldRecs; _ -> diff --git a/lib/mnesia/src/mnesia_dumper.erl b/lib/mnesia/src/mnesia_dumper.erl index e6b02b501a8c..2a70667d8615 100644 --- a/lib/mnesia/src/mnesia_dumper.erl +++ b/lib/mnesia/src/mnesia_dumper.erl @@ -404,8 +404,12 @@ dets_insert(Op,Tab,Key,Val, Storage0) -> dets_updated(Tab,Key), mnesia_lib:db_match_erase(Storage, Tab, Val); clear_table -> - dets_cleared(Tab), - ok = mnesia_lib:db_match_erase(Storage, Tab, '_') + %% Val is a match_delete pattern + case Val of + '_' -> dets_cleared(Tab); + _ -> dets_updated(Tab, Val) + end, + ok = mnesia_lib:db_match_erase(Storage, Tab, Val) end. dets_updated(Tab,Key) -> diff --git a/lib/mnesia/src/mnesia_log.erl b/lib/mnesia/src/mnesia_log.erl index b1514bfbe64e..519b8fac4828 100644 --- a/lib/mnesia/src/mnesia_log.erl +++ b/lib/mnesia/src/mnesia_log.erl @@ -1024,9 +1024,14 @@ add_recs([LogH|Rest], N) LogH#log_header.log_kind == dcl_log, LogH#log_header.log_version >= "1.0" -> add_recs(Rest, N); -add_recs([{{Tab, _Key}, _Val, clear_table} | Rest], N) -> +add_recs([{{Tab, _Key}, '_', clear_table} | Rest], N) -> Size = ets:info(Tab, size), true = ets:delete_all_objects(Tab), add_recs(Rest, N+Size); +add_recs([{{Tab, _Key}, Pattern, clear_table} | Rest], N) -> + SizeBefore = ets:info(Tab, size), + true = ets:match_delete(Tab, Pattern), + SizeAfter = ets:info(Tab, size), + add_recs(Rest, N+SizeBefore-SizeAfter); add_recs([], N) -> N. diff --git a/lib/mnesia/src/mnesia_subscr.erl b/lib/mnesia/src/mnesia_subscr.erl index d0c298e4253c..894fb3a0813a 100644 --- a/lib/mnesia/src/mnesia_subscr.erl +++ b/lib/mnesia/src/mnesia_subscr.erl @@ -152,7 +152,7 @@ report_table_event(Tab, Tid, Obj, Op) -> report_table_event(Subscr, Tab, Tid, Obj, Op) -> report_table_event(Subscr, Tab, Tid, Obj, Op, undefined). -report_table_event({subscribers, S1, S2}, Tab, Tid, _Obj, clear_table, _Old) -> +report_table_event({subscribers, S1, S2}, Tab, Tid, '_' = _Obj, clear_table, _Old) -> What = {delete, {schema, Tab}, Tid}, deliver(S1, {mnesia_table_event, What}), TabDef = mnesia_schema:cs2list(?catch_val({Tab, cstruct})), @@ -163,6 +163,15 @@ report_table_event({subscribers, S1, S2}, Tab, Tid, _Obj, clear_table, _Old) -> What4 = {write, schema, {schema, Tab, TabDef}, [], Tid}, deliver(S2, {mnesia_table_event, What4}); +report_table_event({subscribers, S1, _S2}, Tab, Tid, Obj, clear_table, _Old) -> + %% Obj is a match pattern here. + %% Sending delete_object event is compatible with `mnesia_loader`, + %% that uses `db_match_erase/2` which actually removes records by pattern. + %% Extended event is omitted: it's possible to match and get `OldRecords`, + %% but the list can be quite large. + Standard = {delete_object, patch_record(Tab, Obj), Tid}, + deliver(S1, {mnesia_table_event, Standard}); + report_table_event({subscribers, Subscr, []}, Tab, Tid, Obj, Op, _Old) -> What = {Op, patch_record(Tab, Obj), Tid}, deliver(Subscr, {mnesia_table_event, What}); diff --git a/lib/mnesia/src/mnesia_tm.erl b/lib/mnesia/src/mnesia_tm.erl index 53c67f5775ea..1676055bc4a5 100644 --- a/lib/mnesia/src/mnesia_tm.erl +++ b/lib/mnesia/src/mnesia_tm.erl @@ -1955,7 +1955,7 @@ commit_del_object([H|R], Tid, Storage, Tab, K, Obj) when element(1, H) == index commit_clear([], _, _, _, _, _) -> ok; commit_clear([{checkpoints, CpList}|R], Tid, Storage, Tab, K, Obj) -> - mnesia_checkpoint:tm_retain(Tid, Tab, K, clear_table, CpList), + mnesia_checkpoint:tm_retain(Tid, Tab, K, clear_table, CpList, Obj), commit_clear(R, Tid, Storage, Tab, K, Obj); commit_clear([H|R], Tid, Storage, Tab, K, Obj) when element(1, H) == subscribers -> diff --git a/lib/mnesia/test/Makefile b/lib/mnesia/test/Makefile index c3fbad88ca1b..ddd0a9f0d1a0 100644 --- a/lib/mnesia/test/Makefile +++ b/lib/mnesia/test/Makefile @@ -54,7 +54,8 @@ MODULES= \ mnesia_cost \ mnesia_dbn_meters \ ext_test \ - mnesia_index_plugin_test + mnesia_index_plugin_test \ + mnesia_match_delete_test DocExamplesDir := ../doc/src/ diff --git a/lib/mnesia/test/mnesia_SUITE.erl b/lib/mnesia/test/mnesia_SUITE.erl index 123d16023f11..1b1574ca08e2 100644 --- a/lib/mnesia/test/mnesia_SUITE.erl +++ b/lib/mnesia/test/mnesia_SUITE.erl @@ -70,7 +70,7 @@ groups() -> [{light, [], [{group, install}, {group, nice}, {group, evil}, {group, mnesia_frag_test, light}, {group, qlc}, {group, index_plugins}, - {group, registry}, {group, config}, {group, examples}]}, + {group, registry}, {group, config}, {group, examples}, {group, match_delete}]}, {install, [], [{mnesia_install_test, all}]}, {nice, [], [{mnesia_nice_coverage_test, all}]}, {evil, [], [{mnesia_evil_coverage_test, all}]}, @@ -79,6 +79,7 @@ groups() -> {registry, [], [{mnesia_registry_test, all}]}, {config, [], [{mnesia_config_test, all}]}, {examples, [], [{mnesia_examples_test, all}]}, + {match_delete, [], [{mnesia_match_delete_test, all}]}, %% The 'medium' test suite verfies the ACID (atomicity, consistency %% isolation and durability) properties and various recovery scenarios %% These tests may take quite while to run. diff --git a/lib/mnesia/test/mnesia_match_delete_test.erl b/lib/mnesia/test/mnesia_match_delete_test.erl new file mode 100644 index 000000000000..c77a260fb0b7 --- /dev/null +++ b/lib/mnesia/test/mnesia_match_delete_test.erl @@ -0,0 +1,217 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2023. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% +-module(mnesia_match_delete_test). +-include("mnesia_test_lib.hrl"). + +-export([all/0, groups/0, + init_per_group/2, end_per_group/2, + init_per_testcase/2, end_per_testcase/2]). + +-export([match_delete/1, + match_delete_checkpoint/1, + match_delete_subscribe/1, + match_delete_index/1, + match_delete_restart/1, + match_delete_dump_restart/1, + match_delete_frag/1]). + +all() -> + [match_delete, + match_delete_checkpoint, + match_delete_subscribe, + match_delete_index, + match_delete_restart, + match_delete_dump_restart, + match_delete_frag]. + +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + +init_per_testcase(Func, Conf) -> + mnesia_test_lib:init_per_testcase(Func, Conf). + +end_per_testcase(Func, Conf) -> + mnesia_test_lib:end_per_testcase(Func, Conf). + +match_delete(suite) -> []; +match_delete(Config) when is_list(Config) -> + [Node1, Node2, Node3] = Nodes = ?acquire_nodes(3, Config), + Tab = match_delete_tab, + Def = [{ram_copies, [Node1]}, {disc_copies, [Node2]}, {disc_only_copies, [Node3]}], + ?match({atomic, ok}, mnesia:create_table(Tab, Def)), + ?match({atomic, ok}, write(Tab)), + ?match({atomic, ok}, mnesia:match_delete(Tab, {Tab, '_', bar})), + ?match({atomic, [1,2,5]}, ?sort(mnesia:transaction(fun() -> mnesia:all_keys(Tab) end))), + ?verify_mnesia(Nodes, []). + +match_delete_checkpoint(suite) -> []; +match_delete_checkpoint(Config) when is_list(Config) -> + [Node1, Node2, Node3] = Nodes = ?acquire_nodes(3, Config), + Tab = match_delete_retain_tab, + Def = [{disc_copies, [Node1, Node2]}, {disc_only_copies, [Node3]}], + Checkpoint = ?FUNCTION_NAME, + ?match({atomic, ok}, mnesia:create_table(Tab, Def)), + ?match({atomic, ok}, write(Tab)), + + ?match({ok, Checkpoint, _}, mnesia:activate_checkpoint([{name, Checkpoint}, {max, [Tab]}])), + ?match({atomic, ok}, mnesia:match_delete(Tab, {Tab, '_', bar})), + ?match({atomic, [1,2,5]}, ?sort(mnesia:transaction(fun() -> mnesia:all_keys(Tab) end))), + + File = "match_delete_backup.BUP", + ?match(ok, mnesia:backup_checkpoint(Checkpoint, File)), + ?match(ok, mnesia:deactivate_checkpoint(?FUNCTION_NAME)), + + ?match({atomic, [Tab]}, mnesia:restore(File, [{default_op, clear_tables}])), + ?match({atomic, [1,2,3,4,5]}, ?sort(mnesia:transaction(fun() -> mnesia:all_keys(Tab) end))), + + ?match(ok, file:delete(File)), + ?verify_mnesia(Nodes, []). + +match_delete_subscribe(suite) -> []; +match_delete_subscribe(Config) when is_list(Config) -> + Nodes = ?acquire_nodes(3, Config), + Tab = match_delete_sub_tab, + Def = [{ram_copies, Nodes}], + ?match({atomic, ok}, mnesia:create_table(Tab, Def)), + ?match({atomic, ok}, write(Tab)), + Pattern = {Tab, '_', bar}, + ?match({ok, _}, mnesia:subscribe({table, Tab})), + ?match({atomic, ok}, mnesia:match_delete(Tab, Pattern)), + ?match_receive({mnesia_table_event, {delete_object, Pattern, _}}), + ?match({atomic, [1,2,5]}, ?sort(mnesia:transaction(fun() -> mnesia:all_keys(Tab) end))), + ?verify_mnesia(Nodes, []). + +match_delete_index(suite) -> []; +match_delete_index(Config) when is_list(Config) -> + Nodes = ?acquire_nodes(3, Config), + {atomic, ok} = mnesia:create_table(match_delete_index, + [{index, [ix]}, {attributes, [key, ix, val]}, + {disc_copies, Nodes}]), + {atomic, ok} = mnesia:create_table(match_delete_index_ram, + [{index, [ix]}, {attributes, [key, ix, val]}, + {ram_copies, Nodes}]), + {atomic, ok} = mnesia:create_table(match_delete_index_do, + [{index, [ix]}, {attributes, [key, ix, val]}, + {disc_only_copies, Nodes}]), + Test = fun(Tab) -> + Rec = {Tab, 1, 4, data}, + Rec2 = {Tab, 2, 5, data}, + Rec3 = {Tab, 3, 5, data}, + Rec4 = {Tab, 4, 6, data}, + Pattern = {Tab, '_', 5, '_'}, + + {atomic, ok} = mnesia:transaction(fun() -> mnesia:write(Rec), + mnesia:write(Rec2), + mnesia:write(Rec3), + mnesia:write(Rec4) + end), + + ?match({atomic, ok}, mnesia:match_delete(Tab, Pattern)), + + ?match([Rec], mnesia:dirty_index_read(Tab, 4, ix)), + ?match([Rec4], mnesia:dirty_index_read(Tab, 6, ix)), + ?match({atomic, [Rec]}, mnesia:transaction(fun() -> mnesia:index_read(Tab, 4, ix) end)), + ?match({atomic, [Rec4]}, mnesia:transaction(fun() -> mnesia:index_read(Tab, 6, ix) end)), + + ?match([], mnesia:dirty_index_match_object(Pattern, ix)), + ?match({atomic, []}, mnesia:transaction(fun() -> mnesia:index_match_object(Pattern, ix) end)), + + ?match([Rec], mnesia:dirty_index_match_object({Tab, '_', 4, '_'}, ix)), + ?match({atomic, [Rec4]}, + mnesia:transaction(fun() -> mnesia:index_match_object({Tab, '_', 6, data}, ix) end)) + end, + [Test(Tab) || Tab <- [match_delete_index, match_delete_index_ram, match_delete_index_do]], + ?verify_mnesia(Nodes, []). + +match_delete_restart(suite) -> []; +match_delete_restart(Config) when is_list(Config) -> + Nodes = ?acquire_nodes(1, Config), + Tab = match_delete_log_tab, + Def = [{disc_copies, Nodes}], + ?match({atomic, ok}, mnesia:create_table(Tab, Def)), + ?match({atomic, ok}, write(Tab)), + Pattern = {Tab, '_', bar}, + ?match({atomic, ok}, mnesia:match_delete(Tab, Pattern)), + %% Restart Mnesia right after calling match_delete/2 to verify that + %% the table is correctly loaded + ?match([], mnesia_test_lib:stop_mnesia(Nodes)), + ?match([], mnesia_test_lib:start_mnesia(Nodes, [Tab])), + ?match({atomic, [1,2,5]}, ?sort(mnesia:transaction(fun() -> mnesia:all_keys(Tab) end))), + ?verify_mnesia(Nodes, []). + +match_delete_dump_restart(suite) -> []; +match_delete_dump_restart(Config) when is_list(Config) -> + [Node1] = Nodes = ?acquire_nodes(1, Config), + Tab = match_delete_dump_tab, + Def = [{disc_copies, Nodes}], + ?match({atomic, ok}, mnesia:create_table(Tab, Def)), + ?match({atomic, ok}, write(Tab)), + Pattern = {Tab, '_', bar}, + ?match({atomic, ok}, mnesia:match_delete(Tab, Pattern)), + dumped = rpc:call(Node1, mnesia, dump_log, []), + ?match({atomic, [1,2,5]}, ?sort(mnesia:transaction(fun() -> mnesia:all_keys(Tab) end))), + ?match([], mnesia_test_lib:stop_mnesia(Nodes)), + ?match([], mnesia_test_lib:start_mnesia(Nodes, [Tab])), + ?match({atomic, [1,2,5]}, ?sort(mnesia:transaction(fun() -> mnesia:all_keys(Tab) end))), + ?verify_mnesia(Nodes, []). + +match_delete_frag(suite) -> []; +match_delete_frag(Config) when is_list(Config) -> + Nodes = ?acquire_nodes(2, Config), + Tab = match_delete_frag_tab, + FragProps = [{n_fragments, 2}, {node_pool, Nodes}], + Def = [{frag_properties, FragProps}, {ram_copies, Nodes}], + ?match({atomic, ok}, mnesia:create_table(Tab, Def)), + KVs = [{1, foo}, {2, foo}, + {3, bar}, {4, bar}, + {5, baz}, {6, baz}, + {7, foo}, {8, foo}], + ?match([ok, ok | _], frag_write(Tab, KVs)), + Pattern = {Tab, '_', bar}, + %% match_delete/2 is a transaction itself + ?match({atomic, ok}, + mnesia:activity( + async_dirty, fun(P) -> mnesia:match_delete(Tab, P) end, [Pattern], mnesia_frag) + ), + Keys = mnesia:activity(transaction, fun() -> mnesia:all_keys(Tab) end, [], mnesia_frag), + ?match([1,2,5,6,7,8], ?sort(Keys)), + ?verify_mnesia(Nodes, []). + +frag_write(Tab, KVs) -> + Fun = fun(KVs1) -> [mnesia:write(Tab, {Tab, K, V}, write) || {K, V} <- KVs1] end, + mnesia:activity(transaction, Fun, [KVs], mnesia_frag). + +write(Tab) -> + mnesia:transaction( + fun() -> + mnesia:write({Tab, 1, foo}), + mnesia:write({Tab, 2, foo}), + mnesia:write({Tab, 3, bar}), + mnesia:write({Tab, 4, bar}), + mnesia:write({Tab, 5, baz}) + end). From 6630e28bf49b387f885fec6acefd8cb8f397ea34 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Thu, 14 Dec 2023 23:29:30 +0200 Subject: [PATCH 25/28] chore: bump OTP_VERSION to 26.1.2-3 --- OTP_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OTP_VERSION b/OTP_VERSION index 12bd8af6dac2..709950307161 100644 --- a/OTP_VERSION +++ b/OTP_VERSION @@ -1 +1 @@ -26.1.2-2 +26.1.2-3 From 5848055e7b6bd16e2dfccf5273f08643be152f1f Mon Sep 17 00:00:00 2001 From: Ingela Anderton Andin Date: Wed, 3 Jan 2024 15:26:33 +0100 Subject: [PATCH 26/28] ssl: Fix legacy name handling in certificate request too Closes #7978 --- lib/ssl/src/ssl_handshake.erl | 3 ++- lib/ssl/test/tls_1_3_version_SUITE.erl | 27 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/lib/ssl/src/ssl_handshake.erl b/lib/ssl/src/ssl_handshake.erl index 97ee2d09e7ca..74d2d1d5b825 100644 --- a/lib/ssl/src/ssl_handshake.erl +++ b/lib/ssl/src/ssl_handshake.erl @@ -1704,7 +1704,7 @@ select_hashsign(#certificate_request{ hash_sign_algos = HashSigns}, certificate_types = Types}, Cert, - SupportedHashSigns, + SupportedHashSigns0, ?TLS_1_2) -> {SignAlgo0, Param, PublicKeyAlgo0, _, _} = get_cert_params(Cert), SignAlgo = sign_algo(SignAlgo0, Param), @@ -1712,6 +1712,7 @@ select_hashsign(#certificate_request{ case is_acceptable_cert_type(PublicKeyAlgo, Types) andalso is_supported_sign(SignAlgo, HashSigns) of true -> + SupportedHashSigns = ssl_cipher:signature_schemes_1_2(SupportedHashSigns0), do_select_hashsign(HashSigns, PublicKeyAlgo, SupportedHashSigns); false -> ?ALERT_REC(?FATAL, ?INSUFFICIENT_SECURITY, no_suitable_signature_algorithm) diff --git a/lib/ssl/test/tls_1_3_version_SUITE.erl b/lib/ssl/test/tls_1_3_version_SUITE.erl index df9a2496cbfa..31192db88610 100644 --- a/lib/ssl/test/tls_1_3_version_SUITE.erl +++ b/lib/ssl/test/tls_1_3_version_SUITE.erl @@ -54,6 +54,8 @@ tls12_client_tls_server/1, legacy_tls12_client_tls_server/0, legacy_tls12_client_tls_server/1, + legacy_tls12_server_tls_client/0, + legacy_tls12_server_tls_client/1, middle_box_tls13_client/0, middle_box_tls13_client/1, middle_box_tls12_enabled_client/0, @@ -93,6 +95,7 @@ tls_1_3_1_2_tests() -> tls_client_tls12_server, tls12_client_tls_server, legacy_tls12_client_tls_server, + legacy_tls12_server_tls_client, middle_box_tls13_client, middle_box_tls12_enabled_client, middle_box_client_tls_v2_session_reused, @@ -305,6 +308,30 @@ legacy_tls12_client_tls_server(Config) when is_list(Config) -> | ssl_test_lib:ssl_options(server_cert_opts, Config)], ssl_test_lib:basic_test(ClientOpts, ServerOpts, Config). +legacy_tls12_server_tls_client() -> + [{doc,"Test that a TLS 1.3 enabled client can connect to legacy TLS-1.2 server."}]. + +legacy_tls12_server_tls_client(Config) when is_list(Config) -> + SHA = sha384, + Prop = proplists:get_value(tc_group_properties, Config), + Alg = proplists:get_value(name, Prop), + #{client_config := ClientOpts0, + server_config := ServerOpts0} = ssl_test_lib:make_cert_chains_der(Alg, [{server_chain, + [[{digest, SHA}], + [{digest, SHA}], + [{digest, SHA}]]}, + {client_chain, + [[{digest, SHA}], + [{digest, SHA}], + [{digest, SHA}]]} + ]), + + ClientOpts = [{versions, ['tlsv1.3', 'tlsv1.2']} | ClientOpts0], + ServerOpts = [{versions, ['tlsv1.2']}, {verify, verify_peer}, {fail_if_no_peer_cert, true}, + {signature_algs, [{SHA, Alg}]} + | ServerOpts0], + ssl_test_lib:basic_test(ClientOpts, ServerOpts, Config). + middle_box_tls13_client() -> [{doc,"Test that a TLS 1.3 client can connect to a 1.3 server with and without middle box compatible mode."}]. middle_box_tls13_client(Config) when is_list(Config) -> From 26b68b6dc63c35f9caa6e68b06b08039d6f29a2a Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 10 Jan 2024 10:46:04 +0100 Subject: [PATCH 27/28] chore: bump to version 26.2.1-2 --- OTP_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OTP_VERSION b/OTP_VERSION index 59343a2c18b0..1ec37c8ddf08 100644 --- a/OTP_VERSION +++ b/OTP_VERSION @@ -1 +1 @@ -26.2.1-1 +26.2.1-2 From 89a171aab0ceaed8d85c7a9ed2eb7b8d81645c41 Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 6 Mar 2024 18:11:38 +0100 Subject: [PATCH 28/28] chore: bump emqx OTP version 26.2.2-1 --- OTP_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OTP_VERSION b/OTP_VERSION index 8217b64276a6..b01ed58d7783 100644 --- a/OTP_VERSION +++ b/OTP_VERSION @@ -1 +1 @@ -26.2.2 +26.2.2-1