diff --git a/.gitignore b/.gitignore index 3f5319cd..9afb3079 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +data/ greengrass-build/ integration/target/** +target/ +.gradle/ *.iml diff --git a/integration/pom.xml b/integration/pom.xml index c7c5af89..dc360d2a 100644 --- a/integration/pom.xml +++ b/integration/pom.xml @@ -88,7 +88,7 @@ io.moquette moquette-broker - 0.16-gg + 0.17-gg diff --git a/integration/src/main/java/com/aws/greengrass/mqtt/moquette/ClientDeviceAuthorizer.java b/integration/src/main/java/com/aws/greengrass/mqtt/moquette/ClientDeviceAuthorizer.java index 80a424ce..24a642bb 100644 --- a/integration/src/main/java/com/aws/greengrass/mqtt/moquette/ClientDeviceAuthorizer.java +++ b/integration/src/main/java/com/aws/greengrass/mqtt/moquette/ClientDeviceAuthorizer.java @@ -218,6 +218,11 @@ public void onConnectionLost(InterceptConnectionLostMessage msg) { closeAuthSession(msg.getClientID(), msg.getUsername()); } + @Override + public void onSessionLoopError(Throwable error) { + LOG.atWarn().log("Moquette session error", error); + } + private void closeAuthSession(String clientId, String username) { UserSessionPair sessionPair = getSessionForClient(clientId, username); if (sessionPair != null) { diff --git a/integration/src/main/java/com/aws/greengrass/mqtt/moquette/MQTTService.java b/integration/src/main/java/com/aws/greengrass/mqtt/moquette/MQTTService.java index 61c89773..f03944be 100644 --- a/integration/src/main/java/com/aws/greengrass/mqtt/moquette/MQTTService.java +++ b/integration/src/main/java/com/aws/greengrass/mqtt/moquette/MQTTService.java @@ -143,8 +143,14 @@ private synchronized void startWithProperties(Properties properties, boolean for IConfig config = new MemoryConfig(properties); ISslContextCreator sslContextCreator = new GreengrassMoquetteSslContextCreator(config, clientDeviceTrustManager); - mqttBroker.startServer(config, interceptHandlers, sslContextCreator, clientDeviceAuthorizer, - clientDeviceAuthorizer); + try { + mqttBroker.startServer(config, interceptHandlers, sslContextCreator, clientDeviceAuthorizer, + clientDeviceAuthorizer); + } catch (IOException e) { + // IO Exception can only be thrown from H2 right now and we do not configure moquette to use h2. + serviceErrored(e); + return; + } serverRunning = true; runningProperties = properties; } diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/config/IConfig.java b/moquette-0.16/broker/src/main/java/io/moquette/broker/config/IConfig.java deleted file mode 100644 index 3bd06b60..00000000 --- a/moquette-0.16/broker/src/main/java/io/moquette/broker/config/IConfig.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (c) 2012-2018 The original author or authors - * ------------------------------------------------------ - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Eclipse Public License v1.0 - * and Apache License v2.0 which accompanies this distribution. - * - * The Eclipse Public License is available at - * http://www.eclipse.org/legal/epl-v10.html - * - * The Apache License v2.0 is available at - * http://www.opensource.org/licenses/apache2.0.php - * - * You may elect to redistribute this code under either of these licenses. - */ - -package io.moquette.broker.config; - -import io.moquette.BrokerConstants; - -/** - * Base interface for all configuration implementations (filesystem, memory or classpath) - */ -public abstract class IConfig { - - public static final String DEFAULT_CONFIG = "config/moquette.conf"; - - public abstract void setProperty(String name, String value); - - /** - * Same semantic of Properties - * - * @param name property name. - * @return property value. - * */ - public abstract String getProperty(String name); - - /** - * Same semantic of Properties - * - * @param name property name. - * @param defaultValue default value to return in case the property doesn't exists. - * @return property value. - * */ - public abstract String getProperty(String name, String defaultValue); - - void assignDefaults() { - setProperty(BrokerConstants.PORT_PROPERTY_NAME, Integer.toString(BrokerConstants.PORT)); - setProperty(BrokerConstants.HOST_PROPERTY_NAME, BrokerConstants.HOST); - // setProperty(BrokerConstants.WEB_SOCKET_PORT_PROPERTY_NAME, - // Integer.toString(BrokerConstants.WEBSOCKET_PORT)); - setProperty(BrokerConstants.PASSWORD_FILE_PROPERTY_NAME, ""); - // setProperty(BrokerConstants.PERSISTENT_STORE_PROPERTY_NAME, - // BrokerConstants.DEFAULT_PERSISTENT_PATH); - setProperty(BrokerConstants.ALLOW_ANONYMOUS_PROPERTY_NAME, Boolean.TRUE.toString()); - setProperty(BrokerConstants.AUTHENTICATOR_CLASS_NAME, ""); - setProperty(BrokerConstants.AUTHORIZATOR_CLASS_NAME, ""); - setProperty(BrokerConstants.NETTY_MAX_BYTES_PROPERTY_NAME, - String.valueOf(BrokerConstants.DEFAULT_NETTY_MAX_BYTES_IN_MESSAGE)); - } - - public abstract IResourceLoader getResourceLoader(); - - public int intProp(String propertyName, int defaultValue) { - String propertyValue = getProperty(propertyName); - if (propertyValue == null) { - return defaultValue; - } - return Integer.parseInt(propertyValue); - } - - public boolean boolProp(String propertyName, boolean defaultValue) { - String propertyValue = getProperty(propertyName); - if (propertyValue == null) { - return defaultValue; - } - return Boolean.parseBoolean(propertyValue); - } -} diff --git a/moquette-0.17/.editorconfig b/moquette-0.17/.editorconfig new file mode 100644 index 00000000..3cd30306 --- /dev/null +++ b/moquette-0.17/.editorconfig @@ -0,0 +1,16 @@ +# EditorConfig is awesome: http://EditorConfig.org + +# top-most EditorConfig file +root = true + +# Unix-style newlines with a newline ending every file +[*] +end_of_line = lf +insert_final_newline = true +charset = utf-8 +indent_style = space +indent_size = 4 +max_line_length=120 + +[*.gradle] +indent_size = 2 diff --git a/moquette-0.17/.github/ISSUE_TEMPLATE.md b/moquette-0.17/.github/ISSUE_TEMPLATE.md new file mode 100644 index 00000000..9baa8e2a --- /dev/null +++ b/moquette-0.17/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,13 @@ +### Expected behavior + +### Actual behavior + +### Steps to reproduce + +### Minimal yet complete reproducer code (or URL to code) or complete log file + +### Moquette MQTT version + +### JVM version (e.g. `java -version`) + +### OS version (e.g. `uname -a`) diff --git a/moquette-0.17/.github/workflows/maven_build.yml b/moquette-0.17/.github/workflows/maven_build.yml new file mode 100644 index 00000000..4908fd9e --- /dev/null +++ b/moquette-0.17/.github/workflows/maven_build.yml @@ -0,0 +1,45 @@ +# This workflow will build a Java project with Maven +# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven + +name: Java CI with Maven + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-20.04, ubuntu-latest] + java: [11, 17] + arch: [x64] # when ARM will be present add aarch64 + fail-fast: false + max-parallel: 4 + name: Test JDK ${{ matrix.java }}, ${{ matrix.os }} + + steps: + - name: Check out repository + uses: actions/checkout@v2 + + - name: Cache maven repository + uses: actions/cache@v2 + with: + path: | + ~/.m2/repository + key: ${{ matrix.arch }}-${{ runner.os }}-maven-${{ matrix.java }}-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ matrix.arch }}-${{ runner.os }}-maven-${{ matrix.java }} + ${{ matrix.arch }}-${{ runner.os }}-maven + + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + architecture: ${{ matrix.arch }} + - name: Test with Maven + run: mvn verify -B --file pom.xml + diff --git a/moquette-0.17/.github/workflows/maven_central_release_version.yml b/moquette-0.17/.github/workflows/maven_central_release_version.yml new file mode 100644 index 00000000..882fd0a4 --- /dev/null +++ b/moquette-0.17/.github/workflows/maven_central_release_version.yml @@ -0,0 +1,32 @@ +# This workflow deploy the packages to Maven Central + +name: Build and ship to Maven Central + +#on: +# push: +# tags: +# - "v*" +on: + release: + types: [released] + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Maven Central Repository + uses: actions/setup-java@v3 + with: + java-version: 11 + server-id: ossrh + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + - name: Publish package + run: mvn --batch-mode -Prelease-sign-artifacts clean deploy + env: + MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }} + MAVEN_PASSWORD: ${{ secrets.OSSRH_TOKEN }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }} diff --git a/moquette-0.17/.github/workflows/maven_central_snapshot_version.yml b/moquette-0.17/.github/workflows/maven_central_snapshot_version.yml new file mode 100644 index 00000000..e1ebfd04 --- /dev/null +++ b/moquette-0.17/.github/workflows/maven_central_snapshot_version.yml @@ -0,0 +1,30 @@ +# This workflow deploy the snapshot packages to Maven Central + +name: Build and ship SNAPSHOT versions to Maven Central + +on: + push: + branches: [ main ] + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Maven Central Repository + uses: actions/setup-java@v1 + with: + java-version: 11 + server-id: ossrh + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + - name: Display settings.xml + run: cat /home/runner/.m2/settings.xml + - name: Publish package + run: mvn --batch-mode -Prelease-sign-artifacts clean deploy + env: + MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }} + MAVEN_PASSWORD: ${{ secrets.OSSRH_TOKEN }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }} diff --git a/moquette-0.17/.mvn/wrapper/MavenWrapperDownloader.java b/moquette-0.17/.mvn/wrapper/MavenWrapperDownloader.java new file mode 100644 index 00000000..b901097f --- /dev/null +++ b/moquette-0.17/.mvn/wrapper/MavenWrapperDownloader.java @@ -0,0 +1,117 @@ +/* + * Copyright 2007-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.net.*; +import java.io.*; +import java.nio.channels.*; +import java.util.Properties; + +public class MavenWrapperDownloader { + + private static final String WRAPPER_VERSION = "0.5.6"; + /** + * Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided. + */ + private static final String DEFAULT_DOWNLOAD_URL = "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/" + + WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar"; + + /** + * Path to the maven-wrapper.properties file, which might contain a downloadUrl property to + * use instead of the default one. + */ + private static final String MAVEN_WRAPPER_PROPERTIES_PATH = + ".mvn/wrapper/maven-wrapper.properties"; + + /** + * Path where the maven-wrapper.jar will be saved to. + */ + private static final String MAVEN_WRAPPER_JAR_PATH = + ".mvn/wrapper/maven-wrapper.jar"; + + /** + * Name of the property which should be used to override the default download url for the wrapper. + */ + private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl"; + + public static void main(String args[]) { + System.out.println("- Downloader started"); + File baseDirectory = new File(args[0]); + System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath()); + + // If the maven-wrapper.properties exists, read it and check if it contains a custom + // wrapperUrl parameter. + File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH); + String url = DEFAULT_DOWNLOAD_URL; + if(mavenWrapperPropertyFile.exists()) { + FileInputStream mavenWrapperPropertyFileInputStream = null; + try { + mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile); + Properties mavenWrapperProperties = new Properties(); + mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream); + url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url); + } catch (IOException e) { + System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'"); + } finally { + try { + if(mavenWrapperPropertyFileInputStream != null) { + mavenWrapperPropertyFileInputStream.close(); + } + } catch (IOException e) { + // Ignore ... + } + } + } + System.out.println("- Downloading from: " + url); + + File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH); + if(!outputFile.getParentFile().exists()) { + if(!outputFile.getParentFile().mkdirs()) { + System.out.println( + "- ERROR creating output directory '" + outputFile.getParentFile().getAbsolutePath() + "'"); + } + } + System.out.println("- Downloading to: " + outputFile.getAbsolutePath()); + try { + downloadFileFromURL(url, outputFile); + System.out.println("Done"); + System.exit(0); + } catch (Throwable e) { + System.out.println("- Error downloading"); + e.printStackTrace(); + System.exit(1); + } + } + + private static void downloadFileFromURL(String urlString, File destination) throws Exception { + if (System.getenv("MVNW_USERNAME") != null && System.getenv("MVNW_PASSWORD") != null) { + String username = System.getenv("MVNW_USERNAME"); + char[] password = System.getenv("MVNW_PASSWORD").toCharArray(); + Authenticator.setDefault(new Authenticator() { + @Override + protected PasswordAuthentication getPasswordAuthentication() { + return new PasswordAuthentication(username, password); + } + }); + } + URL website = new URL(urlString); + ReadableByteChannel rbc; + rbc = Channels.newChannel(website.openStream()); + FileOutputStream fos = new FileOutputStream(destination); + fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); + fos.close(); + rbc.close(); + } + +} diff --git a/moquette-0.17/.mvn/wrapper/maven-wrapper.jar b/moquette-0.17/.mvn/wrapper/maven-wrapper.jar new file mode 100644 index 00000000..2cc7d4a5 Binary files /dev/null and b/moquette-0.17/.mvn/wrapper/maven-wrapper.jar differ diff --git a/moquette-0.17/.mvn/wrapper/maven-wrapper.properties b/moquette-0.17/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 00000000..3745b114 --- /dev/null +++ b/moquette-0.17/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1,2 @@ +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.2/apache-maven-3.6.2-bin.zip +wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar diff --git a/moquette-0.16/ChangeLog.txt b/moquette-0.17/ChangeLog.txt similarity index 81% rename from moquette-0.16/ChangeLog.txt rename to moquette-0.17/ChangeLog.txt index 335b4fea..5feb4ecb 100644 --- a/moquette-0.16/ChangeLog.txt +++ b/moquette-0.17/ChangeLog.txt @@ -1,3 +1,19 @@ +Version 0.17-SNAPSHOT: + [feature] Introduced DSL FluentConfig to simplify Server's API instantiation #761. + [fix] resolved issue #633 of bad perfomance when adding many subscriptions to few topics, resolved in #758. + [fix] resolved issue #629 that originated from subscription trees wide and flat, resolved in #630 + [dependency] updated Netty to 4.1.93 and tcnative to 2.0.61 (#755) + [feature] add saved session expiry configurable through the `persistent_client_expiration` setting (#739). + [feature] implemented methods to forcibly disconnects a clients from a Server instance (#744). + [refactory] moved code to handle session event loops from PostOffice into separate SessionEventLoopGroup (#742). + [feature] added new callback method in interceptor to notify exceptions that happens on SessionEventLoop (#736). + [feature] introduce new `buffer_flush_millis` deprecating the old `immediate_buffer_flush` and switch default behavior to flush on every write (#738). + [refactory] purge of session state also on disconnect and reused logic (#715). + [feature] add `moquette.session_loop.debug` property to enable session loop checking assignments (#714). + [break] deprecate `persistent_store` to separate the enablement of persistence with `persistence_enabled` and the path `data_path` (#706). + [enhancement] introduced new queue implementation based on segments in memory mapped files. The type of queue implementation + could be selected by setting `persistent_queue_type` (#691, #704). + Version 0.16: [build] drop generation of broker-test, removed distribution and embedding_moquette modules from deploy phase (#616) [fix] introduces sessions event processors to segregate changes to a session in one single thread, simplifying concurrency and code (#631) diff --git a/moquette-0.16/README.md b/moquette-0.17/README.md similarity index 95% rename from moquette-0.16/README.md rename to moquette-0.17/README.md index 61311456..95aa83fd 100644 --- a/moquette-0.16/README.md +++ b/moquette-0.17/README.md @@ -46,13 +46,13 @@ Include dependency in your project: io.moquette moquette-broker - 0.15 + 0.17 ``` ## Build from sources After a git clone of the repository, cd into the cloned sources and: `./gradlew package`, at the end the distribution -package is present at `distribution/target/distribution-0.16-bundle.tar.gz` +package is present at `distribution/target/distribution-0.17-bundle.tar.gz` In distribution/target directory will be produced the selfcontained file for the broker with all dependencies and a running script. diff --git a/moquette-0.16/about.html b/moquette-0.17/about.html similarity index 100% rename from moquette-0.16/about.html rename to moquette-0.17/about.html diff --git a/moquette-0.16/broker/config/moquette.conf b/moquette-0.17/broker/config/moquette.conf similarity index 100% rename from moquette-0.16/broker/config/moquette.conf rename to moquette-0.17/broker/config/moquette.conf diff --git a/moquette-0.16/broker/config/password_file.conf b/moquette-0.17/broker/config/password_file.conf similarity index 100% rename from moquette-0.16/broker/config/password_file.conf rename to moquette-0.17/broker/config/password_file.conf diff --git a/moquette-0.17/broker/moquette_messages.log b/moquette-0.17/broker/moquette_messages.log new file mode 100644 index 00000000..e69de29b diff --git a/moquette-0.16/broker/pom.xml b/moquette-0.17/broker/pom.xml similarity index 95% rename from moquette-0.16/broker/pom.xml rename to moquette-0.17/broker/pom.xml index 86f05468..a2c78944 100644 --- a/moquette-0.16/broker/pom.xml +++ b/moquette-0.17/broker/pom.xml @@ -5,7 +5,7 @@ ../pom.xml moquette-parent io.moquette - 0.16-gg + 0.17-gg moquette-broker @@ -16,8 +16,8 @@ UTF-8 4.1.94.Final - 2.0.52.Final + https://github.com/netty/netty/blob/netty-4.1.93.Final/pom.xml#L625 --> + 2.0.61.Final 1.2.5 2.2.220 @@ -167,6 +167,12 @@ ${h2.version} test + + org.jetbrains + annotations + RELEASE + test + diff --git a/moquette-0.16/broker/src/main/java/io/moquette/BrokerConstants.java b/moquette-0.17/broker/src/main/java/io/moquette/BrokerConstants.java similarity index 56% rename from moquette-0.16/broker/src/main/java/io/moquette/BrokerConstants.java rename to moquette-0.17/broker/src/main/java/io/moquette/BrokerConstants.java index 89152abe..3982db6b 100644 --- a/moquette-0.16/broker/src/main/java/io/moquette/BrokerConstants.java +++ b/moquette-0.17/broker/src/main/java/io/moquette/BrokerConstants.java @@ -16,45 +16,74 @@ package io.moquette; +import io.moquette.broker.config.IConfig; + import java.io.File; public final class BrokerConstants { public static final String INTERCEPT_HANDLER_PROPERTY_NAME = "intercept.handler"; public static final String BROKER_INTERCEPTOR_THREAD_POOL_SIZE = "intercept.thread_pool.size"; + /** + * @deprecated use the DATA_PATH_PROPERTY_NAME to define the path where to store + * the broker files (es queues and subscriptions). + * Enable persistence with PERSISTENCE_ENABLED_PROPERTY_NAME + * */ + @Deprecated public static final String PERSISTENT_STORE_PROPERTY_NAME = "persistent_store"; + @Deprecated + public static final String DATA_PATH_PROPERTY_NAME = IConfig.DATA_PATH_PROPERTY_NAME; + @Deprecated + public static final String PERSISTENT_QUEUE_TYPE_PROPERTY_NAME = IConfig.PERSISTENT_QUEUE_TYPE_PROPERTY_NAME; + @Deprecated + public static final String PERSISTENCE_ENABLED_PROPERTY_NAME = IConfig.PERSISTENCE_ENABLED_PROPERTY_NAME; + public static final String SEGMENTED_QUEUE_PAGE_SIZE = "queue_page_size"; + public static final int MB = 1024 * 1024; + public static final int DEFAULT_SEGMENTED_QUEUE_PAGE_SIZE = 64 * MB; + public static final String SEGMENTED_QUEUE_SEGMENT_SIZE = "queue_segment_size"; + public static final int DEFAULT_SEGMENTED_QUEUE_SEGMENT_SIZE = 4 * MB; public static final String AUTOSAVE_INTERVAL_PROPERTY_NAME = "autosave_interval"; - public static final String PASSWORD_FILE_PROPERTY_NAME = "password_file"; - public static final String PORT_PROPERTY_NAME = "port"; - public static final String HOST_PROPERTY_NAME = "host"; + @Deprecated + public static final String PASSWORD_FILE_PROPERTY_NAME = IConfig.PASSWORD_FILE_PROPERTY_NAME; + @Deprecated // use IConfig.PORT_PROPERTY_NAME + public static final String PORT_PROPERTY_NAME = IConfig.PORT_PROPERTY_NAME; + @Deprecated + public static final String HOST_PROPERTY_NAME = IConfig.HOST_PROPERTY_NAME; public static final String DEFAULT_MOQUETTE_STORE_H2_DB_FILENAME = "moquette_store.h2"; public static final String DEFAULT_PERSISTENT_PATH = System.getProperty("user.dir") + File.separator + DEFAULT_MOQUETTE_STORE_H2_DB_FILENAME; - public static final String WEB_SOCKET_PORT_PROPERTY_NAME = "websocket_port"; - public static final String WSS_PORT_PROPERTY_NAME = "secure_websocket_port"; - public static final String WEB_SOCKET_PATH_PROPERTY_NAME = "websocket_path"; + @Deprecated + public static final String WEB_SOCKET_PORT_PROPERTY_NAME = IConfig.WEB_SOCKET_PORT_PROPERTY_NAME; + @Deprecated + public static final String WSS_PORT_PROPERTY_NAME = IConfig.WSS_PORT_PROPERTY_NAME; + @Deprecated + public static final String WEB_SOCKET_PATH_PROPERTY_NAME = IConfig.WEB_SOCKET_PATH_PROPERTY_NAME; public static final String WEB_SOCKET_MAX_FRAME_SIZE_PROPERTY_NAME = "websocket_max_frame_size"; - public static final String SESSION_QUEUE_SIZE = "session_queue_size"; - - /** - * Defines the SSL implementation to use, default to "JDK". - * @see io.netty.handler.ssl.SslProvider#name() - */ - public static final String SSL_PROVIDER = "ssl_provider"; - public static final String SSL_PORT_PROPERTY_NAME = "ssl_port"; - public static final String JKS_PATH_PROPERTY_NAME = "jks_path"; - - /** @see java.security.KeyStore#getInstance(String) for allowed types, default to "jks" */ - public static final String KEY_STORE_TYPE = "key_store_type"; - public static final String KEY_STORE_PASSWORD_PROPERTY_NAME = "key_store_password"; - public static final String KEY_MANAGER_PASSWORD_PROPERTY_NAME = "key_manager_password"; - public static final String ALLOW_ANONYMOUS_PROPERTY_NAME = "allow_anonymous"; + @Deprecated + public static final String SESSION_QUEUE_SIZE = IConfig.SESSION_QUEUE_SIZE; + @Deprecated + public static final String SSL_PROVIDER = IConfig.SSL_PROVIDER; + @Deprecated + public static final String SSL_PORT_PROPERTY_NAME = IConfig.SSL_PORT_PROPERTY_NAME; + @Deprecated + public static final String JKS_PATH_PROPERTY_NAME = IConfig.JKS_PATH_PROPERTY_NAME; + @Deprecated + public static final String KEY_STORE_TYPE = IConfig.KEY_STORE_TYPE; + @Deprecated + public static final String KEY_STORE_PASSWORD_PROPERTY_NAME = IConfig.KEY_STORE_PASSWORD_PROPERTY_NAME; + @Deprecated + public static final String KEY_MANAGER_PASSWORD_PROPERTY_NAME = IConfig.KEY_MANAGER_PASSWORD_PROPERTY_NAME; + @Deprecated + public static final String ALLOW_ANONYMOUS_PROPERTY_NAME = IConfig.ALLOW_ANONYMOUS_PROPERTY_NAME; public static final String PEER_CERTIFICATE_AS_USERNAME = "peer_certificate_as_username"; public static final String REAUTHORIZE_SUBSCRIPTIONS_ON_CONNECT = "reauthorize_subscriptions_on_connect"; public static final String ALLOW_ZERO_BYTE_CLIENT_ID_PROPERTY_NAME = "allow_zero_byte_client_id"; - public static final String ACL_FILE_PROPERTY_NAME = "acl_file"; - public static final String AUTHORIZATOR_CLASS_NAME = "authorizator_class"; - public static final String AUTHENTICATOR_CLASS_NAME = "authenticator_class"; + @Deprecated + public static final String ACL_FILE_PROPERTY_NAME = IConfig.ACL_FILE_PROPERTY_NAME; + @Deprecated + public static final String AUTHORIZATOR_CLASS_NAME = IConfig.AUTHORIZATOR_CLASS_NAME; + @Deprecated + public static final String AUTHENTICATOR_CLASS_NAME = IConfig.AUTHENTICATOR_CLASS_NAME; public static final String DB_AUTHENTICATOR_DRIVER = "authenticator.db.driver"; public static final String DB_AUTHENTICATOR_URL = "authenticator.db.url"; public static final String DB_AUTHENTICATOR_QUERY = "authenticator.db.query"; @@ -71,16 +100,28 @@ public final class BrokerConstants { public static final String NETTY_SO_KEEPALIVE_PROPERTY_NAME = "netty.so_keepalive"; public static final String NETTY_CHANNEL_TIMEOUT_SECONDS_PROPERTY_NAME = "netty.channel_timeout.seconds"; public static final String NETTY_EPOLL_PROPERTY_NAME = "netty.epoll"; - public static final String NETTY_MAX_BYTES_PROPERTY_NAME = "netty.mqtt.message_size"; - public static final int DEFAULT_NETTY_MAX_BYTES_IN_MESSAGE = 8092; + @Deprecated + public static final String NETTY_MAX_BYTES_PROPERTY_NAME = IConfig.NETTY_MAX_BYTES_PROPERTY_NAME; + @Deprecated + public static final int DEFAULT_NETTY_MAX_BYTES_IN_MESSAGE = IConfig.DEFAULT_NETTY_MAX_BYTES_IN_MESSAGE; public static final String NETTY_ENABLED_TLS_PROTOCOLS_PROPERTY_NAME = "netty.enabled.tls.protocols"; + /** + * @deprecated use the BUFFER_FLUSH_MS_PROPERTY_NAME + * */ + @Deprecated public static final String IMMEDIATE_BUFFER_FLUSH_PROPERTY_NAME = "immediate_buffer_flush"; + @Deprecated + public static final String BUFFER_FLUSH_MS_PROPERTY_NAME = IConfig.BUFFER_FLUSH_MS_PROPERTY_NAME; + public static final int NO_BUFFER_FLUSH = -1; + public static final int IMMEDIATE_BUFFER_FLUSH = 0; + public static final String METRICS_ENABLE_PROPERTY_NAME = "use_metrics"; public static final String METRICS_LIBRATO_EMAIL_PROPERTY_NAME = "metrics.librato.email"; public static final String METRICS_LIBRATO_TOKEN_PROPERTY_NAME = "metrics.librato.token"; public static final String METRICS_LIBRATO_SOURCE_PROPERTY_NAME = "metrics.librato.source"; - public static final String ENABLE_TELEMETRY_NAME = "telemetry_enabled"; + @Deprecated + public static final String ENABLE_TELEMETRY_NAME = IConfig.ENABLE_TELEMETRY_NAME; public static final String BUGSNAG_ENABLE_PROPERTY_NAME = "use_bugsnag"; public static final String BUGSNAG_TOKEN_PROPERTY_NAME = "bugsnag.token"; @@ -92,6 +133,9 @@ public final class BrokerConstants { public static final String NETTY_CHANNEL_READ_LIMIT_PROPERTY_NAME = "netty.channel.read.limit"; public static final int DEFAULT_NETTY_CHANNEL_READ_LIMIT_BYTES = 512 * 1024; + @Deprecated + public static final String PERSISTENT_CLIENT_EXPIRATION_PROPERTY_NAME = IConfig.PERSISTENT_CLIENT_EXPIRATION_PROPERTY_NAME; + public static final int FLIGHT_BEFORE_RESEND_MS = 5_000; public static final int INFLIGHT_WINDOW_SIZE = 10; diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/AbstractSessionMessageQueue.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/AbstractSessionMessageQueue.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/AbstractSessionMessageQueue.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/AbstractSessionMessageQueue.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/Authorizator.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/Authorizator.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/Authorizator.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/Authorizator.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/AutoFlushHandler.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/AutoFlushHandler.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/AutoFlushHandler.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/AutoFlushHandler.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/BrokerConfiguration.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/BrokerConfiguration.java similarity index 54% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/BrokerConfiguration.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/BrokerConfiguration.java index 929a7024..d4bc833d 100644 --- a/moquette-0.16/broker/src/main/java/io/moquette/broker/BrokerConfiguration.java +++ b/moquette-0.17/broker/src/main/java/io/moquette/broker/BrokerConfiguration.java @@ -18,35 +18,65 @@ import io.moquette.BrokerConstants; import io.moquette.broker.config.IConfig; +import java.util.Locale; + class BrokerConfiguration { private final boolean allowAnonymous; private final boolean allowZeroByteClientId; private final boolean reauthorizeSubscriptionsOnConnect; - private final boolean immediateBufferFlush; + private final int bufferFlushMillis; private final boolean peerCertificateAsUsername; BrokerConfiguration(IConfig props) { allowAnonymous = props.boolProp(BrokerConstants.ALLOW_ANONYMOUS_PROPERTY_NAME, true); allowZeroByteClientId = props.boolProp(BrokerConstants.ALLOW_ZERO_BYTE_CLIENT_ID_PROPERTY_NAME, false); reauthorizeSubscriptionsOnConnect = props.boolProp(BrokerConstants.REAUTHORIZE_SUBSCRIPTIONS_ON_CONNECT, false); - immediateBufferFlush = props.boolProp(BrokerConstants.IMMEDIATE_BUFFER_FLUSH_PROPERTY_NAME, false); peerCertificateAsUsername = props.boolProp(BrokerConstants.PEER_CERTIFICATE_AS_USERNAME, false); + + // BUFFER_FLUSH_MS_PROPERTY_NAME has precedence over the deprecated IMMEDIATE_BUFFER_FLUSH_PROPERTY_NAME + final String bufferFlushMillisProp = props.getProperty(BrokerConstants.BUFFER_FLUSH_MS_PROPERTY_NAME); + if (bufferFlushMillisProp != null && !bufferFlushMillisProp.isEmpty()) { + switch (bufferFlushMillisProp.toLowerCase(Locale.ROOT)) { + case "immediate": + bufferFlushMillis = BrokerConstants. IMMEDIATE_BUFFER_FLUSH; + break; + case "full": + bufferFlushMillis = BrokerConstants.NO_BUFFER_FLUSH; + break; + default: + final String errorMsg = String.format("Can't state value of %s property. Has to be 'immediate', " + + "'full' or a number >= -1, found %s", BrokerConstants.BUFFER_FLUSH_MS_PROPERTY_NAME, bufferFlushMillisProp); + try { + bufferFlushMillis = Integer.parseInt(bufferFlushMillisProp); + if (bufferFlushMillis < -1) { + throw new IllegalArgumentException(errorMsg); + } + } catch (NumberFormatException ex) { + throw new IllegalArgumentException(errorMsg); + } + } + } else { + if (props.boolProp(BrokerConstants.IMMEDIATE_BUFFER_FLUSH_PROPERTY_NAME, true)) { + bufferFlushMillis = BrokerConstants.IMMEDIATE_BUFFER_FLUSH; + } else { + bufferFlushMillis = BrokerConstants.NO_BUFFER_FLUSH; + } + } } public BrokerConfiguration(boolean allowAnonymous, boolean allowZeroByteClientId, - boolean reauthorizeSubscriptionsOnConnect, boolean immediateBufferFlush) { - this(allowAnonymous, allowZeroByteClientId, - reauthorizeSubscriptionsOnConnect, immediateBufferFlush, false); + boolean reauthorizeSubscriptionsOnConnect, int bufferFlushMillis) { + this(allowAnonymous, allowZeroByteClientId, reauthorizeSubscriptionsOnConnect, bufferFlushMillis, false); } public BrokerConfiguration(boolean allowAnonymous, boolean allowZeroByteClientId, - boolean reauthorizeSubscriptionsOnConnect, boolean immediateBufferFlush, + boolean reauthorizeSubscriptionsOnConnect, int bufferFlushMillis, boolean peerCertificateAsUsername) { this.allowAnonymous = allowAnonymous; this.allowZeroByteClientId = allowZeroByteClientId; this.reauthorizeSubscriptionsOnConnect = reauthorizeSubscriptionsOnConnect; - this.immediateBufferFlush = immediateBufferFlush; + this.bufferFlushMillis = bufferFlushMillis; this.peerCertificateAsUsername = peerCertificateAsUsername; } @@ -62,8 +92,8 @@ public boolean isReauthorizeSubscriptionsOnConnect() { return reauthorizeSubscriptionsOnConnect; } - public boolean isImmediateBufferFlush() { - return immediateBufferFlush; + public int getBufferFlushMillis() { + return bufferFlushMillis; } public boolean isPeerCertificateAsUsername() { diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/BugSnagErrorsHandler.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/BugSnagErrorsHandler.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/BugSnagErrorsHandler.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/BugSnagErrorsHandler.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/ClientDescriptor.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/ClientDescriptor.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/ClientDescriptor.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/ClientDescriptor.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/DebugUtils.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/DebugUtils.java similarity index 81% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/DebugUtils.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/DebugUtils.java index b49eb3c4..0fd667ef 100644 --- a/moquette-0.16/broker/src/main/java/io/moquette/broker/DebugUtils.java +++ b/moquette-0.17/broker/src/main/java/io/moquette/broker/DebugUtils.java @@ -23,7 +23,10 @@ public final class DebugUtils { public static String payload2Str(ByteBuf content) { - return content.copy().toString(StandardCharsets.UTF_8); + final int readerPin = content.readableBytes(); + final String result = content.toString(StandardCharsets.UTF_8); + content.readerIndex(readerPin); + return result; } private DebugUtils() { diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/DefaultMoquetteSslContextCreator.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/DefaultMoquetteSslContextCreator.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/DefaultMoquetteSslContextCreator.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/DefaultMoquetteSslContextCreator.java index 0853040f..e26c4d56 100644 --- a/moquette-0.16/broker/src/main/java/io/moquette/broker/DefaultMoquetteSslContextCreator.java +++ b/moquette-0.17/broker/src/main/java/io/moquette/broker/DefaultMoquetteSslContextCreator.java @@ -16,6 +16,15 @@ package io.moquette.broker; +import io.moquette.BrokerConstants; +import io.moquette.broker.config.IConfig; +import io.netty.handler.ssl.ClientAuth; +import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.SslContextBuilder; +import io.netty.handler.ssl.SslProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -32,17 +41,8 @@ import java.security.cert.X509Certificate; import java.util.Collections; import java.util.Objects; - -import io.moquette.BrokerConstants; -import io.moquette.broker.config.IConfig; -import io.netty.handler.ssl.ClientAuth; -import io.netty.handler.ssl.SslContext; -import io.netty.handler.ssl.SslContextBuilder; -import io.netty.handler.ssl.SslProvider; import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.TrustManagerFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Moquette integration implementation to load SSL certificate from local filesystem path configured in diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/IQueueRepository.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/IQueueRepository.java similarity index 91% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/IQueueRepository.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/IQueueRepository.java index 60b46948..854d32e1 100644 --- a/moquette-0.16/broker/src/main/java/io/moquette/broker/IQueueRepository.java +++ b/moquette-0.17/broker/src/main/java/io/moquette/broker/IQueueRepository.java @@ -1,6 +1,5 @@ package io.moquette.broker; -import java.util.Queue; import java.util.Set; public interface IQueueRepository { @@ -10,4 +9,6 @@ public interface IQueueRepository { boolean containsQueue(String clientId); SessionMessageQueue getOrCreateQueue(String clientId); + + void close(); } diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/IRetainedRepository.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/IRetainedRepository.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/IRetainedRepository.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/IRetainedRepository.java diff --git a/moquette-0.17/broker/src/main/java/io/moquette/broker/ISessionsRepository.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/ISessionsRepository.java new file mode 100644 index 00000000..3e3e252b --- /dev/null +++ b/moquette-0.17/broker/src/main/java/io/moquette/broker/ISessionsRepository.java @@ -0,0 +1,123 @@ +package io.moquette.broker; + +import io.netty.handler.codec.mqtt.MqttVersion; + +import java.time.Clock; +import java.time.Instant; +import java.util.Collection; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.Delayed; +import java.util.concurrent.TimeUnit; + +/** + * Used to store data about persisted sessions like MQTT version, session's properties. + * */ +public interface ISessionsRepository { + + // Data class + final class SessionData implements Delayed { + private final String clientId; + private Instant expireAt = null; + final MqttVersion version; + private final int expiryInterval; + private transient final Clock clock; + + /** + * Construct a new SessionData without expiration set yet. + * + * @param expiryInterval seconds after which the persistent session could be dropped. + * */ + public SessionData(String clientId, MqttVersion version, int expiryInterval, Clock clock) { + this.clientId = clientId; + this.clock = clock; + this.expiryInterval = expiryInterval; + this.version = version; + } + + /** + * Construct SessionData with an expiration instant, created by loading from the storage. + * + * @param expiryInterval seconds after which the persistent session could be dropped. + * */ + public SessionData(String clientId, Instant expireAt, MqttVersion version, int expiryInterval, Clock clock) { + Objects.requireNonNull(expireAt, "An expiration time is requested"); + this.clock = clock; + this.clientId = clientId; + this.expireAt = expireAt; + this.expiryInterval = expiryInterval; + this.version = version; + } + + public String clientId() { + return clientId; + } + + public MqttVersion protocolVersion() { + return version; + } + + public Optional expireAt() { + return Optional.ofNullable(expireAt); + } + + public Optional expiryInstant() { + return expireAt() + .map(Instant::toEpochMilli); + } + + public int expiryInterval() { + return expiryInterval; + } + + public SessionData withExpirationComputed() { + final Instant expiresAt = clock.instant().plusSeconds(expiryInterval); + return new SessionData(clientId, expiresAt, version, expiryInterval, clock); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + SessionData that = (SessionData) o; + return clientId.equals(that.clientId); + } + + @Override + public int hashCode() { + return Objects.hash(clientId); + } + + @Override + public String toString() { + return "SessionData{" + + "clientId='" + clientId + '\'' + + ", expireAt=" + expireAt + + ", version=" + version + + ", expiryInterval=" + expiryInterval + + '}'; + } + + @Override + public long getDelay(TimeUnit unit) { + return unit.convert(expireAt.toEpochMilli() - clock.millis(), TimeUnit.MILLISECONDS); + } + + @Override + public int compareTo(Delayed o) { + return Long.compare(getDelay(TimeUnit.MILLISECONDS), o.getDelay(TimeUnit.MILLISECONDS)); + } + } + + /** + * @return the full list of persisted sessions data. + * */ + Collection list(); + + /** + * Save data composing a session, es MQTT version, creation date and properties but not queues or subscriptions. + * */ + void saveSession(SessionData session); + + void delete(SessionData session); +} diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/ISslContextCreator.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/ISslContextCreator.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/ISslContextCreator.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/ISslContextCreator.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/ISubscriptionsRepository.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/ISubscriptionsRepository.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/ISubscriptionsRepository.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/ISubscriptionsRepository.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/InMemoryQueue.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/InMemoryQueue.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/InMemoryQueue.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/InMemoryQueue.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/InflightResender.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/InflightResender.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/InflightResender.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/InflightResender.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/MQTTConnection.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/MQTTConnection.java similarity index 85% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/MQTTConnection.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/MQTTConnection.java index 4a2e421a..d2e6c26e 100644 --- a/moquette-0.16/broker/src/main/java/io/moquette/broker/MQTTConnection.java +++ b/moquette-0.17/broker/src/main/java/io/moquette/broker/MQTTConnection.java @@ -15,41 +15,67 @@ */ package io.moquette.broker; -import io.moquette.broker.subscriptions.Topic; +import io.moquette.BrokerConstants; import io.moquette.broker.security.IAuthenticator; import io.moquette.broker.security.PemUtils; +import io.moquette.broker.subscriptions.Topic; import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufHolder; import io.netty.channel.Channel; import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelFutureListener; import io.netty.channel.ChannelPipeline; -import io.netty.handler.codec.mqtt.*; +import io.netty.handler.codec.mqtt.MqttConnAckMessage; +import io.netty.handler.codec.mqtt.MqttConnectMessage; +import io.netty.handler.codec.mqtt.MqttConnectPayload; +import io.netty.handler.codec.mqtt.MqttConnectReturnCode; +import io.netty.handler.codec.mqtt.MqttFixedHeader; +import io.netty.handler.codec.mqtt.MqttMessage; +import io.netty.handler.codec.mqtt.MqttMessageBuilders; +import io.netty.handler.codec.mqtt.MqttMessageIdVariableHeader; +import io.netty.handler.codec.mqtt.MqttMessageType; +import io.netty.handler.codec.mqtt.MqttPubAckMessage; +import io.netty.handler.codec.mqtt.MqttPublishMessage; +import io.netty.handler.codec.mqtt.MqttPublishVariableHeader; +import io.netty.handler.codec.mqtt.MqttQoS; +import io.netty.handler.codec.mqtt.MqttSubAckMessage; +import io.netty.handler.codec.mqtt.MqttSubscribeMessage; +import io.netty.handler.codec.mqtt.MqttUnsubAckMessage; +import io.netty.handler.codec.mqtt.MqttUnsubscribeMessage; +import io.netty.handler.codec.mqtt.MqttVersion; import io.netty.handler.ssl.SslHandler; import io.netty.handler.timeout.IdleStateHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import javax.net.ssl.SSLPeerUnverifiedException; import java.net.InetSocketAddress; import java.security.cert.Certificate; import java.security.cert.CertificateEncodingException; -import java.util.*; +import java.util.List; +import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import javax.net.ssl.SSLPeerUnverifiedException; import static io.netty.channel.ChannelFutureListener.CLOSE_ON_FAILURE; import static io.netty.channel.ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE; -import static io.netty.handler.codec.mqtt.MqttConnectReturnCode.*; +import static io.netty.handler.codec.mqtt.MqttConnectReturnCode.CONNECTION_ACCEPTED; +import static io.netty.handler.codec.mqtt.MqttConnectReturnCode.CONNECTION_REFUSED_BAD_USER_NAME_OR_PASSWORD; +import static io.netty.handler.codec.mqtt.MqttConnectReturnCode.CONNECTION_REFUSED_IDENTIFIER_REJECTED; +import static io.netty.handler.codec.mqtt.MqttConnectReturnCode.CONNECTION_REFUSED_SERVER_UNAVAILABLE; +import static io.netty.handler.codec.mqtt.MqttConnectReturnCode.CONNECTION_REFUSED_UNACCEPTABLE_PROTOCOL_VERSION; import static io.netty.handler.codec.mqtt.MqttMessageIdVariableHeader.from; -import static io.netty.handler.codec.mqtt.MqttQoS.*; +import static io.netty.handler.codec.mqtt.MqttQoS.AT_LEAST_ONCE; +import static io.netty.handler.codec.mqtt.MqttQoS.AT_MOST_ONCE; final class MQTTConnection { private static final Logger LOG = LoggerFactory.getLogger(MQTTConnection.class); + static final boolean sessionLoopDebug = Boolean.parseBoolean(System.getProperty("moquette.session_loop.debug", "false")); + final Channel channel; private final BrokerConfiguration brokerConfig; private final IAuthenticator authenticator; @@ -114,15 +140,19 @@ void handleMessage(MqttMessage msg) { private void processPubComp(MqttMessage msg) { final int messageID = ((MqttMessageIdVariableHeader) msg.variableHeader()).messageId(); - this.postOffice.routeCommand(bindedSession.getClientID(), "PUBCOMP", () -> { + final String clientID = bindedSession.getClientID(); + this.postOffice.routeCommand(clientID, "PUBCOMP", () -> { + checkMatchSessionLoop(clientID); bindedSession.processPubComp(messageID); - return bindedSession.getClientID(); + return clientID; }); } private void processPubRec(MqttMessage msg) { final int messageID = ((MqttMessageIdVariableHeader) msg.variableHeader()).messageId(); - this.postOffice.routeCommand(bindedSession.getClientID(), "PUBREC", () -> { + final String clientID = bindedSession.getClientID(); + this.postOffice.routeCommand(clientID, "PUBREC", () -> { + checkMatchSessionLoop(clientID); bindedSession.processPubRec(messageID); return null; }); @@ -137,6 +167,7 @@ private void processPubAck(MqttMessage msg) { final int messageID = ((MqttMessageIdVariableHeader) msg.variableHeader()).messageId(); final String clientId = getClientId(); this.postOffice.routeCommand(clientId, "PUB ACK", () -> { + checkMatchSessionLoop(clientId); bindedSession.pubAckReceived(messageID); return null; }); @@ -180,11 +211,23 @@ PostOffice.RouteResult processConnect(MqttConnectMessage msg) { final String sessionId = clientId; return postOffice.routeCommand(clientId, "CONN", () -> { + checkMatchSessionLoop(sessionId); executeConnect(msg, sessionId); return null; }); } + private void checkMatchSessionLoop(String clientId) { + if (!sessionLoopDebug) { + return; + } + final String currentThreadName = Thread.currentThread().getName(); + final String expectedThreadName = postOffice.sessionLoopThreadName(clientId); + if (!expectedThreadName.equals(currentThreadName)) { + throw new IllegalStateException("Expected to be executed on thread " + expectedThreadName + " but running on " + currentThreadName + ". This means a programming error"); + } + } + /** * Invoked by the Session's event loop. * */ @@ -200,6 +243,7 @@ private void executeConnect(MqttConnectMessage msg, String clientId) { abortConnection(CONNECTION_REFUSED_SERVER_UNAVAILABLE); return; } + NettyUtils.clientID(channel, clientId); final boolean msgCleanSessionFlag = msg.variableHeader().isCleanSession(); boolean isSessionAlreadyPresent = !msgCleanSessionFlag && result.alreadyStored; @@ -218,12 +262,15 @@ public void operationComplete(ChannelFuture future) throws Exception { channel.writeAndFlush(disconnectMsg).addListener(CLOSE); LOG.warn("CONNACK is sent but the session created can't transition in CONNECTED state"); } else { - NettyUtils.clientID(channel, clientIdUsed); connected = true; // OK continue with sending queued messages and normal flow if (result.mode == SessionRegistry.CreationModeEnum.REOPEN_EXISTING) { - result.session.sendQueuedMessagesWhileOffline(); + final Session session = result.session; + postOffice.routeCommand(session.getClientID(), "sendOfflineMessages", () -> { + session.sendQueuedMessagesWhileOffline(); + return null; + }); } initializeKeepAliveTimeout(channel, msg, clientIdUsed); @@ -233,8 +280,7 @@ public void operationComplete(ChannelFuture future) throws Exception { LOG.trace("dispatch connection: {}", msg); } } else { - bindedSession.disconnect(); - sessionRegistry.remove(bindedSession.getClientID()); + sessionRegistry.connectionClosed(bindedSession); LOG.error("CONNACK send failed, cleanup session and close the connection", future.cause()); channel.close(); } @@ -331,6 +377,7 @@ void handleConnectionLost() { // this must not be done on the netty thread LOG.debug("Notifying connection lost event"); postOffice.routeCommand(clientID, "CONN LOST", () -> { + checkMatchSessionLoop(clientID); if (isBoundToSession() || isSessionUnbound()) { LOG.debug("Cleaning {}", clientID); processConnectionLost(clientID); @@ -341,17 +388,16 @@ void handleConnectionLost() { }); } + // Invoked when a TCP connection drops and not when a client send DISCONNECT and close. private void processConnectionLost(String clientID) { if (bindedSession.hasWill()) { postOffice.fireWill(bindedSession.getWill()); } - if (bindedSession.isClean()) { - LOG.debug("Remove session for client {}", clientID); - sessionRegistry.remove(bindedSession.getClientID()); - } else { - bindedSession.disconnect(); + if (bindedSession.connected()) { + LOG.debug("Closing session on connectionLost {}", clientID); + sessionRegistry.connectionClosed(bindedSession); + connected = false; } - connected = false; //dispatch connection lost to intercept. String userName = NettyUtils.userName(channel); postOffice.dispatchConnectionLost(clientID,userName); @@ -375,11 +421,13 @@ PostOffice.RouteResult processDisconnect(MqttMessage msg) { } return this.postOffice.routeCommand(clientID, "DISCONN", () -> { + checkMatchSessionLoop(clientID); if (!isBoundToSession()) { LOG.debug("NOT processing disconnect {}, not bound.", clientID); return null; } - bindedSession.disconnect(); + LOG.debug("Closing session on disconnect {}", clientID); + sessionRegistry.connectionClosed(bindedSession); connected = false; channel.close().addListener(FIRE_EXCEPTION_ON_FAILURE); String userName = NettyUtils.userName(channel); @@ -398,6 +446,7 @@ PostOffice.RouteResult processSubscribe(MqttSubscribeMessage msg) { } final String username = NettyUtils.userName(channel); return postOffice.routeCommand(clientID, "SUB", () -> { + checkMatchSessionLoop(clientID); if (isBoundToSession()) postOffice.subscribeClientToTopics(msg, clientID, username, this); return null; @@ -415,6 +464,7 @@ private void processUnsubscribe(MqttUnsubscribeMessage msg) { final int messageId = msg.variableHeader().messageId(); postOffice.routeCommand(clientID, "UNSUB", () -> { + checkMatchSessionLoop(clientID); if (!isBoundToSession()) return null; LOG.trace("Processing UNSUBSCRIBE message. topics: {}", topics); @@ -452,6 +502,7 @@ PostOffice.RouteResult processPublish(MqttPublishMessage msg) { switch (qos) { case AT_MOST_ONCE: return postOffice.routeCommand(clientId, "PUB QoS0", () -> { + checkMatchSessionLoop(clientId); if (!isBoundToSession()) return null; postOffice.receivedPublishQos0(topic, username, clientId, msg); @@ -459,6 +510,7 @@ PostOffice.RouteResult processPublish(MqttPublishMessage msg) { }).ifFailed(msg::release); case AT_LEAST_ONCE: return postOffice.routeCommand(clientId, "PUB QoS1", () -> { + checkMatchSessionLoop(clientId); if (!isBoundToSession()) return null; postOffice.receivedPublishQos1(this, topic, username, messageID, msg); @@ -466,6 +518,7 @@ PostOffice.RouteResult processPublish(MqttPublishMessage msg) { }).ifFailed(msg::release); case EXACTLY_ONCE: { final PostOffice.RouteResult firstStepResult = postOffice.routeCommand(clientId, "PUB QoS2", () -> { + checkMatchSessionLoop(clientId); if (!isBoundToSession()) return null; bindedSession.receivedPublishQos2(messageID, msg); @@ -497,7 +550,9 @@ void sendPubRec(int messageID) { private void processPubRel(MqttMessage msg) { final int messageID = ((MqttMessageIdVariableHeader) msg.variableHeader()).messageId(); - postOffice.routeCommand(bindedSession.getClientID(), "PUBREL", () -> { + final String clientID = bindedSession.getClientID(); + postOffice.routeCommand(clientID, "PUBREL", () -> { + checkMatchSessionLoop(clientID); executePubRel(messageID); return null; }); @@ -535,7 +590,7 @@ void sendIfWritableElseDrop(MqttMessage msg) { } ChannelFuture channelFuture; - if (brokerConfig.isImmediateBufferFlush()) { + if (brokerConfig.getBufferFlushMillis() == BrokerConstants.IMMEDIATE_BUFFER_FLUSH) { channelFuture = channel.writeAndFlush(retainedDup); } else { channelFuture = channel.write(retainedDup); @@ -547,7 +602,10 @@ void sendIfWritableElseDrop(MqttMessage msg) { public void writabilityChanged() { if (channel.isWritable()) { LOG.debug("Channel is again writable"); - bindedSession.writabilityChanged(); + postOffice.routeCommand(getClientId(), "writabilityChanged", () -> { + bindedSession.writabilityChanged(); + return null; + }); } } @@ -632,10 +690,17 @@ public void readCompleted() { LOG.debug("readCompleted client CId: {}", getClientId()); if (getClientId() != null) { // TODO drain all messages in target's session in-flight message queue - bindedSession.flushAllQueuedMessages(); + queueDrainQueueCommand(); } } + private void queueDrainQueueCommand() { + postOffice.routeCommand(getClientId(), "flushQueues", () -> { + bindedSession.flushAllQueuedMessages(); + return null; + }); + } + public void flush() { channel.flush(); } diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/MQTTConnectionFactory.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/MQTTConnectionFactory.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/MQTTConnectionFactory.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/MQTTConnectionFactory.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/MemoryQueueRepository.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/MemoryQueueRepository.java similarity index 93% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/MemoryQueueRepository.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/MemoryQueueRepository.java index cf8346d9..539dc5a5 100644 --- a/moquette-0.16/broker/src/main/java/io/moquette/broker/MemoryQueueRepository.java +++ b/moquette-0.17/broker/src/main/java/io/moquette/broker/MemoryQueueRepository.java @@ -30,6 +30,11 @@ public SessionMessageQueue getOrCreateQueue(Str return queue; } + @Override + public void close() { + queues.clear(); + } + void dropQueue(String queueName) { queues.remove(queueName); } diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/MemoryRetainedRepository.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/MemoryRetainedRepository.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/MemoryRetainedRepository.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/MemoryRetainedRepository.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/MoquetteIdleTimeoutHandler.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/MoquetteIdleTimeoutHandler.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/MoquetteIdleTimeoutHandler.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/MoquetteIdleTimeoutHandler.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/NettyUtils.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/NettyUtils.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/NettyUtils.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/NettyUtils.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/NewNettyAcceptor.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/NewNettyAcceptor.java similarity index 88% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/NewNettyAcceptor.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/NewNettyAcceptor.java index df938304..dd6c1b7b 100644 --- a/moquette-0.16/broker/src/main/java/io/moquette/broker/NewNettyAcceptor.java +++ b/moquette-0.17/broker/src/main/java/io/moquette/broker/NewNettyAcceptor.java @@ -18,10 +18,25 @@ import io.moquette.BrokerConstants; import io.moquette.broker.config.IConfig; -import io.moquette.broker.metrics.*; +import io.moquette.broker.metrics.BytesMetrics; +import io.moquette.broker.metrics.BytesMetricsCollector; +import io.moquette.broker.metrics.BytesMetricsHandler; +import io.moquette.broker.metrics.DropWizardMetricsHandler; +import io.moquette.broker.metrics.MQTTMessageLogger; +import io.moquette.broker.metrics.MessageMetrics; +import io.moquette.broker.metrics.MessageMetricsCollector; +import io.moquette.broker.metrics.MessageMetricsHandler; import io.netty.bootstrap.ServerBootstrap; import io.netty.buffer.ByteBuf; -import io.netty.channel.*; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandler; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.ChannelPipeline; +import io.netty.channel.EventLoopGroup; import io.netty.channel.epoll.EpollEventLoopGroup; import io.netty.channel.epoll.EpollServerSocketChannel; import io.netty.channel.nio.NioEventLoopGroup; @@ -46,7 +61,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.net.ssl.SSLEngine; import java.net.BindException; import java.net.InetSocketAddress; import java.net.SocketAddress; @@ -55,8 +69,16 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.TimeUnit; +import javax.net.ssl.SSLEngine; -import static io.moquette.BrokerConstants.*; +import static io.moquette.BrokerConstants.BUGSNAG_ENABLE_PROPERTY_NAME; +import static io.moquette.BrokerConstants.DISABLED_PORT_BIND; +import static io.moquette.BrokerConstants.IMMEDIATE_BUFFER_FLUSH; +import static io.moquette.BrokerConstants.METRICS_ENABLE_PROPERTY_NAME; +import static io.moquette.BrokerConstants.PORT_PROPERTY_NAME; +import static io.moquette.BrokerConstants.SSL_PORT_PROPERTY_NAME; +import static io.moquette.BrokerConstants.WEB_SOCKET_PORT_PROPERTY_NAME; +import static io.moquette.BrokerConstants.WSS_PORT_PROPERTY_NAME; import static io.netty.channel.ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE; class NewNettyAcceptor { @@ -139,7 +161,7 @@ public void operationComplete(ChannelFuture future) throws Exception { private int trafficMaxWriteBytesPerSecondPerChannel; private Class channelClass; - public void initialize(NewNettyMQTTHandler mqttHandler, IConfig props, ISslContextCreator sslCtxCreator) { + public void initialize(NewNettyMQTTHandler mqttHandler, IConfig props, ISslContextCreator sslCtxCreator, BrokerConfiguration brokerConfiguration) { LOG.debug("Initializing Netty acceptor"); nettySoBacklog = props.intProp(BrokerConstants.NETTY_SO_BACKLOG_PROPERTY_NAME, 128); @@ -153,6 +175,7 @@ public void initialize(NewNettyMQTTHandler mqttHandler, IConfig props, ISslConte BrokerConstants.DEFAULT_NETTY_CHANNEL_READ_LIMIT_BYTES), 0); trafficMaxWriteBytesPerSecondPerChannel = Math.max(props.intProp(BrokerConstants.NETTY_CHANNEL_WRITE_LIMIT_PROPERTY_NAME, BrokerConstants.DEFAULT_NETTY_CHANNEL_WRITE_LIMIT_BYTES), 0); + boolean epoll = props.boolProp(BrokerConstants.NETTY_EPOLL_PROPERTY_NAME, false); if (epoll) { LOG.info("Netty is using Epoll"); @@ -183,16 +206,16 @@ public void initialize(NewNettyMQTTHandler mqttHandler, IConfig props, ISslConte } else { this.errorsCather = Optional.empty(); } - initializePlainTCPTransport(mqttHandler, props); - initializeWebSocketTransport(mqttHandler, props); + initializePlainTCPTransport(mqttHandler, props, brokerConfiguration); + initializeWebSocketTransport(mqttHandler, props, brokerConfiguration); if (securityPortsConfigured(props)) { SslContext sslContext = sslCtxCreator.initSSLContext(); if (sslContext == null) { LOG.error("Can't initialize SSLHandler layer! Exiting, check your configuration of jks"); return; } - initializeSSLTCPTransport(mqttHandler, props, sslContext); - initializeWSSTransport(mqttHandler, props, sslContext); + initializeSSLTCPTransport(mqttHandler, props, sslContext, brokerConfiguration); + initializeWSSTransport(mqttHandler, props, sslContext, brokerConfiguration); } } @@ -244,7 +267,7 @@ public int getSslPort() { return ports.computeIfAbsent(SSL_MQTT_PROTO, i -> 0); } - private void initializePlainTCPTransport(NewNettyMQTTHandler handler, IConfig props) { + private void initializePlainTCPTransport(NewNettyMQTTHandler handler, IConfig props, BrokerConfiguration brokerConfiguration) { LOG.debug("Configuring TCP MQTT transport"); final MoquetteIdleTimeoutHandler timeoutHandler = new MoquetteIdleTimeoutHandler(); String host = props.getProperty(BrokerConstants.HOST_PROPERTY_NAME); @@ -260,18 +283,19 @@ private void initializePlainTCPTransport(NewNettyMQTTHandler handler, IConfig pr } int port = Integer.parseInt(tcpPortProp); + final int writeFlushMillis = brokerConfiguration.getBufferFlushMillis(); initFactory(host, port, PLAIN_MQTT_PROTO, new PipelineInitializer() { @Override void init(SocketChannel channel) { ChannelPipeline pipeline = channel.pipeline(); - configureMQTTPipeline(pipeline, timeoutHandler, handler); + configureMQTTPipeline(pipeline, timeoutHandler, handler, writeFlushMillis); } }); } private void configureMQTTPipeline(ChannelPipeline pipeline, MoquetteIdleTimeoutHandler timeoutHandler, - NewNettyMQTTHandler handler) { + NewNettyMQTTHandler handler, int writeFlushMillis) { pipeline.addFirst("idleStateHandler", new IdleStateHandler(nettyChannelTimeoutSeconds, 0, 0)); pipeline.addAfter("idleStateHandler", "idleEventHandler", timeoutHandler); // pipeline.addLast("logger", new LoggingHandler("Netty", LogLevel.ERROR)); @@ -279,13 +303,13 @@ private void configureMQTTPipeline(ChannelPipeline pipeline, MoquetteIdleTimeout pipeline.addLast("bugsnagCatcher", errorsCather.get()); } pipeline.addFirst("bytemetrics", new BytesMetricsHandler(bytesMetricsCollector)); - pipeline.addLast("autoflush", new AutoFlushHandler(1, TimeUnit.SECONDS)); - + if (writeFlushMillis > IMMEDIATE_BUFFER_FLUSH) { + pipeline.addLast("autoflush", new AutoFlushHandler(writeFlushMillis, TimeUnit.MILLISECONDS)); + } if (trafficMaxReadBytesPerSecondPerChannel > 0 || trafficMaxWriteBytesPerSecondPerChannel > 0) { pipeline.addLast("trafficShaping", new ChannelTrafficShapingHandler(trafficMaxWriteBytesPerSecondPerChannel, trafficMaxReadBytesPerSecondPerChannel, TimeUnit.SECONDS.toMillis(1))); } - pipeline.addLast("decoder", new MqttDecoder(maxBytesInMessage)); pipeline.addLast("encoder", MqttEncoder.INSTANCE); pipeline.addLast("metrics", new MessageMetricsHandler(metricsCollector)); @@ -296,7 +320,7 @@ private void configureMQTTPipeline(ChannelPipeline pipeline, MoquetteIdleTimeout pipeline.addLast("handler", handler); } - private void initializeWebSocketTransport(final NewNettyMQTTHandler handler, IConfig props) { + private void initializeWebSocketTransport(final NewNettyMQTTHandler handler, IConfig props, BrokerConfiguration brokerConfiguration) { LOG.debug("Configuring Websocket MQTT transport"); String webSocketPortProp = props.getProperty(WEB_SOCKET_PORT_PROPERTY_NAME, DISABLED_PORT_BIND); if (DISABLED_PORT_BIND.equals(webSocketPortProp)) { @@ -312,6 +336,7 @@ private void initializeWebSocketTransport(final NewNettyMQTTHandler handler, ICo String host = props.getProperty(BrokerConstants.HOST_PROPERTY_NAME); String path = props.getProperty(BrokerConstants.WEB_SOCKET_PATH_PROPERTY_NAME, BrokerConstants.WEBSOCKET_PATH); int maxFrameSize = props.intProp(BrokerConstants.WEB_SOCKET_MAX_FRAME_SIZE_PROPERTY_NAME, 65536); + final int writeFlushMillis = brokerConfiguration.getBufferFlushMillis(); initFactory(host, port, "Websocket MQTT", new PipelineInitializer() { @Override @@ -323,12 +348,12 @@ void init(SocketChannel channel) { new WebSocketServerProtocolHandler(path, MQTT_SUBPROTOCOL_CSV_LIST, false, maxFrameSize)); pipeline.addLast("ws2bytebufDecoder", new WebSocketFrameToByteBufDecoder()); pipeline.addLast("bytebuf2wsEncoder", new ByteBufToWebSocketFrameEncoder()); - configureMQTTPipeline(pipeline, timeoutHandler, handler); + configureMQTTPipeline(pipeline, timeoutHandler, handler, writeFlushMillis); } }); } - private void initializeSSLTCPTransport(NewNettyMQTTHandler handler, IConfig props, SslContext sslContext) { + private void initializeSSLTCPTransport(NewNettyMQTTHandler handler, IConfig props, SslContext sslContext, BrokerConfiguration brokerConfiguration) { LOG.debug("Configuring SSL MQTT transport"); String sslPortProp = props.getProperty(SSL_PORT_PROPERTY_NAME, DISABLED_PORT_BIND); if (DISABLED_PORT_BIND.equals(sslPortProp)) { @@ -345,18 +370,19 @@ private void initializeSSLTCPTransport(NewNettyMQTTHandler handler, IConfig prop String host = props.getProperty(BrokerConstants.HOST_PROPERTY_NAME); String sNeedsClientAuth = props.getProperty(BrokerConstants.NEED_CLIENT_AUTH, "false"); final boolean needsClientAuth = Boolean.valueOf(sNeedsClientAuth); + final int writeFlushMillis = brokerConfiguration.getBufferFlushMillis(); initFactory(host, sslPort, SSL_MQTT_PROTO, new PipelineInitializer() { @Override void init(SocketChannel channel) throws Exception { ChannelPipeline pipeline = channel.pipeline(); pipeline.addLast("ssl", createSslHandler(channel, sslContext, needsClientAuth)); - configureMQTTPipeline(pipeline, timeoutHandler, handler); + configureMQTTPipeline(pipeline, timeoutHandler, handler, writeFlushMillis); } }); } - private void initializeWSSTransport(NewNettyMQTTHandler handler, IConfig props, SslContext sslContext) { + private void initializeWSSTransport(NewNettyMQTTHandler handler, IConfig props, SslContext sslContext, BrokerConfiguration brokerConfiguration) { LOG.debug("Configuring secure websocket MQTT transport"); String sslPortProp = props.getProperty(WSS_PORT_PROPERTY_NAME, DISABLED_PORT_BIND); if (DISABLED_PORT_BIND.equals(sslPortProp)) { @@ -372,6 +398,7 @@ private void initializeWSSTransport(NewNettyMQTTHandler handler, IConfig props, int maxFrameSize = props.intProp(BrokerConstants.WEB_SOCKET_MAX_FRAME_SIZE_PROPERTY_NAME, 65536); String sNeedsClientAuth = props.getProperty(BrokerConstants.NEED_CLIENT_AUTH, "false"); final boolean needsClientAuth = Boolean.valueOf(sNeedsClientAuth); + final int writeFlushMillis = brokerConfiguration.getBufferFlushMillis(); initFactory(host, sslPort, "Secure websocket", new PipelineInitializer() { @Override @@ -386,7 +413,7 @@ void init(SocketChannel channel) throws Exception { pipeline.addLast("ws2bytebufDecoder", new WebSocketFrameToByteBufDecoder()); pipeline.addLast("bytebuf2wsEncoder", new ByteBufToWebSocketFrameEncoder()); - configureMQTTPipeline(pipeline, timeoutHandler, handler); + configureMQTTPipeline(pipeline, timeoutHandler, handler, writeFlushMillis); } }); } diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/NewNettyMQTTHandler.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/NewNettyMQTTHandler.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/NewNettyMQTTHandler.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/NewNettyMQTTHandler.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/PostOffice.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/PostOffice.java similarity index 90% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/PostOffice.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/PostOffice.java index b432b269..fa2631f2 100644 --- a/moquette-0.16/broker/src/main/java/io/moquette/broker/PostOffice.java +++ b/moquette-0.17/broker/src/main/java/io/moquette/broker/PostOffice.java @@ -33,7 +33,10 @@ import java.util.List; import java.util.Objects; import java.util.Set; -import java.util.concurrent.*; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -177,30 +180,18 @@ public RouteResult ifFailed(Runnable action) { private final IRetainedRepository retainedRepository; private SessionRegistry sessionRegistry; private BrokerInterceptor interceptor; - - private final Thread[] sessionExecutors; - private final BlockingQueue>[] sessionQueues; - private final int eventLoops = Runtime.getRuntime().availableProcessors(); private final FailedPublishCollection failedPublishes = new FailedPublishCollection(); + private final SessionEventLoopGroup sessionLoops; PostOffice(ISubscriptionsDirectory subscriptions, IRetainedRepository retainedRepository, - SessionRegistry sessionRegistry, BrokerInterceptor interceptor, Authorizator authorizator, int sessionQueueSize) { + SessionRegistry sessionRegistry, BrokerInterceptor interceptor, Authorizator authorizator, + SessionEventLoopGroup sessionLoops) { this.authorizator = authorizator; this.subscriptions = subscriptions; this.retainedRepository = retainedRepository; this.sessionRegistry = sessionRegistry; this.interceptor = interceptor; - - this.sessionQueues = new BlockingQueue[eventLoops]; - for (int i = 0; i < eventLoops; i++) { - this.sessionQueues[i] = new ArrayBlockingQueue<>(sessionQueueSize); - } - this.sessionExecutors = new Thread[eventLoops]; - for (int i = 0; i < eventLoops; i++) { - this.sessionExecutors[i] = new Thread(new SessionEventLoop(this.sessionQueues[i])); - this.sessionExecutors[i].setName("Session Executor " + i); - this.sessionExecutors[i].start(); - } + this.sessionLoops = sessionLoops; } public void init(SessionRegistry sessionRegistry) { @@ -402,10 +393,12 @@ private RoutingResults publish2Subscribers(ByteBuf payload, Topic topic, MqttQoS private class BatchingPublishesCollector { final List[] subscriptions; private final int eventLoops; + private final SessionEventLoopGroup loopGroup; - BatchingPublishesCollector(int eventLoops) { + BatchingPublishesCollector(SessionEventLoopGroup loopGroup) { + eventLoops = loopGroup.getEventLoopCount(); + this.loopGroup = loopGroup; subscriptions = new List[eventLoops]; - this.eventLoops = eventLoops; } public void add(Subscription sub) { @@ -417,7 +410,7 @@ public void add(Subscription sub) { } private int subscriberEventLoop(String clientId) { - return Math.abs(clientId.hashCode()) % this.eventLoops; + return loopGroup.targetQueueOrdinal(clientId); } List routeBatchedPublishes(Consumer> action) { @@ -462,14 +455,14 @@ public int countBatches() { private RoutingResults publish2Subscribers(ByteBuf payload, Topic topic, MqttQoS publishingQos, Set filterTargetClients) { - Set topicMatchingSubscriptions = subscriptions.matchQosSharpening(topic); + List topicMatchingSubscriptions = subscriptions.matchQosSharpening(topic); if (topicMatchingSubscriptions.isEmpty()) { // no matching subscriptions, clean exit LOG.trace("No matching subscriptions for topic: {}", topic); return new RoutingResults(Collections.emptyList(), Collections.emptyList(), CompletableFuture.completedFuture(null)); } - final BatchingPublishesCollector collector = new BatchingPublishesCollector(eventLoops); + final BatchingPublishesCollector collector = new BatchingPublishesCollector(sessionLoops); for (final Subscription sub : topicMatchingSubscriptions) { if (filterTargetClients == NO_FILTER || filterTargetClients.contains(sub.getClientId())) { @@ -503,9 +496,10 @@ private RoutingResults publish2Subscribers(ByteBuf payload, Topic topic, MqttQoS } private void publishToSession(ByteBuf payload, Topic topic, Collection subscriptions, MqttQoS publishingQos) { + ByteBuf duplicate = payload.duplicate(); for (Subscription sub : subscriptions) { MqttQoS qos = lowerQosToTheSubscriptionDesired(sub, publishingQos); - publishToSession(payload, topic, sub, qos); + publishToSession(duplicate, topic, sub, qos); } } @@ -624,41 +618,19 @@ void dispatchConnectionLost(String clientId,String userName) { interceptor.notifyClientConnectionLost(clientId, userName); } + String sessionLoopThreadName(String clientId) { + return sessionLoops.sessionLoopThreadName(clientId); + } + /** * Route the command to the owning SessionEventLoop * */ public RouteResult routeCommand(String clientId, String actionDescription, Callable action) { - SessionCommand cmd = new SessionCommand(clientId, action); - final int targetQueueId = Math.abs(cmd.getSessionId().hashCode()) % this.eventLoops; - LOG.debug("Routing cmd [{}] for session [{}] to event processor {}", actionDescription, cmd.getSessionId(), targetQueueId); - final FutureTask task = new FutureTask<>(() -> { - cmd.execute(); - cmd.complete(); - return cmd.getSessionId(); - }); - if (Thread.currentThread() == sessionExecutors[targetQueueId]) { - SessionEventLoop.executeTask(task); - return RouteResult.success(clientId, cmd.completableFuture()); - } - if (this.sessionQueues[targetQueueId].offer(task)) { - return RouteResult.success(clientId, cmd.completableFuture()); - } else { - LOG.warn("Session command queue {} is full executing action {}", targetQueueId, actionDescription); - return RouteResult.failed(clientId); - } + return sessionLoops.routeCommand(clientId, actionDescription, action); } public void terminate() { - for (Thread processor : sessionExecutors) { - processor.interrupt(); - } - for (Thread processor : sessionExecutors) { - try { - processor.join(5_000); - } catch (InterruptedException ex) { - LOG.info("Interrupted while joining session event loop {}", processor.getName(), ex); - } - } + sessionLoops.terminate(); } /** diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/RetainedMessage.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/RetainedMessage.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/RetainedMessage.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/RetainedMessage.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/RoutingResults.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/RoutingResults.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/RoutingResults.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/RoutingResults.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/Server.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/Server.java similarity index 74% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/Server.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/Server.java index 50c44926..d47de5c2 100644 --- a/moquette-0.16/broker/src/main/java/io/moquette/broker/Server.java +++ b/moquette-0.17/broker/src/main/java/io/moquette/broker/Server.java @@ -17,6 +17,7 @@ import io.moquette.BrokerConstants; import io.moquette.broker.config.FileResourceLoader; +import io.moquette.broker.config.FluentConfig; import io.moquette.broker.config.IConfig; import io.moquette.broker.config.IResourceLoader; import io.moquette.broker.config.MemoryConfig; @@ -28,12 +29,15 @@ import io.moquette.broker.security.IAuthorizatorPolicy; import io.moquette.broker.security.PermitAllAuthorizatorPolicy; import io.moquette.broker.security.ResourceAuthenticator; +import io.moquette.broker.unsafequeues.QueueException; import io.moquette.interception.InterceptHandler; import io.moquette.persistence.H2Builder; +import io.moquette.persistence.MemorySessionsRepository; import io.moquette.persistence.MemorySubscriptionsRepository; import io.moquette.interception.BrokerInterceptor; import io.moquette.broker.subscriptions.CTrieSubscriptionDirectory; import io.moquette.broker.subscriptions.ISubscriptionsDirectory; +import io.moquette.persistence.SegmentQueueRepository; import io.netty.handler.codec.mqtt.MqttPublishMessage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,7 +45,10 @@ import java.io.File; import java.io.IOException; import java.lang.reflect.InvocationTargetException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.text.ParseException; +import java.time.Clock; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -50,12 +57,13 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import static io.moquette.broker.Session.INFINITE_EXPIRY; import static io.moquette.logging.LoggingUtils.getInterceptorIds; public class Server { private static final Logger LOG = LoggerFactory.getLogger(io.moquette.broker.Server.class); - public static final String MOQUETTE_VERSION = "0.16"; + public static final String MOQUETTE_VERSION = "0.17"; private ScheduledExecutorService scheduler; private NewNettyAcceptor acceptor; @@ -157,7 +165,7 @@ public void startServer(IConfig config, List handler } public void startServer(IConfig config, List handlers, ISslContextCreator sslCtxCreator, - IAuthenticator authenticator, IAuthorizatorPolicy authorizatorPolicy) { + IAuthenticator authenticator, IAuthorizatorPolicy authorizatorPolicy) throws IOException { final long start = System.currentTimeMillis(); if (handlers == null) { handlers = Collections.emptyList(); @@ -170,8 +178,6 @@ public void startServer(IConfig config, List handler if (handlerProp != null) { config.setProperty(BrokerConstants.INTERCEPT_HANDLER_PROPERTY_NAME, handlerProp); } - final String persistencePath = config.getProperty(BrokerConstants.PERSISTENT_STORE_PROPERTY_NAME); - LOG.debug("Configuring Using persistent store file, path: {}", persistencePath); initInterceptors(config, handlers); LOG.debug("Initialized MQTT protocol processor"); if (sslCtxCreator == null) { @@ -181,36 +187,75 @@ public void startServer(IConfig config, List handler authenticator = initializeAuthenticator(authenticator, config); authorizatorPolicy = initializeAuthorizatorPolicy(authorizatorPolicy, config); + final ISessionsRepository sessionsRepository; final ISubscriptionsRepository subscriptionsRepository; final IQueueRepository queueRepository; final IRetainedRepository retainedRepository; - if (persistencePath != null && !persistencePath.isEmpty()) { - LOG.trace("Configuring H2 subscriptions store to {}", persistencePath); - h2Builder = new H2Builder(config, scheduler).initStore(); + + if (config.getProperty(BrokerConstants.PERSISTENT_STORE_PROPERTY_NAME) != null) { + LOG.warn("Using a deprecated setting {} please update to {}", + BrokerConstants.PERSISTENT_STORE_PROPERTY_NAME, IConfig.DATA_PATH_PROPERTY_NAME); + LOG.warn("Forcing {} to true", IConfig.PERSISTENCE_ENABLED_PROPERTY_NAME); + config.setProperty(IConfig.PERSISTENCE_ENABLED_PROPERTY_NAME, Boolean.TRUE.toString()); + + final String persistencePath = config.getProperty(BrokerConstants.PERSISTENT_STORE_PROPERTY_NAME); + final String dataPath = persistencePath.substring(0, persistencePath.lastIndexOf("/")); + LOG.warn("Forcing {} to {}", IConfig.DATA_PATH_PROPERTY_NAME, dataPath); + config.setProperty(IConfig.DATA_PATH_PROPERTY_NAME, dataPath); + } + + final Clock clock = Clock.systemDefaultZone(); + + if (Boolean.parseBoolean(config.getProperty(IConfig.PERSISTENCE_ENABLED_PROPERTY_NAME))) { + final Path dataPath = Paths.get(config.getProperty(IConfig.DATA_PATH_PROPERTY_NAME)); + if (!dataPath.toFile().exists()) { + if (dataPath.toFile().mkdirs()) { + LOG.debug("Created data_path {} folder", dataPath); + } else { + LOG.warn("Impossible to create the data_path {}", dataPath); + } + } + + LOG.debug("Configuring persistent subscriptions store and queues, path: {}", dataPath); + final int autosaveInterval = Integer.parseInt(config.getProperty(BrokerConstants.AUTOSAVE_INTERVAL_PROPERTY_NAME, "30")); + h2Builder = new H2Builder(scheduler, dataPath, autosaveInterval, clock).initStore(); + queueRepository = initQueuesRepository(config, dataPath, h2Builder); + LOG.trace("Configuring H2 subscriptions repository"); subscriptionsRepository = h2Builder.subscriptionsRepository(); - queueRepository = h2Builder.queueRepository(); retainedRepository = h2Builder.retainedRepository(); + sessionsRepository = h2Builder.sessionsRepository(); } else { LOG.trace("Configuring in-memory subscriptions store"); subscriptionsRepository = new MemorySubscriptionsRepository(); queueRepository = new MemoryQueueRepository(); retainedRepository = new MemoryRetainedRepository(); + sessionsRepository = new MemorySessionsRepository(); } ISubscriptionsDirectory subscriptions = new CTrieSubscriptionDirectory(); subscriptions.init(subscriptionsRepository); final Authorizator authorizator = new Authorizator(authorizatorPolicy); - sessions = new SessionRegistry(subscriptions, queueRepository, authorizator); - final int sessionQueueSize = config.intProp(BrokerConstants.SESSION_QUEUE_SIZE, 1024); + + final int globalSessionExpiry; + if (config.getProperty(IConfig.PERSISTENT_CLIENT_EXPIRATION_PROPERTY_NAME) != null) { + globalSessionExpiry = (int) config.durationProp(IConfig.PERSISTENT_CLIENT_EXPIRATION_PROPERTY_NAME).toMillis() / 1000; + } else { + globalSessionExpiry = INFINITE_EXPIRY; + } + + final int sessionQueueSize = config.intProp(IConfig.SESSION_QUEUE_SIZE, 1024); + final SessionEventLoopGroup loopsGroup = new SessionEventLoopGroup(interceptor, sessionQueueSize); + sessions = new SessionRegistry(subscriptions, sessionsRepository, queueRepository, authorizator, scheduler, + clock, globalSessionExpiry, loopsGroup); dispatcher = new PostOffice(subscriptions, retainedRepository, sessions, interceptor, authorizator, - sessionQueueSize); + loopsGroup); final BrokerConfiguration brokerConfig = new BrokerConfiguration(config); MQTTConnectionFactory connectionFactory = new MQTTConnectionFactory(brokerConfig, authenticator, sessions, dispatcher); final NewNettyMQTTHandler mqttHandler = new NewNettyMQTTHandler(connectionFactory); acceptor = new NewNettyAcceptor(); - acceptor.initialize(mqttHandler, config, sslCtxCreator); + acceptor.initialize(mqttHandler, config, sslCtxCreator, brokerConfig); final long startTime = System.currentTimeMillis() - start; LOG.info("Moquette integration has been started successfully in {} ms", startTime); @@ -218,15 +263,37 @@ public void startServer(IConfig config, List handler initialized = true; } + private static IQueueRepository initQueuesRepository(IConfig config, Path dataPath, H2Builder h2Builder) throws IOException { + final IQueueRepository queueRepository; + final String queueType = config.getProperty(IConfig.PERSISTENT_QUEUE_TYPE_PROPERTY_NAME); + if ("h2".equalsIgnoreCase(queueType)) { + LOG.info("Configuring H2 queue store"); + queueRepository = h2Builder.queueRepository(); + } else if ("segmented".equalsIgnoreCase(queueType)) { + LOG.info("Configuring segmented queue store to {}", dataPath); + final int pageSize = config.intProp(BrokerConstants.SEGMENTED_QUEUE_PAGE_SIZE, BrokerConstants.DEFAULT_SEGMENTED_QUEUE_PAGE_SIZE); + final int segmentSize = config.intProp(BrokerConstants.SEGMENTED_QUEUE_SEGMENT_SIZE, BrokerConstants.DEFAULT_SEGMENTED_QUEUE_SEGMENT_SIZE); + try { + queueRepository = new SegmentQueueRepository(dataPath, pageSize, segmentSize); + } catch (QueueException e) { + throw new IOException("Problem in configuring persistent queue on path " + dataPath, e); + } + } else { + final String errMsg = String.format("Invalid property for %s found [%s] while only h2 or segmented are admitted", IConfig.PERSISTENT_QUEUE_TYPE_PROPERTY_NAME, queueType); + throw new RuntimeException(errMsg); + } + return queueRepository; + } + private IAuthorizatorPolicy initializeAuthorizatorPolicy(IAuthorizatorPolicy authorizatorPolicy, IConfig props) { LOG.debug("Configuring MQTT authorizator policy"); - String authorizatorClassName = props.getProperty(BrokerConstants.AUTHORIZATOR_CLASS_NAME, ""); + String authorizatorClassName = props.getProperty(IConfig.AUTHORIZATOR_CLASS_NAME, ""); if (authorizatorPolicy == null && !authorizatorClassName.isEmpty()) { authorizatorPolicy = loadClass(authorizatorClassName, IAuthorizatorPolicy.class, IConfig.class, props); } if (authorizatorPolicy == null) { - String aclFilePath = props.getProperty(BrokerConstants.ACL_FILE_PROPERTY_NAME, ""); + String aclFilePath = props.getProperty(IConfig.ACL_FILE_PROPERTY_NAME, ""); if (aclFilePath != null && !aclFilePath.isEmpty()) { authorizatorPolicy = new DenyAllAuthorizatorPolicy(); try { @@ -246,7 +313,7 @@ private IAuthorizatorPolicy initializeAuthorizatorPolicy(IAuthorizatorPolicy aut private IAuthenticator initializeAuthenticator(IAuthenticator authenticator, IConfig props) { LOG.debug("Configuring MQTT authenticator"); - String authenticatorClassName = props.getProperty(BrokerConstants.AUTHENTICATOR_CLASS_NAME, ""); + String authenticatorClassName = props.getProperty(IConfig.AUTHENTICATOR_CLASS_NAME, ""); if (authenticator == null && !authenticatorClassName.isEmpty()) { authenticator = loadClass(authenticatorClassName, IAuthenticator.class, IConfig.class, props); @@ -254,7 +321,7 @@ private IAuthenticator initializeAuthenticator(IAuthenticator authenticator, ICo IResourceLoader resourceLoader = props.getResourceLoader(); if (authenticator == null) { - String passwdPath = props.getProperty(BrokerConstants.PASSWORD_FILE_PROPERTY_NAME, ""); + String passwdPath = props.getProperty(IConfig.PASSWORD_FILE_PROPERTY_NAME, ""); if (passwdPath.isEmpty()) { authenticator = new AcceptAllAuthenticator(); } else { @@ -341,6 +408,10 @@ public RoutingResults internalPublish(MqttPublishMessage msg, final String clien public void stopServer() { LOG.info("Unbinding integration from the configured ports"); + if (acceptor == null) { + LOG.error("Closing a badly started server, exit immediately"); + return; + } acceptor.close(); LOG.trace("Stopping MQTT protocol processor"); initialized = false; @@ -349,6 +420,8 @@ public void stopServer() { // and SessionsRepository does not stop its tasks. Thus shutdownNow(). scheduler.shutdownNow(); + sessions.close(); + if (h2Builder != null) { LOG.trace("Shutting down H2 persistence"); h2Builder.closeStore(); @@ -415,6 +488,30 @@ public void removeInterceptHandler(InterceptHandler interceptHandler) { * Return a list of descriptors of connected clients. * */ public Collection listConnectedClients() { + if (!initialized) { + LOG.error("Moquette is not started, MQTT clients listing unavailable"); + throw new IllegalStateException("Can't get clients list from a Server that is not yet started"); + } return sessions.listConnectedClients(); } + /** + * Force the disconnection of a client, closing the related session. + * @param clientId the name of the client to drop session. + */ + public boolean disconnectClient(final String clientId) { + return sessions.dropSession(clientId, false); + } + + /** + * Force the disconnection of a client, closing the related session and removing any session state from + * the broker, such as subscriptions and queue. + * @param clientId the name of the client to drop session. + */ + public boolean disconnectAndPurgeClientState(final String clientId) { + return sessions.dropSession(clientId, true); + } + + public FluentConfig withConfig() { + return new FluentConfig(this); + } } diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/Session.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/Session.java similarity index 91% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/Session.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/Session.java index 612a3453..17829f6a 100644 --- a/moquette-0.16/broker/src/main/java/io/moquette/broker/Session.java +++ b/moquette-0.17/broker/src/main/java/io/moquette/broker/Session.java @@ -22,13 +22,25 @@ import io.moquette.broker.subscriptions.Subscription; import io.moquette.broker.subscriptions.Topic; import io.netty.buffer.ByteBuf; -import io.netty.handler.codec.mqtt.*; +import io.netty.handler.codec.mqtt.MqttFixedHeader; +import io.netty.handler.codec.mqtt.MqttMessage; +import io.netty.handler.codec.mqtt.MqttMessageType; +import io.netty.handler.codec.mqtt.MqttPublishMessage; +import io.netty.handler.codec.mqtt.MqttPublishVariableHeader; +import io.netty.handler.codec.mqtt.MqttQoS; import io.netty.util.ReferenceCountUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.net.InetSocketAddress; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; import java.util.concurrent.DelayQueue; import java.util.concurrent.Delayed; import java.util.concurrent.TimeUnit; @@ -38,6 +50,9 @@ class Session { private static final Logger LOG = LoggerFactory.getLogger(Session.class); + // By specification session expiry value of 0xEFFFFFFF (UINT_MAX) (seconds) means + // session that doesn't expire, it's ~68 years. + static final int INFINITE_EXPIRY = Integer.MAX_VALUE; static class InFlightPacket implements Delayed { @@ -87,7 +102,6 @@ static final class Will { } } - private final String clientId; private boolean clean; private Will will; private final SessionMessageQueue sessionQueue; @@ -98,21 +112,26 @@ static final class Will { private final DelayQueue inflightTimeouts = new DelayQueue<>(); private final Map qos2Receiving = new HashMap<>(); private final AtomicInteger inflightSlots = new AtomicInteger(INFLIGHT_WINDOW_SIZE); // this should be configurable + private final ISessionsRepository.SessionData data; - Session(String clientId, boolean clean, Will will, SessionMessageQueue sessionQueue) { - this(clientId, clean, sessionQueue); + Session(ISessionsRepository.SessionData data, boolean clean, Will will, SessionMessageQueue sessionQueue) { + this(data, clean, sessionQueue); this.will = will; } - Session(String clientId, boolean clean, SessionMessageQueue sessionQueue) { + Session(ISessionsRepository.SessionData data, boolean clean, SessionMessageQueue sessionQueue) { if (sessionQueue == null) { throw new IllegalArgumentException("sessionQueue parameter can't be null"); } - this.clientId = clientId; + this.data = data; this.clean = clean; this.sessionQueue = sessionQueue; } + public boolean expireImmediately() { + return data.expiryInterval() == 0; + } + void update(boolean clean, Will will) { this.clean = clean; this.will = will; @@ -143,7 +162,7 @@ public boolean connected() { } public String getClientID() { - return clientId; + return data.clientId(); } public List getSubscriptions() { @@ -155,7 +174,7 @@ public void addSubscriptions(List newSubscriptions) { } public void removeSubscription(Topic topic) { - subscriptions.remove(new Subscription(clientId, topic, MqttQoS.EXACTLY_ONCE)); + subscriptions.remove(new Subscription(data.clientId(), topic, MqttQoS.EXACTLY_ONCE)); } public boolean hasWill() { @@ -410,7 +429,7 @@ private MqttPublishMessage publishNotRetainedDuplicated(InFlightPacket notAckPac private void drainQueueToConnection() { // consume the queue - while (!sessionQueue.isEmpty() && inflighHasSlotsAndConnectionIsUp()) { + while (connected() && !sessionQueue.isEmpty() && inflighHasSlotsAndConnectionIsUp()) { final SessionRegistry.EnqueuedMessage msg = sessionQueue.dequeue(); if (msg == null) { // Our message was already fetched by another Thread. @@ -482,10 +501,14 @@ public void cleanUp() { } } + ISessionsRepository.SessionData getSessionData() { + return this.data; + } + @Override public String toString() { return "Session{" + - "clientId='" + clientId + '\'' + + "clientId='" + data.clientId() + '\'' + ", clean=" + clean + ", status=" + status + ", inflightSlots=" + inflightSlots + diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/SessionCommand.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/SessionCommand.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/SessionCommand.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/SessionCommand.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/SessionCorruptedException.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/SessionCorruptedException.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/SessionCorruptedException.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/SessionCorruptedException.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/SessionEventLoop.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/SessionEventLoop.java similarity index 91% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/SessionEventLoop.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/SessionEventLoop.java index dea036ba..7b07d2f7 100644 --- a/moquette-0.16/broker/src/main/java/io/moquette/broker/SessionEventLoop.java +++ b/moquette-0.17/broker/src/main/java/io/moquette/broker/SessionEventLoop.java @@ -6,7 +6,7 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.FutureTask; -final class SessionEventLoop implements Runnable { +final class SessionEventLoop extends Thread { private static final Logger LOG = LoggerFactory.getLogger(SessionEventLoop.class); @@ -48,7 +48,8 @@ public static void executeTask(final FutureTask task) { // we ran it, but we have to grab the exception if raised task.get(); } catch (Throwable th) { - LOG.info("SessionEventLoop {} reached exception in processing command", Thread.currentThread().getName(), th); + LOG.warn("SessionEventLoop {} reached exception in processing command", Thread.currentThread().getName(), th); + throw new RuntimeException(th); } } } diff --git a/moquette-0.17/broker/src/main/java/io/moquette/broker/SessionEventLoopGroup.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/SessionEventLoopGroup.java new file mode 100644 index 00000000..cc4b8cdb --- /dev/null +++ b/moquette-0.17/broker/src/main/java/io/moquette/broker/SessionEventLoopGroup.java @@ -0,0 +1,111 @@ +package io.moquette.broker; + +import io.moquette.interception.BrokerInterceptor; +import io.moquette.interception.messages.InterceptExceptionMessage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.FutureTask; + +class SessionEventLoopGroup { + private static final Logger LOG = LoggerFactory.getLogger(SessionEventLoopGroup.class); + + private final SessionEventLoop[] sessionExecutors; + private final BlockingQueue>[] sessionQueues; + private final int eventLoops = Runtime.getRuntime().availableProcessors(); + private final ConcurrentMap loopThrownExceptions = new ConcurrentHashMap<>(); + + SessionEventLoopGroup(BrokerInterceptor interceptor, int sessionQueueSize) { + this.sessionQueues = new BlockingQueue[eventLoops]; + for (int i = 0; i < eventLoops; i++) { + this.sessionQueues[i] = new ArrayBlockingQueue<>(sessionQueueSize); + } + this.sessionExecutors = new SessionEventLoop[eventLoops]; + for (int i = 0; i < eventLoops; i++) { + SessionEventLoop newLoop = new SessionEventLoop(this.sessionQueues[i]); + newLoop.setName(sessionLoopName(i)); + newLoop.setUncaughtExceptionHandler((loopThread, ex) -> { + // executed in session loop thread + // collect the exception thrown to later re-throw + loopThrownExceptions.put(loopThread.getName(), ex); + + // This is done in asynch from another thread in BrokerInterceptor + interceptor.notifyLoopException(new InterceptExceptionMessage(ex)); + }); + newLoop.start(); + this.sessionExecutors[i] = newLoop; + } + } + + int targetQueueOrdinal(String clientId) { + return Math.abs(clientId.hashCode()) % this.eventLoops; + } + + private String sessionLoopName(int i) { + return "Session Executor " + i; + } + + String sessionLoopThreadName(String clientId) { + final int targetQueueId = targetQueueOrdinal(clientId); + return sessionLoopName(targetQueueId); + } + + /** + * Route the command to the owning SessionEventLoop + */ + public PostOffice.RouteResult routeCommand(String clientId, String actionDescription, Callable action) { + SessionCommand cmd = new SessionCommand(clientId, action); + + if (clientId == null) { + LOG.warn("Routing collision for action [{}]", actionDescription); + return PostOffice.RouteResult.failed(null, "Seems awaiting new route feature completion, skipping."); + } + + final int targetQueueId = targetQueueOrdinal(cmd.getSessionId()); + LOG.debug("Routing cmd [{}] for session [{}] to event processor {}", actionDescription, cmd.getSessionId(), targetQueueId); + final FutureTask task = new FutureTask<>(() -> { + cmd.execute(); + cmd.complete(); + return cmd.getSessionId(); + }); + if (Thread.currentThread() == sessionExecutors[targetQueueId]) { + SessionEventLoop.executeTask(task); + return PostOffice.RouteResult.success(clientId, cmd.completableFuture()); + } + if (this.sessionQueues[targetQueueId].offer(task)) { + return PostOffice.RouteResult.success(clientId, cmd.completableFuture()); + } else { + LOG.warn("Session command queue {} is full executing action {}", targetQueueId, actionDescription); + return PostOffice.RouteResult.failed(clientId); + } + } + + public void terminate() { + for (SessionEventLoop processor : sessionExecutors) { + processor.interrupt(); + } + for (SessionEventLoop processor : sessionExecutors) { + try { + processor.join(5_000); + } catch (InterruptedException ex) { + LOG.info("Interrupted while joining session event loop {}", processor.getName(), ex); + } + } + + for (Map.Entry loopThrownExceptionEntry : loopThrownExceptions.entrySet()) { + String threadName = loopThrownExceptionEntry.getKey(); + Throwable threadError = loopThrownExceptionEntry.getValue(); + LOG.error("Session event loop {} terminated with error", threadName, threadError); + } + } + + public int getEventLoopCount() { + return eventLoops; + } +} diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/SessionMessageQueue.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/SessionMessageQueue.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/SessionMessageQueue.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/SessionMessageQueue.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/SessionRegistry.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/SessionRegistry.java similarity index 55% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/SessionRegistry.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/SessionRegistry.java index 8932459e..2bf0af52 100644 --- a/moquette-0.16/broker/src/main/java/io/moquette/broker/SessionRegistry.java +++ b/moquette-0.17/broker/src/main/java/io/moquette/broker/SessionRegistry.java @@ -23,33 +23,52 @@ import io.netty.buffer.Unpooled; import io.netty.handler.codec.mqtt.MqttConnectMessage; import io.netty.handler.codec.mqtt.MqttQoS; +import io.netty.handler.codec.mqtt.MqttVersion; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.net.InetSocketAddress; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.List; import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.DelayQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import static io.moquette.broker.Session.INFINITE_EXPIRY; + public class SessionRegistry { + private final ScheduledFuture scheduledExpiredSessions; + private int globalExpirySeconds; + private final SessionEventLoopGroup loopsGroup; + static final Duration EXPIRED_SESSION_CLEANER_TASK_INTERVAL = Duration.ofSeconds(1); + public abstract static class EnqueuedMessage { /** * Releases any held resources. Must be called when the EnqueuedMessage is no * longer needed. */ - public void release() {} + public void release() { + } /** * Retains any held resources. Must be called when the EnqueuedMessage is added * to a store. */ - public void retain() {} + public void retain() { + } } public static class PublishedMessage extends EnqueuedMessage { @@ -114,31 +133,80 @@ public SessionCreationResult(Session session, CreationModeEnum mode, boolean alr private final ConcurrentMap pool = new ConcurrentHashMap<>(); private final ISubscriptionsDirectory subscriptionsDirectory; + private final ISessionsRepository sessionsRepository; private final IQueueRepository queueRepository; private final Authorizator authorizator; + private final DelayQueue removableSessions = new DelayQueue<>(); + private final Clock clock; + // Used in testing SessionRegistry(ISubscriptionsDirectory subscriptionsDirectory, + ISessionsRepository sessionsRepository, IQueueRepository queueRepository, - Authorizator authorizator) { + Authorizator authorizator, + ScheduledExecutorService scheduler, + SessionEventLoopGroup loopsGroup) { + this(subscriptionsDirectory, sessionsRepository, queueRepository, authorizator, scheduler, Clock.systemDefaultZone(), INFINITE_EXPIRY, loopsGroup); + } + + SessionRegistry(ISubscriptionsDirectory subscriptionsDirectory, + ISessionsRepository sessionsRepository, + IQueueRepository queueRepository, + Authorizator authorizator, + ScheduledExecutorService scheduler, + Clock clock, int globalExpirySeconds, + SessionEventLoopGroup loopsGroup) { this.subscriptionsDirectory = subscriptionsDirectory; + this.sessionsRepository = sessionsRepository; this.queueRepository = queueRepository; this.authorizator = authorizator; + this.scheduledExpiredSessions = scheduler.scheduleWithFixedDelay(this::checkExpiredSessions, + EXPIRED_SESSION_CLEANER_TASK_INTERVAL.getSeconds(), EXPIRED_SESSION_CLEANER_TASK_INTERVAL.getSeconds(), + TimeUnit.SECONDS); + this.clock = clock; + this.globalExpirySeconds = globalExpirySeconds; + this.loopsGroup = loopsGroup; recreateSessionPool(); } + private void checkExpiredSessions() { + List expiredSessions = new ArrayList<>(); + int drainedSessions = removableSessions.drainTo(expiredSessions); + LOG.debug("Retrieved {} expired sessions or {}", drainedSessions, removableSessions.size()); + for (ISessionsRepository.SessionData expiredSession : expiredSessions) { + final String expiredAt = expiredSession.expireAt().map(Instant::toString).orElse("UNDEFINED"); + LOG.debug("Removing session {}, expired on {}", expiredSession.clientId(), expiredAt); + remove(expiredSession.clientId()); + sessionsRepository.delete(expiredSession); + } + } + + private void trackForRemovalOnExpiration(ISessionsRepository.SessionData session) { + if (!session.expireAt().isPresent()) { + throw new RuntimeException("Can't track for expiration a session without expiry instant, client_id: " + session.clientId()); + } + LOG.debug("start tracking the session {} for removal", session.clientId()); + removableSessions.add(session); + } + + private void untrackFromRemovalOnExpiration(ISessionsRepository.SessionData session) { + removableSessions.remove(session); + } + private void recreateSessionPool() { final Set queues = queueRepository.listQueueNames(); - for (String clientId : subscriptionsDirectory.listAllSessionIds()) { + for (ISessionsRepository.SessionData session : sessionsRepository.list()) { // if the subscriptions are present is obviously false - if (queueRepository.containsQueue(clientId)) { - final SessionMessageQueue persistentQueue = queueRepository.getOrCreateQueue(clientId); - queues.remove(clientId); - Session rehydrated = new Session(clientId, false, persistentQueue); - pool.put(clientId, rehydrated); + if (queueRepository.containsQueue(session.clientId())) { + final SessionMessageQueue persistentQueue = queueRepository.getOrCreateQueue(session.clientId()); + queues.remove(session.clientId()); + Session rehydrated = new Session(session, false, persistentQueue); + pool.put(session.clientId(), rehydrated); + trackForRemovalOnExpiration(session); } } if (!queues.isEmpty()) { - LOG.error("Recreating sessions left {} unused queues. This is probably bug. Session IDs: {}", queues.size(), Arrays.toString(queues.toArray())); + LOG.error("Recreating sessions left {} unused queues. This is probably a bug. Session IDs: {}", queues.size(), Arrays.toString(queues.toArray())); } } @@ -174,17 +242,10 @@ private SessionCreationResult reopenExistingSession(MqttConnectMessage msg, Stri oldSession.closeImmediately(); } - if (newIsClean) { - boolean result = oldSession.assignState(SessionStatus.DISCONNECTED, SessionStatus.DESTROYED); - if (!result) { - throw new SessionCorruptedException("old session has already changed state"); - } - - // case 2, reopening existing session but with cleanSession true + // case 2, reopening existing session but with a clean session + purgeSessionState(oldSession); // publish new session - unsubscribe(oldSession); - remove(clientId); final Session newSession = createNewSession(msg, clientId); pool.put(clientId, newSession); @@ -195,7 +256,7 @@ private SessionCreationResult reopenExistingSession(MqttConnectMessage msg, Stri if (!connecting) { throw new SessionCorruptedException("old session moved in connected state by other thread"); } - // case 3, reopening existing session without cleanSession, so keep the existing subscriptions + // case 3, reopening existing session not clean session, so keep the existing subscriptions copySessionConfig(msg, oldSession); reactivateSubscriptions(oldSession, username); @@ -203,6 +264,8 @@ private SessionCreationResult reopenExistingSession(MqttConnectMessage msg, Stri creationResult = new SessionCreationResult(oldSession, CreationModeEnum.REOPEN_EXISTING, true); } + untrackFromRemovalOnExpiration(creationResult.session.getSessionData()); + // case not covered new session is clean true/false and old session not in CONNECTED/DISCONNECTED return creationResult; } @@ -211,7 +274,7 @@ private void reactivateSubscriptions(Session session, String username) { //verify if subscription still satisfy read ACL permissions for (Subscription existingSub : session.getSubscriptions()) { final boolean topicReadable = authorizator.canRead(existingSub.getTopicFilter(), username, - session.getClientID()); + session.getClientID()); if (!topicReadable) { subscriptionsDirectory.removeSubscription(existingSub.getTopicFilter(), session.getClientID()); } @@ -235,14 +298,20 @@ private Session createNewSession(MqttConnectMessage msg, String clientId) { } else { queue = new InMemoryQueue(); } + // in MQTT3 cleanSession = true means expiryInterval=0 else infinite + final int expiryInterval = clean ? 0 : globalExpirySeconds; + + final ISessionsRepository.SessionData sessionData = new ISessionsRepository.SessionData(clientId, + MqttVersion.MQTT_3_1_1, expiryInterval, clock); if (msg.variableHeader().isWillFlag()) { final Session.Will will = createWill(msg); - newSession = new Session(clientId, clean, will, queue); + newSession = new Session(sessionData, clean, will, queue); } else { - newSession = new Session(clientId, clean, queue); + newSession = new Session(sessionData, clean, queue); } newSession.markConnecting(); + sessionsRepository.saveSession(sessionData); return newSession; } @@ -269,10 +338,36 @@ Session retrieve(String clientID) { return pool.get(clientID); } + void connectionClosed(Session session) { + session.disconnect(); + if (session.expireImmediately()) { + purgeSessionState(session); + } else { + //bound session has expiry, disconnect it and add to the queue for removal + trackForRemovalOnExpiration(session.getSessionData().withExpirationComputed()); + } + } + + private void purgeSessionState(Session session) { + LOG.debug("Remove session state for client {}", session.getClientID()); + boolean result = session.assignState(SessionStatus.DISCONNECTED, SessionStatus.DESTROYED); + if (!result) { + throw new SessionCorruptedException("Session has already changed state: " + session); + } + + unsubscribe(session); + remove(session.getClientID()); + } + void remove(String clientID) { final Session old = pool.remove(clientID); if (old != null) { - old.cleanUp(); + // remove from expired tracker if present + removableSessions.remove(old.getSessionData()); + loopsGroup.routeCommand(clientID, "Clean up removed session", () -> { + old.cleanUp(); + return null; + }); } } @@ -285,9 +380,61 @@ Collection listConnectedClients() { .collect(Collectors.toList()); } + /** + * Close the connection bound to the session for the clintId. If removeSessionState is provided + * remove any session state like queues and subscription from broker memory. + * + * @param clientId the name of the client to drop the session. + * @param removeSessionState boolean flag to request the removal of session state from broker. + */ + boolean dropSession(final String clientId, boolean removeSessionState) { + LOG.debug("Disconnecting client: {}", clientId); + if (clientId == null) { + return false; + } + + final Session client = pool.get(clientId); + if (client == null) { + LOG.debug("Client {} not found, nothing disconnected", clientId); + return false; + } + + client.closeImmediately(); + if (removeSessionState) { + purgeSessionState(client); + } + + LOG.debug("Client {} successfully disconnected from broker", clientId); + return true; + } + private Optional createClientDescriptor(Session s) { final String clientID = s.getClientID(); final Optional remoteAddressOpt = s.remoteAddress(); return remoteAddressOpt.map(r -> new ClientDescriptor(clientID, r.getHostString(), r.getPort())); } + + /** + * Close all resources related to session management + */ + public void close() { + if (scheduledExpiredSessions.cancel(false)) { + LOG.info("Successfully cancelled expired sessions task"); + } else { + LOG.warn("Can't cancel the execution of expired sessions task, was already cancelled? {}, was done? {}", + scheduledExpiredSessions.isCancelled(), scheduledExpiredSessions.isDone()); + } + // Update all not clean session with the proper expiry date + updateNotCleanSessionsWithProperExpire(); + queueRepository.close(); + } + + private void updateNotCleanSessionsWithProperExpire() { + pool.values().stream() + .filter(s -> !s.isClean()) // not clean session + .map(Session::getSessionData) + .filter(s -> !s.expireAt().isPresent()) // without expire set + .map(ISessionsRepository.SessionData::withExpirationComputed) // new SessionData with expireAt valued + .forEach(sessionsRepository::saveSession); // update the storage + } } diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/Utils.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/Utils.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/Utils.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/Utils.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/config/ClasspathResourceLoader.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/config/ClasspathResourceLoader.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/config/ClasspathResourceLoader.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/config/ClasspathResourceLoader.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/config/ConfigurationParser.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/config/ConfigurationParser.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/config/ConfigurationParser.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/config/ConfigurationParser.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/config/FileResourceLoader.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/config/FileResourceLoader.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/config/FileResourceLoader.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/config/FileResourceLoader.java diff --git a/moquette-0.17/broker/src/main/java/io/moquette/broker/config/FluentConfig.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/config/FluentConfig.java new file mode 100644 index 00000000..2511d9ed --- /dev/null +++ b/moquette-0.17/broker/src/main/java/io/moquette/broker/config/FluentConfig.java @@ -0,0 +1,285 @@ +package io.moquette.broker.config; + +import io.moquette.BrokerConstants; +import io.moquette.broker.Server; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Locale; +import java.util.Properties; +import java.util.function.Consumer; + +import static io.moquette.broker.config.IConfig.ACL_FILE_PROPERTY_NAME; +import static io.moquette.broker.config.IConfig.ALLOW_ANONYMOUS_PROPERTY_NAME; +import static io.moquette.broker.config.IConfig.AUTHENTICATOR_CLASS_NAME; +import static io.moquette.broker.config.IConfig.AUTHORIZATOR_CLASS_NAME; +import static io.moquette.broker.config.IConfig.BUFFER_FLUSH_MS_PROPERTY_NAME; +import static io.moquette.broker.config.IConfig.DATA_PATH_PROPERTY_NAME; +import static io.moquette.broker.config.IConfig.DEFAULT_NETTY_MAX_BYTES_IN_MESSAGE; +import static io.moquette.broker.config.IConfig.ENABLE_TELEMETRY_NAME; +import static io.moquette.broker.config.IConfig.PORT_PROPERTY_NAME; +import static io.moquette.broker.config.IConfig.JKS_PATH_PROPERTY_NAME; +import static io.moquette.broker.config.IConfig.KEY_MANAGER_PASSWORD_PROPERTY_NAME; +import static io.moquette.broker.config.IConfig.KEY_STORE_PASSWORD_PROPERTY_NAME; +import static io.moquette.broker.config.IConfig.KEY_STORE_TYPE; +import static io.moquette.broker.config.IConfig.NETTY_MAX_BYTES_PROPERTY_NAME; +import static io.moquette.broker.config.IConfig.PASSWORD_FILE_PROPERTY_NAME; +import static io.moquette.broker.config.IConfig.PERSISTENCE_ENABLED_PROPERTY_NAME; +import static io.moquette.broker.config.IConfig.PERSISTENT_CLIENT_EXPIRATION_PROPERTY_NAME; +import static io.moquette.broker.config.IConfig.PERSISTENT_QUEUE_TYPE_PROPERTY_NAME; +import static io.moquette.broker.config.IConfig.HOST_PROPERTY_NAME; +import static io.moquette.broker.config.IConfig.SESSION_QUEUE_SIZE; +import static io.moquette.broker.config.IConfig.SSL_PORT_PROPERTY_NAME; +import static io.moquette.broker.config.IConfig.SSL_PROVIDER; +import static io.moquette.broker.config.IConfig.WEB_SOCKET_PORT_PROPERTY_NAME; + +/** + * DSL to create Moquette config. + * It provides methods to configure every available setting. + * To be used instead of Properties instance used combined with MemoryConfig. + * */ +public class FluentConfig { + + private Server server; + private TLSConfig tlsConfig; + + public enum BufferFlushKind { + IMMEDIATE, FULL; + } + + public enum PersistentQueueType { + H2, SEGMENTED; + } + + public enum SSLProvider { + SSL, OPENSSL, OPENSSL_REFCNT; + } + + public enum KeyStoreType { + JKS, JCEKS, PKCS12; + } + + private enum CreationKind { + API, SERVER + } + + private final Properties configAccumulator = new Properties(); + + private final CreationKind creationKind; + + public FluentConfig() { + initializeDefaultValues(); + creationKind = CreationKind.API; + } + + // Invoked only when initialized directly by the server + public FluentConfig(Server server) { + initializeDefaultValues(); + creationKind = CreationKind.SERVER; + this.server = server; + } + + private void initializeDefaultValues() { + // preload with default values + configAccumulator.put(PORT_PROPERTY_NAME, Integer.toString(BrokerConstants.PORT)); + configAccumulator.put(HOST_PROPERTY_NAME, BrokerConstants.HOST); + configAccumulator.put(PASSWORD_FILE_PROPERTY_NAME, ""); + configAccumulator.put(ALLOW_ANONYMOUS_PROPERTY_NAME, Boolean.TRUE.toString()); + configAccumulator.put(AUTHENTICATOR_CLASS_NAME, ""); + configAccumulator.put(AUTHORIZATOR_CLASS_NAME, ""); + configAccumulator.put(NETTY_MAX_BYTES_PROPERTY_NAME, String.valueOf(DEFAULT_NETTY_MAX_BYTES_IN_MESSAGE)); + configAccumulator.put(PERSISTENT_QUEUE_TYPE_PROPERTY_NAME, PersistentQueueType.SEGMENTED.name().toLowerCase(Locale.ROOT)); + configAccumulator.put(DATA_PATH_PROPERTY_NAME, "data/"); + configAccumulator.put(PERSISTENCE_ENABLED_PROPERTY_NAME, Boolean.TRUE.toString()); + configAccumulator.put(BUFFER_FLUSH_MS_PROPERTY_NAME, BufferFlushKind.IMMEDIATE.name().toLowerCase(Locale.ROOT)); + } + + public FluentConfig host(String host) { + configAccumulator.put(HOST_PROPERTY_NAME, host); + return this; + } + + public FluentConfig port(int port) { + validatePort(port); + configAccumulator.put(PORT_PROPERTY_NAME, Integer.toString(port)); + return this; + } + + public FluentConfig websocketPort(int port) { + validatePort(port); + configAccumulator.put(WEB_SOCKET_PORT_PROPERTY_NAME, Integer.toString(port)); + return this; + } + + private static void validatePort(int port) { + if (port > 65_535 || port < 0) { + throw new IllegalArgumentException("Port must be in range [0.65535]"); + } + } + + public FluentConfig dataPath(String dataPath) { + configAccumulator.put(DATA_PATH_PROPERTY_NAME, dataPath); + return this; + } + + public FluentConfig dataPath(Path dataPath) { + configAccumulator.put(DATA_PATH_PROPERTY_NAME, dataPath.toAbsolutePath().toString()); + return this; + } + public FluentConfig enablePersistence() { + configAccumulator.put(PERSISTENCE_ENABLED_PROPERTY_NAME, "true"); + return this; + } + + public FluentConfig disablePersistence() { + configAccumulator.put(PERSISTENCE_ENABLED_PROPERTY_NAME, "false"); + return this; + } + + public FluentConfig persistentQueueType(PersistentQueueType type) { + configAccumulator.put(PERSISTENT_QUEUE_TYPE_PROPERTY_NAME, type.name().toLowerCase(Locale.ROOT)); + return this; + } + + public FluentConfig allowAnonymous() { + configAccumulator.put(ALLOW_ANONYMOUS_PROPERTY_NAME, "true"); + return this; + } + + public FluentConfig disallowAnonymous() { + configAccumulator.put(ALLOW_ANONYMOUS_PROPERTY_NAME, "false"); + return this; + } + + /** + * @param aclPath relative path to the resource file that contains the definitions + * */ + public FluentConfig aclFile(String aclPath) { + configAccumulator.put(ACL_FILE_PROPERTY_NAME, aclPath); + return this; + } + + /** + * @param passwordFilePath relative path to the resource file that contains the passwords. + * */ + public FluentConfig passwordFile(String passwordFilePath) { + configAccumulator.put(PASSWORD_FILE_PROPERTY_NAME, passwordFilePath); + return this; + } + public FluentConfig bufferFlushMillis(int value) { + configAccumulator.put(BUFFER_FLUSH_MS_PROPERTY_NAME, Integer.valueOf(value).toString()); + return this; + } + + public FluentConfig bufferFlushMillis(BufferFlushKind value) { + configAccumulator.put(BUFFER_FLUSH_MS_PROPERTY_NAME, value.name().toLowerCase(Locale.ROOT)); + return this; + } + + public FluentConfig persistentClientExpiration(String expiration) { + configAccumulator.put(PERSISTENT_CLIENT_EXPIRATION_PROPERTY_NAME, expiration); + return this; + } + + public FluentConfig sessionQueueSize(int value) { + configAccumulator.put(SESSION_QUEUE_SIZE, Integer.valueOf(value).toString()); + return this; + } + + public FluentConfig disableTelemetry() { + configAccumulator.put(ENABLE_TELEMETRY_NAME, "false"); + return this; + } + + public FluentConfig enableTelemetry() { + configAccumulator.put(ENABLE_TELEMETRY_NAME, "true"); + return this; + } + + public class TLSConfig { + + private SSLProvider providerType; + private KeyStoreType keyStoreType; + private String keyStorePassword; + private String keyManagerPassword; + private String jksPath; + private int sslPort; + + private TLSConfig() {} + + public void port(int port) { + this.sslPort = port; + validatePort(port); + } + + public void sslProvider(SSLProvider providerType) { + this.providerType = providerType; + } + + public void jksPath(String jksPath) { + this.jksPath = jksPath; + } + + public void jksPath(Path jksPath) { + jksPath(jksPath.toAbsolutePath().toString()); + } + + public void keyStoreType(KeyStoreType keyStoreType) { + this.keyStoreType = keyStoreType; + } + + /** + * @param keyStorePassword the password to access the KeyStore + * */ + public void keyStorePassword(String keyStorePassword) { + this.keyStorePassword = keyStorePassword; + } + + /** + * @param keyManagerPassword the password to access the key manager. + * */ + public void keyManagerPassword(String keyManagerPassword) { + this.keyManagerPassword = keyManagerPassword; + } + + private String getSslProvider() { + return providerType.name().toLowerCase(Locale.ROOT); + } + + private String getJksPath() { + return jksPath; + } + + private String getKeyStoreType() { + return keyStoreType.name().toLowerCase(Locale.ROOT); + } + } + + public FluentConfig withTLS(Consumer tlsBlock) { + tlsConfig = new TLSConfig(); + tlsBlock.accept(tlsConfig); + configAccumulator.put(SSL_PORT_PROPERTY_NAME, Integer.toString(tlsConfig.sslPort)); + configAccumulator.put(SSL_PROVIDER, tlsConfig.getSslProvider()); + configAccumulator.put(JKS_PATH_PROPERTY_NAME, tlsConfig.getJksPath()); + configAccumulator.put(KEY_STORE_TYPE, tlsConfig.getKeyStoreType()); + configAccumulator.put(KEY_STORE_PASSWORD_PROPERTY_NAME, tlsConfig.keyStorePassword); + configAccumulator.put(KEY_MANAGER_PASSWORD_PROPERTY_NAME, tlsConfig.keyManagerPassword); + + return this; + } + + public IConfig build() { + if (creationKind != CreationKind.API) { + throw new IllegalStateException("Can't build a configuration started directly by the server, use startServer method instead"); + } + return new MemoryConfig(configAccumulator); + } + + public Server startServer() throws IOException { + if (creationKind != CreationKind.SERVER) { + throw new IllegalStateException("Can't start a sever from a configuration used in API mode, use build method instead"); + } + server.startServer(new MemoryConfig(configAccumulator)); + return server; + } +} diff --git a/moquette-0.17/broker/src/main/java/io/moquette/broker/config/IConfig.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/config/IConfig.java new file mode 100644 index 00000000..d09fedb1 --- /dev/null +++ b/moquette-0.17/broker/src/main/java/io/moquette/broker/config/IConfig.java @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2012-2018 The original author or authors + * ------------------------------------------------------ + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Apache License v2.0 which accompanies this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * The Apache License v2.0 is available at + * http://www.opensource.org/licenses/apache2.0.php + * + * You may elect to redistribute this code under either of these licenses. + */ + +package io.moquette.broker.config; + +import io.moquette.BrokerConstants; + +import java.time.Duration; +import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalUnit; + +/** + * Base interface for all configuration implementations (filesystem, memory or classpath) + */ +public abstract class IConfig { + + public static final String DEFAULT_CONFIG = "config/moquette.conf"; + public static final String PORT_PROPERTY_NAME = "port"; + public static final String HOST_PROPERTY_NAME = "host"; + public static final String PASSWORD_FILE_PROPERTY_NAME = "password_file"; + public static final String ALLOW_ANONYMOUS_PROPERTY_NAME = "allow_anonymous"; + public static final String AUTHENTICATOR_CLASS_NAME = "authenticator_class"; + public static final String AUTHORIZATOR_CLASS_NAME = "authorizator_class"; + public static final String PERSISTENT_QUEUE_TYPE_PROPERTY_NAME = "persistent_queue_type"; // h2 or segmented, default h2 + public static final String DATA_PATH_PROPERTY_NAME = "data_path"; + public static final String PERSISTENCE_ENABLED_PROPERTY_NAME = "persistence_enabled"; // true or false, default true + /** + * 0/immediate means immediate flush, like immediate_buffer_flush = true + * -1/full means no explicit flush, let Netty flush when write buffers are full, like immediate_buffer_flush = false + * a number of milliseconds to between flushes + * */ + public static final String BUFFER_FLUSH_MS_PROPERTY_NAME = "buffer_flush_millis"; + public static final String WEB_SOCKET_PORT_PROPERTY_NAME = "websocket_port"; + public static final String WSS_PORT_PROPERTY_NAME = "secure_websocket_port"; + public static final String WEB_SOCKET_PATH_PROPERTY_NAME = "websocket_path"; + public static final String ACL_FILE_PROPERTY_NAME = "acl_file"; + public static final String PERSISTENT_CLIENT_EXPIRATION_PROPERTY_NAME = "persistent_client_expiration"; + public static final String SESSION_QUEUE_SIZE = "session_queue_size"; + public static final String ENABLE_TELEMETRY_NAME = "telemetry_enabled"; + /** + * Defines the SSL implementation to use, default to "JDK". + * @see io.netty.handler.ssl.SslProvider#name() + */ + public static final String SSL_PROVIDER = "ssl_provider"; + public static final String SSL_PORT_PROPERTY_NAME = "ssl_port"; + public static final String JKS_PATH_PROPERTY_NAME = "jks_path"; + + /** @see java.security.KeyStore#getInstance(String) for allowed types, default to "jks" */ + public static final String KEY_STORE_TYPE = "key_store_type"; + public static final String KEY_STORE_PASSWORD_PROPERTY_NAME = "key_store_password"; + public static final String KEY_MANAGER_PASSWORD_PROPERTY_NAME = "key_manager_password"; + public static final String NETTY_MAX_BYTES_PROPERTY_NAME = "netty.mqtt.message_size"; + public static final int DEFAULT_NETTY_MAX_BYTES_IN_MESSAGE = 8092; + + public abstract void setProperty(String name, String value); + + /** + * Same semantic of Properties + * + * @param name property name. + * @return property value null if not found. + * */ + public abstract String getProperty(String name); + + /** + * Same semantic of Properties + * + * @param name property name. + * @param defaultValue default value to return in case the property doesn't exist. + * @return property value. + * */ + public abstract String getProperty(String name, String defaultValue); + + void assignDefaults() { + setProperty(PORT_PROPERTY_NAME, Integer.toString(BrokerConstants.PORT)); + setProperty(HOST_PROPERTY_NAME, BrokerConstants.HOST); + // setProperty(BrokerConstants.WEB_SOCKET_PORT_PROPERTY_NAME, + // Integer.toString(BrokerConstants.WEBSOCKET_PORT)); + setProperty(PASSWORD_FILE_PROPERTY_NAME, ""); + // setProperty(BrokerConstants.PERSISTENT_STORE_PROPERTY_NAME, + // BrokerConstants.DEFAULT_PERSISTENT_PATH); + setProperty(ALLOW_ANONYMOUS_PROPERTY_NAME, Boolean.TRUE.toString()); + setProperty(AUTHENTICATOR_CLASS_NAME, ""); + setProperty(AUTHORIZATOR_CLASS_NAME, ""); + setProperty(NETTY_MAX_BYTES_PROPERTY_NAME, String.valueOf(DEFAULT_NETTY_MAX_BYTES_IN_MESSAGE)); + setProperty(PERSISTENT_QUEUE_TYPE_PROPERTY_NAME, "segmented"); + setProperty(DATA_PATH_PROPERTY_NAME, "data/"); + setProperty(PERSISTENCE_ENABLED_PROPERTY_NAME, Boolean.TRUE.toString()); + } + + public abstract IResourceLoader getResourceLoader(); + + public int intProp(String propertyName, int defaultValue) { + String propertyValue = getProperty(propertyName); + if (propertyValue == null) { + return defaultValue; + } + return Integer.parseInt(propertyValue); + } + + public boolean boolProp(String propertyName, boolean defaultValue) { + String propertyValue = getProperty(propertyName); + if (propertyValue == null) { + return defaultValue; + } + return Boolean.parseBoolean(propertyValue); + } + + public Duration durationProp(String propertyName) { + String propertyValue = getProperty(propertyName); + final char timeSpecifier = propertyValue.charAt(propertyValue.length() - 1); + final TemporalUnit periodType; + switch (timeSpecifier) { + case 's': + periodType = ChronoUnit.SECONDS; + break; + case 'm': + periodType = ChronoUnit.MINUTES; + break; + case 'h': + periodType = ChronoUnit.HOURS; + break; + case 'd': + periodType = ChronoUnit.DAYS; + break; + case 'w': + periodType = ChronoUnit.WEEKS; + break; + case 'M': + periodType = ChronoUnit.MONTHS; + break; + case 'y': + periodType = ChronoUnit.YEARS; + break; + default: + throw new IllegalStateException("Can' parse duration property " + propertyName + " with value: " + propertyValue + ", admitted only h, d, w, m, y"); + + } + return Duration.of(Integer.parseInt(propertyValue.substring(0, propertyValue.length() - 1)), periodType); + } +} diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/config/IResourceLoader.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/config/IResourceLoader.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/config/IResourceLoader.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/config/IResourceLoader.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/config/MemoryConfig.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/config/MemoryConfig.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/config/MemoryConfig.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/config/MemoryConfig.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/config/ResourceLoaderConfig.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/config/ResourceLoaderConfig.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/config/ResourceLoaderConfig.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/config/ResourceLoaderConfig.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/metrics/BytesMetrics.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/metrics/BytesMetrics.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/metrics/BytesMetrics.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/metrics/BytesMetrics.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/metrics/BytesMetricsCollector.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/metrics/BytesMetricsCollector.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/metrics/BytesMetricsCollector.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/metrics/BytesMetricsCollector.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/metrics/BytesMetricsHandler.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/metrics/BytesMetricsHandler.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/metrics/BytesMetricsHandler.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/metrics/BytesMetricsHandler.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/metrics/DropWizardMetricsHandler.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/metrics/DropWizardMetricsHandler.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/metrics/DropWizardMetricsHandler.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/metrics/DropWizardMetricsHandler.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/metrics/MQTTMessageLogger.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/metrics/MQTTMessageLogger.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/metrics/MQTTMessageLogger.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/metrics/MQTTMessageLogger.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/metrics/MessageMetrics.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/metrics/MessageMetrics.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/metrics/MessageMetrics.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/metrics/MessageMetrics.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/metrics/MessageMetricsCollector.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/metrics/MessageMetricsCollector.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/metrics/MessageMetricsCollector.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/metrics/MessageMetricsCollector.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/metrics/MessageMetricsHandler.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/metrics/MessageMetricsHandler.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/metrics/MessageMetricsHandler.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/metrics/MessageMetricsHandler.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/security/ACLFileParser.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/security/ACLFileParser.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/security/ACLFileParser.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/security/ACLFileParser.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/security/AcceptAllAuthenticator.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/security/AcceptAllAuthenticator.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/security/AcceptAllAuthenticator.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/security/AcceptAllAuthenticator.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/security/Authorization.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/security/Authorization.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/security/Authorization.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/security/Authorization.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/security/AuthorizationsCollector.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/security/AuthorizationsCollector.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/security/AuthorizationsCollector.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/security/AuthorizationsCollector.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/security/DBAuthenticator.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/security/DBAuthenticator.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/security/DBAuthenticator.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/security/DBAuthenticator.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/security/DenyAllAuthorizatorPolicy.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/security/DenyAllAuthorizatorPolicy.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/security/DenyAllAuthorizatorPolicy.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/security/DenyAllAuthorizatorPolicy.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/security/FileAuthenticator.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/security/FileAuthenticator.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/security/FileAuthenticator.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/security/FileAuthenticator.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/security/IAuthenticator.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/security/IAuthenticator.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/security/IAuthenticator.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/security/IAuthenticator.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/security/IAuthorizatorPolicy.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/security/IAuthorizatorPolicy.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/security/IAuthorizatorPolicy.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/security/IAuthorizatorPolicy.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/security/PemUtils.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/security/PemUtils.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/security/PemUtils.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/security/PemUtils.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/security/PermitAllAuthorizatorPolicy.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/security/PermitAllAuthorizatorPolicy.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/security/PermitAllAuthorizatorPolicy.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/security/PermitAllAuthorizatorPolicy.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/security/ResourceAuthenticator.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/security/ResourceAuthenticator.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/security/ResourceAuthenticator.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/security/ResourceAuthenticator.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/subscriptions/CNode.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/subscriptions/CNode.java similarity index 63% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/subscriptions/CNode.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/subscriptions/CNode.java index a356fc1d..be2020d9 100644 --- a/moquette-0.16/broker/src/main/java/io/moquette/broker/subscriptions/CNode.java +++ b/moquette-0.17/broker/src/main/java/io/moquette/broker/subscriptions/CNode.java @@ -17,21 +17,22 @@ import java.util.*; -class CNode { +class CNode implements Comparable { - private Token token; - private List children; - Set subscriptions; + private final Token token; + private final List children; + List subscriptions; - CNode() { + CNode(Token token) { this.children = new ArrayList<>(); - this.subscriptions = new HashSet<>(); + this.subscriptions = new ArrayList<>(); + this.token = token; } //Copy constructor - private CNode(Token token, List children, Set subscriptions) { + private CNode(Token token, List children, List subscriptions) { this.token = token; // keep reference, root comparison in directory logic relies on it for now. - this.subscriptions = new HashSet<>(subscriptions); + this.subscriptions = new ArrayList<>(subscriptions); this.children = new ArrayList<>(children); } @@ -39,32 +40,21 @@ public Token getToken() { return token; } - public void setToken(Token token) { - this.token = token; + List allChildren() { + return new ArrayList<>(this.children); } - boolean anyChildrenMatch(Token token) { - for (INode iNode : children) { - final CNode child = iNode.mainNode(); - if (child.equalsToken(token)) { - return true; - } + Optional childOf(Token token) { + int idx = findIndexForToken(token); + if (idx < 0) { + return Optional.empty(); } - return false; - } - - List allChildren() { - return this.children; + return Optional.of(children.get(idx)); } - INode childOf(Token token) { - for (INode iNode : children) { - final CNode child = iNode.mainNode(); - if (child.equalsToken(token)) { - return iNode; - } - } - throw new IllegalArgumentException("Asked for a token that doesn't exists in any child [" + token + "]"); + private int findIndexForToken(Token token) { + final INode tempTokenNode = new INode(new CNode(token)); + return Collections.binarySearch(children, tempTokenNode, (INode node, INode tokenHolder) -> node.mainNode().token.compareTo(tokenHolder.mainNode().token)); } private boolean equalsToken(Token token) { @@ -81,25 +71,30 @@ CNode copy() { } public void add(INode newINode) { - this.children.add(newINode); + int idx = findIndexForToken(newINode.mainNode().token); + if (idx < 0) { + children.add(-1 - idx, newINode); + } else { + children.add(idx, newINode); + } } public void remove(INode node) { - this.children.remove(node); + int idx = findIndexForToken(node.mainNode().token); + this.children.remove(idx); } CNode addSubscription(Subscription newSubscription) { // if already contains one with same topic and same client, keep that with higher QoS - if (subscriptions.contains(newSubscription)) { - final Subscription existing = subscriptions.stream() - .filter(s -> s.equals(newSubscription)) - .findFirst().get(); + int idx = Collections.binarySearch(subscriptions, newSubscription); + if (idx >= 0) { + // Subscription already exists + final Subscription existing = subscriptions.get(idx); if (existing.getRequestedQos().value() < newSubscription.getRequestedQos().value()) { - subscriptions.remove(existing); - subscriptions.add(new Subscription(newSubscription)); + subscriptions.set(idx, newSubscription); } } else { - this.subscriptions.add(new Subscription(newSubscription)); + this.subscriptions.add(-1 - idx, new Subscription(newSubscription)); } return this; } @@ -136,4 +131,9 @@ void removeSubscriptionsFor(String clientId) { } this.subscriptions.removeAll(toRemove); } + + @Override + public int compareTo(CNode o) { + return token.compareTo(o.token); + } } diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/subscriptions/CTrie.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/subscriptions/CTrie.java similarity index 55% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/subscriptions/CTrie.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/subscriptions/CTrie.java index 1a64a849..70ba7d30 100644 --- a/moquette-0.16/broker/src/main/java/io/moquette/broker/subscriptions/CTrie.java +++ b/moquette-0.17/broker/src/main/java/io/moquette/broker/subscriptions/CTrie.java @@ -1,9 +1,9 @@ package io.moquette.broker.subscriptions; +import java.util.ArrayList; import java.util.Collections; -import java.util.HashSet; +import java.util.List; import java.util.Optional; -import java.util.Set; public class CTrie { @@ -24,17 +24,20 @@ private enum Action { INode root; CTrie() { - final CNode mainNode = new CNode(); - mainNode.setToken(ROOT); + final CNode mainNode = new CNode(ROOT); this.root = new INode(mainNode); } Optional lookup(Topic topic) { INode inode = this.root; Token token = topic.headToken(); - while (!topic.isEmpty() && (inode.mainNode().anyChildrenMatch(token))) { + while (!topic.isEmpty()) { + Optional child = inode.mainNode().childOf(token); + if (!child.isPresent()) { + break; + } topic = topic.exceptHeadToken(); - inode = inode.mainNode().childOf(token); + inode = child.get(); token = topic.headToken(); } if (inode == null || !topic.isEmpty()) { @@ -61,29 +64,42 @@ private NavigationAction evaluate(Topic topic, CNode cnode) { return NavigationAction.GODEEP; } - public Set recursiveMatch(Topic topic) { + public List recursiveMatch(Topic topic) { return recursiveMatch(topic, this.root); } - private Set recursiveMatch(Topic topic, INode inode) { + private List recursiveMatch(Topic topic, INode inode) { CNode cnode = inode.mainNode(); if (cnode instanceof TNode) { - return Collections.emptySet(); + return Collections.emptyList(); } NavigationAction action = evaluate(topic, cnode); if (action == NavigationAction.MATCH) { return cnode.subscriptions; } if (action == NavigationAction.STOP) { - return Collections.emptySet(); + return Collections.emptyList(); } Topic remainingTopic = (ROOT.equals(cnode.getToken())) ? topic : topic.exceptHeadToken(); - Set subscriptions = new HashSet<>(); + List subscriptions = new ArrayList<>(); + + // We should only consider the maximum three children children of + // type #, + or exact match + Optional subInode = cnode.childOf(Token.MULTI); + if (subInode.isPresent()) { + subscriptions.addAll(recursiveMatch(remainingTopic, subInode.get())); + } + subInode = cnode.childOf(Token.SINGLE); + if (subInode.isPresent()) { + subscriptions.addAll(recursiveMatch(remainingTopic, subInode.get())); + } if (remainingTopic.isEmpty()) { subscriptions.addAll(cnode.subscriptions); - } - for (INode subInode : cnode.allChildren()) { - subscriptions.addAll(recursiveMatch(remainingTopic, subInode)); + } else { + subInode = cnode.childOf(remainingTopic.headToken()); + if (subInode.isPresent()) { + subscriptions.addAll(recursiveMatch(remainingTopic, subInode.get())); + } } return subscriptions; } @@ -96,34 +112,41 @@ public void addToTree(Subscription newSubscription) { } private Action insert(Topic topic, final INode inode, Subscription newSubscription) { - Token token = topic.headToken(); - if (!topic.isEmpty() && inode.mainNode().anyChildrenMatch(token)) { - Topic remainingTopic = topic.exceptHeadToken(); - INode nextInode = inode.mainNode().childOf(token); - return insert(remainingTopic, nextInode, newSubscription); - } else { - if (topic.isEmpty()) { - return insertSubscription(inode, newSubscription); - } else { - return createNodeAndInsertSubscription(topic, inode, newSubscription); + final Token token = topic.headToken(); + final CNode cnode = inode.mainNode(); + if (!topic.isEmpty()) { + Optional nextInode = cnode.childOf(token); + if (nextInode.isPresent()) { + Topic remainingTopic = topic.exceptHeadToken(); + return insert(remainingTopic, nextInode.get(), newSubscription); } } + if (topic.isEmpty()) { + return insertSubscription(inode, cnode, newSubscription); + } else { + return createNodeAndInsertSubscription(topic, inode, cnode, newSubscription); + } } - private Action insertSubscription(INode inode, Subscription newSubscription) { - CNode cnode = inode.mainNode(); - CNode updatedCnode = cnode.copy().addSubscription(newSubscription); - if (inode.compareAndSet(cnode, updatedCnode)) { - return Action.OK; + private Action insertSubscription(INode inode, CNode cnode, Subscription newSubscription) { + final CNode updatedCnode; + if (cnode instanceof TNode) { + updatedCnode = new CNode(cnode.getToken()); } else { - return Action.REPEAT; + updatedCnode = cnode.copy(); } + updatedCnode.addSubscription(newSubscription); + return inode.compareAndSet(cnode, updatedCnode) ? Action.OK : Action.REPEAT; } - private Action createNodeAndInsertSubscription(Topic topic, INode inode, Subscription newSubscription) { - INode newInode = createPathRec(topic, newSubscription); - CNode cnode = inode.mainNode(); - CNode updatedCnode = cnode.copy(); + private Action createNodeAndInsertSubscription(Topic topic, INode inode, CNode cnode, Subscription newSubscription) { + final INode newInode = createPathRec(topic, newSubscription); + final CNode updatedCnode; + if (cnode instanceof TNode) { + updatedCnode = new CNode(cnode.getToken()); + } else { + updatedCnode = cnode.copy(); + } updatedCnode.add(newInode); return inode.compareAndSet(cnode, updatedCnode) ? Action.OK : Action.REPEAT; @@ -133,8 +156,7 @@ private INode createPathRec(Topic topic, Subscription newSubscription) { Topic remainingTopic = topic.exceptHeadToken(); if (!remainingTopic.isEmpty()) { INode inode = createPathRec(remainingTopic, newSubscription); - CNode cnode = new CNode(); - cnode.setToken(topic.headToken()); + CNode cnode = new CNode(topic.headToken()); cnode.add(inode); return new INode(cnode); } else { @@ -143,8 +165,7 @@ private INode createPathRec(Topic topic, Subscription newSubscription) { } private INode createLeafNodes(Token token, Subscription newSubscription) { - CNode newLeafCnode = new CNode(); - newLeafCnode.setToken(token); + CNode newLeafCnode = new CNode(token); newLeafCnode.addSubscription(newSubscription); return new INode(newLeafCnode); @@ -159,33 +180,34 @@ public void removeFromTree(Topic topic, String clientID) { private Action remove(String clientId, Topic topic, INode inode, INode iParent) { Token token = topic.headToken(); - if (!topic.isEmpty() && (inode.mainNode().anyChildrenMatch(token))) { - Topic remainingTopic = topic.exceptHeadToken(); - INode nextInode = inode.mainNode().childOf(token); - return remove(clientId, remainingTopic, nextInode, inode); - } else { - final CNode cnode = inode.mainNode(); - if (cnode instanceof TNode) { - // this inode is a tomb, has no clients and should be cleaned up - // Because we implemented cleanTomb below, this should be rare, but possible - // Consider calling cleanTomb here too - return Action.OK; + final CNode cnode = inode.mainNode(); + if (!topic.isEmpty()) { + Optional nextInode = cnode.childOf(token); + if (nextInode.isPresent()) { + Topic remainingTopic = topic.exceptHeadToken(); + return remove(clientId, remainingTopic, nextInode.get(), inode); } - if (cnode.containsOnly(clientId) && topic.isEmpty() && cnode.allChildren().isEmpty()) { - // last client to leave this node, AND there are no downstream children, remove via TNode tomb - if (inode == this.root) { - return inode.compareAndSet(cnode, inode.mainNode().copy()) ? Action.OK : Action.REPEAT; - } - TNode tnode = new TNode(); - return inode.compareAndSet(cnode, tnode) ? cleanTomb(inode, iParent) : Action.REPEAT; - } else if (cnode.contains(clientId) && topic.isEmpty()) { - CNode updatedCnode = cnode.copy(); - updatedCnode.removeSubscriptionsFor(clientId); - return inode.compareAndSet(cnode, updatedCnode) ? Action.OK : Action.REPEAT; - } else { - //someone else already removed - return Action.OK; + } + if (cnode instanceof TNode) { + // this inode is a tomb, has no clients and should be cleaned up + // Because we implemented cleanTomb below, this should be rare, but possible + // Consider calling cleanTomb here too + return Action.OK; + } + if (cnode.containsOnly(clientId) && topic.isEmpty() && cnode.allChildren().isEmpty()) { + // last client to leave this node, AND there are no downstream children, remove via TNode tomb + if (inode == this.root) { + return inode.compareAndSet(cnode, inode.mainNode().copy()) ? Action.OK : Action.REPEAT; } + TNode tnode = new TNode(cnode.getToken()); + return inode.compareAndSet(cnode, tnode) ? cleanTomb(inode, iParent) : Action.REPEAT; + } else if (cnode.contains(clientId) && topic.isEmpty()) { + CNode updatedCnode = cnode.copy(); + updatedCnode.removeSubscriptionsFor(clientId); + return inode.compareAndSet(cnode, updatedCnode) ? Action.OK : Action.REPEAT; + } else { + //someone else already removed + return Action.OK; } } diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/subscriptions/CTrieSubscriptionDirectory.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/subscriptions/CTrieSubscriptionDirectory.java similarity index 93% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/subscriptions/CTrieSubscriptionDirectory.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/subscriptions/CTrieSubscriptionDirectory.java index 00673538..5ce0f32e 100644 --- a/moquette-0.16/broker/src/main/java/io/moquette/broker/subscriptions/CTrieSubscriptionDirectory.java +++ b/moquette-0.17/broker/src/main/java/io/moquette/broker/subscriptions/CTrieSubscriptionDirectory.java @@ -76,13 +76,13 @@ Optional lookup(Topic topic) { * @return the list of matching subscriptions, or empty if not matching. */ @Override - public Set matchWithoutQosSharpening(Topic topic) { + public List matchWithoutQosSharpening(Topic topic) { return ctrie.recursiveMatch(topic); } @Override - public Set matchQosSharpening(Topic topic) { - final Set subscriptions = matchWithoutQosSharpening(topic); + public List matchQosSharpening(Topic topic) { + final List subscriptions = matchWithoutQosSharpening(topic); Map subsGroupedByClient = new HashMap<>(); for (Subscription sub : subscriptions) { @@ -92,7 +92,7 @@ public Set matchQosSharpening(Topic topic) { subsGroupedByClient.put(sub.clientId, sub); } } - return new HashSet<>(subsGroupedByClient.values()); + return new ArrayList<>(subsGroupedByClient.values()); } @Override diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/subscriptions/DumpTreeVisitor.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/subscriptions/DumpTreeVisitor.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/subscriptions/DumpTreeVisitor.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/subscriptions/DumpTreeVisitor.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/subscriptions/INode.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/subscriptions/INode.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/subscriptions/INode.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/subscriptions/INode.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/subscriptions/ISubscriptionsDirectory.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/subscriptions/ISubscriptionsDirectory.java similarity index 89% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/subscriptions/ISubscriptionsDirectory.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/subscriptions/ISubscriptionsDirectory.java index 97f32001..a524e550 100644 --- a/moquette-0.16/broker/src/main/java/io/moquette/broker/subscriptions/ISubscriptionsDirectory.java +++ b/moquette-0.17/broker/src/main/java/io/moquette/broker/subscriptions/ISubscriptionsDirectory.java @@ -26,9 +26,9 @@ public interface ISubscriptionsDirectory { Set listAllSessionIds(); - Set matchWithoutQosSharpening(Topic topic); + List matchWithoutQosSharpening(Topic topic); - Set matchQosSharpening(Topic topic); + List matchQosSharpening(Topic topic); void add(Subscription newSubscription); diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/subscriptions/Subscription.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/subscriptions/Subscription.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/subscriptions/Subscription.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/subscriptions/Subscription.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/subscriptions/SubscriptionCounterVisitor.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/subscriptions/SubscriptionCounterVisitor.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/subscriptions/SubscriptionCounterVisitor.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/subscriptions/SubscriptionCounterVisitor.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/subscriptions/TNode.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/subscriptions/TNode.java similarity index 81% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/subscriptions/TNode.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/subscriptions/TNode.java index 6c073d3f..2392b7c2 100644 --- a/moquette-0.16/broker/src/main/java/io/moquette/broker/subscriptions/TNode.java +++ b/moquette-0.17/broker/src/main/java/io/moquette/broker/subscriptions/TNode.java @@ -15,20 +15,16 @@ */ package io.moquette.broker.subscriptions; +import java.util.Optional; + class TNode extends CNode { - @Override - public Token getToken() { - throw new IllegalStateException("Can't be invoked on TNode"); + public TNode(Token token) { + super(token); } @Override - public void setToken(Token token) { - throw new IllegalStateException("Can't be invoked on TNode"); - } - - @Override - INode childOf(Token token) { + Optional childOf(Token token) { throw new IllegalStateException("Can't be invoked on TNode"); } @@ -62,8 +58,4 @@ void removeSubscriptionsFor(String clientId) { throw new IllegalStateException("Can't be invoked on TNode"); } - @Override - boolean anyChildrenMatch(Token token) { - return false; - } } diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/subscriptions/Token.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/subscriptions/Token.java similarity index 83% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/subscriptions/Token.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/subscriptions/Token.java index 550a6004..b8ccb00e 100644 --- a/moquette-0.16/broker/src/main/java/io/moquette/broker/subscriptions/Token.java +++ b/moquette-0.17/broker/src/main/java/io/moquette/broker/subscriptions/Token.java @@ -19,7 +19,7 @@ /** * Internal use only class. */ -public class Token { +public class Token implements Comparable { static final Token EMPTY = new Token(""); static final Token MULTI = new Token("#"); @@ -72,4 +72,18 @@ public boolean equals(Object obj) { public String toString() { return name; } + + @Override + public int compareTo(Token other) { + if (name == null) { + if (other.name == null) { + return 0; + } + return 1; + } + if (other.name == null) { + return -1; + } + return name.compareTo(other.name); + } } diff --git a/moquette-0.16/broker/src/main/java/io/moquette/broker/subscriptions/Topic.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/subscriptions/Topic.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/broker/subscriptions/Topic.java rename to moquette-0.17/broker/src/main/java/io/moquette/broker/subscriptions/Topic.java diff --git a/moquette-0.17/broker/src/main/java/io/moquette/broker/unsafequeues/PagedFilesAllocator.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/unsafequeues/PagedFilesAllocator.java new file mode 100644 index 00000000..f005fe7b --- /dev/null +++ b/moquette-0.17/broker/src/main/java/io/moquette/broker/unsafequeues/PagedFilesAllocator.java @@ -0,0 +1,123 @@ +package io.moquette.broker.unsafequeues; + +import java.io.IOException; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Properties; + +/** + * Default implementation of SegmentAllocator. It uses a series of files (named pages) and split them in segments. + * + * This class is not thread safe. + * */ +class PagedFilesAllocator implements SegmentAllocator { + + interface AllocationListener { + void segmentedCreated(String name, Segment segment); + } + + private final Path pagesFolder; + private final int pageSize; + private final int segmentSize; + private int lastSegmentAllocated; + private int lastPage; + private MappedByteBuffer currentPage; + private FileChannel currentPageFile; + + PagedFilesAllocator(Path pagesFolder, int pageSize, int segmentSize, int lastPage, int lastSegmentAllocated) throws QueueException { + if (pageSize % segmentSize != 0) { + throw new IllegalArgumentException("The pageSize must be an exact multiple of the segmentSize"); + } + this.pagesFolder = pagesFolder; + this.pageSize = pageSize; + this.segmentSize = segmentSize; + this.lastPage = lastPage; + this.lastSegmentAllocated = lastSegmentAllocated; + this.currentPage = openRWPageFile(this.pagesFolder, this.lastPage); + } + + private MappedByteBuffer openRWPageFile(Path pagesFolder, int pageId) throws QueueException { + final Path pageFile = pagesFolder.resolve(String.format("%d.page", pageId)); + boolean createNew = false; + if (!Files.exists(pageFile)) { + try { + pageFile.toFile().createNewFile(); + createNew = true; + } catch (IOException ex) { + throw new QueueException("Reached an IO error during the bootstrapping of empty 'checkpoint.properties'", ex); + } + } + + try (FileChannel fileChannel = FileChannel.open(pageFile, StandardOpenOption.READ, StandardOpenOption.WRITE)) { + this.currentPageFile = fileChannel; + final MappedByteBuffer mappedPage = fileChannel.map(FileChannel.MapMode.READ_WRITE, 0, pageSize); + // DBG + if (createNew && QueuePool.queueDebug) { + for (int i = 0; i < pageSize; i++) { + mappedPage.put(i, (byte) 'C'); + } + } + // DBG + return mappedPage; + } catch (IOException e) { + throw new QueueException("Can't open page file " + pageFile, e); + } + } + + @Override + public Segment nextFreeSegment() throws QueueException { + if (currentPageIsExhausted()) { + lastPage++; + currentPage = openRWPageFile(pagesFolder, lastPage); + lastSegmentAllocated = 0; + } + + final int beginOffset = lastSegmentAllocated * segmentSize; + final int endOffset = ((lastSegmentAllocated + 1) * segmentSize) - 1; + + lastSegmentAllocated += 1; + return new Segment(currentPage, new SegmentPointer(lastPage, beginOffset), new SegmentPointer(lastPage, endOffset)); + } + + @Override + public Segment reopenSegment(int pageId, int beginOffset) throws QueueException { + final MappedByteBuffer page = openRWPageFile(pagesFolder, pageId); + final SegmentPointer begin = new SegmentPointer(pageId, beginOffset); + final SegmentPointer end = new SegmentPointer(pageId, beginOffset + segmentSize - 1); + return new Segment(page, begin, end); + } + + @Override + public void close() throws QueueException { + if (currentPageFile != null) { + try { + currentPageFile.close(); + } catch (IOException ex) { + throw new QueueException("Problem closing current page file", ex); + } + } + } + + @Override + public void dumpState(Properties checkpoint) { + checkpoint.setProperty("segments.last_page", String.valueOf(this.lastPage)); + checkpoint.setProperty("segments.last_segment", String.valueOf(this.lastSegmentAllocated)); + } + + @Override + public int getPageSize() { + return pageSize; + } + + @Override + public int getSegmentSize() { + return segmentSize; + } + + private boolean currentPageIsExhausted() { + return lastSegmentAllocated * segmentSize == pageSize; + } +} diff --git a/moquette-0.17/broker/src/main/java/io/moquette/broker/unsafequeues/Queue.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/unsafequeues/Queue.java new file mode 100644 index 00000000..f44861cd --- /dev/null +++ b/moquette-0.17/broker/src/main/java/io/moquette/broker/unsafequeues/Queue.java @@ -0,0 +1,321 @@ +package io.moquette.broker.unsafequeues; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.Buffer; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +/** + * Not thread safe disk persisted queue. + * */ +public class Queue { + private static final Logger LOG = LoggerFactory.getLogger(Queue.class); + + public static final int LENGTH_HEADER_SIZE = 4; + private final String name; + /* Last wrote byte, point to head byte */ + private VirtualPointer currentHeadPtr; + private Segment headSegment; + + /* First readable byte, point to the last occupied byte */ + private VirtualPointer currentTailPtr; + private Segment tailSegment; + + private final QueuePool queuePool; + private final SegmentAllocator allocator; + private final PagedFilesAllocator.AllocationListener allocationListener; +// private final ReentrantLock lock = new ReentrantLock(); + + Queue(String name, Segment headSegment, VirtualPointer currentHeadPtr, + Segment tailSegment, VirtualPointer currentTailPtr, + SegmentAllocator allocator, PagedFilesAllocator.AllocationListener allocationListener, QueuePool queuePool) { + this.name = name; + this.headSegment = headSegment; + this.currentHeadPtr = currentHeadPtr; + this.currentTailPtr = currentTailPtr; + this.tailSegment = tailSegment; + this.allocator = allocator; + this.allocationListener = allocationListener; + this.queuePool = queuePool; + } + + /** + * @throws QueueException if an error happens during access to file. + * */ + public void enqueue(ByteBuffer payload) throws QueueException { + final int messageSize = LENGTH_HEADER_SIZE + payload.remaining(); + if (headSegment.hasSpace(currentHeadPtr, messageSize)) { + LOG.debug("Head segment has sufficient space for message length {}", LENGTH_HEADER_SIZE + payload.remaining()); + writeData(headSegment, currentHeadPtr.plus(1), payload); + // move head segment + currentHeadPtr = currentHeadPtr.moveForward(messageSize); + return; + } + + LOG.debug("Head segment doesn't have enough space"); + // the payload can't be fully contained into the current head segment and needs to be splitted + // with another segment. + + + final int dataSize = payload.remaining(); + final ByteBuffer rawData = (ByteBuffer) ByteBuffer.allocate(LENGTH_HEADER_SIZE + dataSize) + .putInt(dataSize) + .put(payload) + .flip(); + + // the bytes written from the payload input + long bytesRemainingInHeaderSegment = Math.min(rawData.remaining(), headSegment.bytesAfter(currentHeadPtr)); + LOG.trace("Writing partial payload to offset {} for {} bytes", currentHeadPtr, bytesRemainingInHeaderSegment); + + if (bytesRemainingInHeaderSegment > 0) { + int copySize = (int) bytesRemainingInHeaderSegment; + ByteBuffer slice = rawData.slice(); + slice.limit(copySize); + writeDataNoHeader(headSegment, currentHeadPtr.plus(1), slice); + currentHeadPtr = currentHeadPtr.moveForward(bytesRemainingInHeaderSegment); + // No need to move newSegmentPointer the pointer because the last spinningMove has already moved it + + // shift forward the consumption point + rawData.position(rawData.position() + copySize); + } + + Segment newSegment = null; + + // till the payload is not completely stored, + // save the remaining part into a new segment. + while (rawData.hasRemaining()) { + // To request the next segment, it's needed to be done in global lock. + newSegment = queuePool.nextFreeSegment(); + //notify segment creation for queue in queue pool + allocationListener.segmentedCreated(name, newSegment); + + int copySize = (int) Math.min(rawData.remaining(), allocator.getSegmentSize()); + ByteBuffer slice = rawData.slice(); + slice.limit(copySize); + + currentHeadPtr = currentHeadPtr.moveForward(copySize); + writeDataNoHeader(newSegment, newSegment.begin, slice); + headSegment = newSegment; + + // shift forward the consumption point + rawData.position(rawData.position() + copySize); + } + } + + private void writeDataNoHeader(Segment segment, SegmentPointer start, ByteBuffer data) { + segment.write(start, data); + } + + private void writeDataNoHeader(Segment segment, VirtualPointer start, ByteBuffer data) { + segment.write(start, data); + } + + /** + * Writes data and size to the current Head segment starting from start pointer. + * */ + private void writeData(Segment segment, VirtualPointer start, ByteBuffer data) { + writeData(segment, start, data.remaining(), data); + } + + /** + * @param segment the target segment. + * @param start where start writing. + * @param size the length of the data to write on the segment. + * @param data the data to write. + * */ + private void writeData(Segment segment, VirtualPointer start, int size, ByteBuffer data) { + ByteBuffer length = (ByteBuffer) ByteBuffer.allocate(LENGTH_HEADER_SIZE).putInt(size).flip(); + segment.write(start, length); // write 4 bytes header + segment.write(start.plus(LENGTH_HEADER_SIZE), data); // write the payload + } + + /** + * Used in test + * */ + void force() { + headSegment.force(); + } + + VirtualPointer currentHead() { + return currentHeadPtr; + } + + VirtualPointer currentTail() { + return currentTailPtr; + } + + public boolean isEmpty() { + if (isTailFirstUsage(currentTailPtr)) { + return currentHeadPtr.compareTo(currentTailPtr) == 0; + } else { + return currentHeadPtr.moveForward(1).compareTo(currentTailPtr) == 0; + } + } + + /** + * Read next message or return null if the queue has no data. + * */ + public Optional dequeue() throws QueueException { + if (!currentHeadPtr.isGreaterThan(currentTailPtr)) { + if (currentTailPtr.isGreaterThan(currentHeadPtr)) { + // sanity check + throw new QueueException("Current tail " + currentTailPtr + " is forward head " + currentHeadPtr); + } + // head and tail pointer are the same, the queue is empty + return Optional.empty(); + } + if (tailSegment == null) { + tailSegment = queuePool.openNextTailSegment(name).get(); + } + + LOG.debug("currentTail is {}", currentTailPtr); + if (containsHeader(tailSegment, currentTailPtr)) { + // currentSegment contains at least the header (payload length) + final VirtualPointer existingTail; + if (isTailFirstUsage(currentTailPtr)) { + // move to the first readable byte + existingTail = currentTailPtr.plus(1); + } else { + existingTail = currentTailPtr.copy(); + } + final int payloadLength = tailSegment.readHeader(existingTail); + // tail must be moved to the next byte to read, so has to move to + // header size + payload size + 1 + final int fullMessageSize = payloadLength + LENGTH_HEADER_SIZE; + long remainingInSegment = tailSegment.bytesAfter(existingTail) + 1; + if (remainingInSegment > fullMessageSize) { + // tail segment fully contains the payload with space left over + currentTailPtr = existingTail.moveForward(fullMessageSize); + // read data from currentTail + 4 bytes(the length) + final VirtualPointer dataStart = existingTail.moveForward(LENGTH_HEADER_SIZE); + + return Optional.of(readData(tailSegment, dataStart, payloadLength)); + } else { + // payload is split across currentSegment and next ones + VirtualPointer dataStart = existingTail.moveForward(LENGTH_HEADER_SIZE); + + if (remainingInSegment - LENGTH_HEADER_SIZE == 0) { + queuePool.consumedTailSegment(name); + if (QueuePool.queueDebug) { + tailSegment.fillWith((byte) 'D'); + } + tailSegment = queuePool.openNextTailSegment(name).get(); + } + + LOG.debug("Loading payload size {}", payloadLength); + return Optional.of(loadPayloadFromSegments(payloadLength, tailSegment, dataStart)); + } + } else { + // header is split across 2 segments + // the currentSegment is still the tailSegment + // read the length header that's crossing 2 segments + final CrossSegmentHeaderResult result = decodeCrossHeader(tailSegment, currentTailPtr); + + // load all payload parts from the segments + LOG.debug("Loading payload size {}", result.payloadLength); + return Optional.of(loadPayloadFromSegments(result.payloadLength, result.segment, result.pointer)); + } + } + + private static boolean containsHeader(Segment segment, VirtualPointer tail) { + return segment.bytesAfter(tail) + 1 >= LENGTH_HEADER_SIZE; + } + + private static class CrossSegmentHeaderResult { + private final Segment segment; + private final VirtualPointer pointer; + private final int payloadLength; + + private CrossSegmentHeaderResult(Segment segment, VirtualPointer pointer, int payloadLength) { + this.segment = segment; + this.pointer = pointer; + this.payloadLength = payloadLength; + } + } + + // TO BE called owning the lock + private CrossSegmentHeaderResult decodeCrossHeader(Segment segment, VirtualPointer pointer) throws QueueException { + // read first part + ByteBuffer lengthBuffer = ByteBuffer.allocate(LENGTH_HEADER_SIZE); + final ByteBuffer partialHeader = segment.readAllBytesAfter(pointer); + final int consumedHeaderSize = partialHeader.remaining(); + lengthBuffer.put(partialHeader); + queuePool.consumedTailSegment(name); + + if (QueuePool.queueDebug) { + segment.fillWith((byte) 'D'); + } + + // read second part + final int remainingHeaderSize = LENGTH_HEADER_SIZE - consumedHeaderSize; + Segment nextTailSegment = queuePool.openNextTailSegment(name).get(); + lengthBuffer.put(nextTailSegment.read(nextTailSegment.begin, remainingHeaderSize)); + final VirtualPointer dataStart = pointer.moveForward(LENGTH_HEADER_SIZE); + int payloadLength = ((ByteBuffer) lengthBuffer.flip()).getInt(); + + return new CrossSegmentHeaderResult(nextTailSegment, dataStart, payloadLength); + } + + // TO BE called owning the lock on segments allocator + private ByteBuffer loadPayloadFromSegments(int remaining, Segment segment, VirtualPointer tail) throws QueueException { + List createdBuffers = new ArrayList<>(segmentCountFromSize(remaining)); + VirtualPointer scan = tail; + + do { + LOG.debug("Looping remaining {}", remaining); + final int availableDataLength = Math.min(remaining, (int) segment.bytesAfter(scan) + 1); + final ByteBuffer buffer = segment.read(scan, availableDataLength); + createdBuffers.add(buffer); + final boolean segmentCompletelyConsumed = (segment.bytesAfter(scan) + 1) == availableDataLength; + scan = scan.moveForward(availableDataLength); + remaining -= buffer.remaining(); + + if (remaining > 0 || segmentCompletelyConsumed) { + queuePool.consumedTailSegment(name); + if (QueuePool.queueDebug) { + segment.fillWith((byte) 'D'); + } + segment = queuePool.openNextTailSegment(name).orElse(null); + } + } while (remaining > 0); + + // assign to tailSegment without CAS because we are in lock + tailSegment = segment; + currentTailPtr = scan; + LOG.debug("Moved currentTailPointer to {} from {}", scan, tail); + + return joinBuffers(createdBuffers); + } + + private int segmentCountFromSize(int remaining) { + return (int) Math.ceil((double) remaining / allocator.getSegmentSize()); + } + + private boolean isTailFirstUsage(VirtualPointer tail) { + return tail.isUntouched(); + } + + /** + * @return a ByteBuffer that's a composition of all buffers + * */ + private ByteBuffer joinBuffers(List buffers) { + final int neededSpace = buffers.stream().mapToInt(Buffer::remaining).sum(); + byte[] heapBuffer = new byte[neededSpace]; + int offset = 0; + for (ByteBuffer buffer : buffers) { + final int readBytes = buffer.remaining(); + buffer.get(heapBuffer, offset, readBytes); + offset += readBytes; + } + + return ByteBuffer.wrap(heapBuffer); + } + + private ByteBuffer readData(Segment source, VirtualPointer start, int length) { + return source.read(start, length); + } +} diff --git a/moquette-0.17/broker/src/main/java/io/moquette/broker/unsafequeues/QueueException.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/unsafequeues/QueueException.java new file mode 100644 index 00000000..bf233169 --- /dev/null +++ b/moquette-0.17/broker/src/main/java/io/moquette/broker/unsafequeues/QueueException.java @@ -0,0 +1,14 @@ +package io.moquette.broker.unsafequeues; + +public class QueueException extends Exception { + + private static final long serialVersionUID = -4782799401089093829L; + + public QueueException(String message, Throwable cause) { + super(message, cause); + } + + public QueueException(String message) { + super(message); + } +} diff --git a/moquette-0.17/broker/src/main/java/io/moquette/broker/unsafequeues/QueuePool.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/unsafequeues/QueuePool.java new file mode 100644 index 00000000..2b22f065 --- /dev/null +++ b/moquette-0.17/broker/src/main/java/io/moquette/broker/unsafequeues/QueuePool.java @@ -0,0 +1,490 @@ +package io.moquette.broker.unsafequeues; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Properties; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; + +public class QueuePool { + + private static final Logger LOG = LoggerFactory.getLogger(QueuePool.class); + + static final boolean queueDebug = Boolean.parseBoolean(System.getProperty("moquette.queue.debug", "false")); + + private final SegmentAllocationCallback callback; + + // visible for testing + static class SegmentRef implements Comparable { + final int pageId; + final int offset; + + // visible for testing + SegmentRef(int pageId, int offset) { + this.pageId = pageId; + this.offset = offset; + } + + public SegmentRef(Segment segment) { + this.pageId = segment.begin.pageId(); + this.offset = segment.begin.offset(); + } + + @Override + public String toString() { + return String.format("(%d, %d)", pageId, offset); + } + + @Override + public int compareTo(SegmentRef o) { + final int pageCompare = Integer.compare(pageId, o.pageId); + if (pageCompare != 0) { + return pageCompare; + } + return Integer.compare(offset, o.offset); + } + } + + private static class QueueName { + final String name; + + private QueueName(String name) { + this.name = name; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + QueueName queueName = (QueueName) o; + return Objects.equals(name, queueName.name); + } + + @Override + public int hashCode() { + return Objects.hash(name); + } + + @Override + public String toString() { + return "QueueName{name='" + name + '\'' + '}'; + } + } + + private final SegmentAllocator allocator; + private final Path dataPath; + private final int segmentSize; + private final ConcurrentMap> queueSegments = new ConcurrentHashMap<>(); + private final ConcurrentMap queues = new ConcurrentHashMap<>(); + private final ConcurrentSkipListSet recycledSegments = new ConcurrentSkipListSet<>(); + private final ReentrantLock segmentsAllocationLock = new ReentrantLock(); + + private QueuePool(SegmentAllocator allocator, Path dataPath, int segmentSize) { + this.allocator = allocator; + this.dataPath = dataPath; + this.segmentSize = segmentSize; + this.callback = new SegmentAllocationCallback(this); + } + + private static class SegmentAllocationCallback implements PagedFilesAllocator.AllocationListener { + + private final QueuePool queuePool; + + private SegmentAllocationCallback(QueuePool queuePool) { + this.queuePool = queuePool; + } + + @Override + public void segmentedCreated(String name, Segment segment) { + queuePool.segmentedCreated(name, segment); + } + } + + private void segmentedCreated(String name, Segment segment) { + LOG.debug("Registering new segment {} for queue {}", segment, name); + final QueueName queueName = new QueueName(name); + List segmentRefs = this.queueSegments.computeIfAbsent(queueName, k -> new LinkedList<>()); + + // adds in head + segmentRefs.add(0, new SegmentRef(segment)); + + LOG.debug("queueSegments for queue {} after insertion {}", queueName, segmentRefs); + } + + public static QueuePool loadQueues(Path dataPath, int pageSize, int segmentSize) throws QueueException { + // read in checkpoint.properties + final Properties checkpointProps = createOrLoadCheckpointFile(dataPath); + + // load last references to segment and instantiate the allocator + final int lastPage = Integer.parseInt(checkpointProps.getProperty("segments.last_page", "0")); + final int lastSegment = Integer.parseInt(checkpointProps.getProperty("segments.last_segment", "0")); + + final PagedFilesAllocator allocator = new PagedFilesAllocator(dataPath, pageSize, segmentSize, lastPage, lastSegment); + + final QueuePool queuePool = new QueuePool(allocator, dataPath, segmentSize); + queuePool.loadQueueDefinitions(checkpointProps); + LOG.debug("Loaded queues definitions: {}", queuePool.queueSegments); + + queuePool.loadRecycledSegments(checkpointProps); + LOG.debug("Recyclable segments are: {}", queuePool.recycledSegments); + return queuePool; + } + + public Set queueNames() { + return queues.keySet().stream().map(qn -> qn.name).collect(Collectors.toSet()); + } + + private static Properties createOrLoadCheckpointFile(Path dataPath) throws QueueException { + final Path checkpointPath = dataPath.resolve("checkpoint.properties"); + if (!Files.exists(checkpointPath)) { + LOG.info("Can't find any file named 'checkpoint.properties' in path: {}, creating new one", dataPath); + final boolean notExisted; + try { + notExisted = checkpointPath.toFile().createNewFile(); + } catch (IOException e) { + LOG.error("IO Error creating the file {}", checkpointPath, e); + throw new QueueException("Reached an IO error during the bootstrapping of empty 'checkpoint.properties'", e); + } + if (!notExisted) { + LOG.warn("Found a checkpoint file while bootstrapping {}", checkpointPath); + } + } + + final FileReader fileReader; + try { + fileReader = new FileReader(checkpointPath.toFile()); + } catch (FileNotFoundException e) { + throw new QueueException("Can't find any file named 'checkpoint.properties' in path: " + dataPath, e); + } + final Properties checkpointProps = new Properties(); + try { + checkpointProps.load(fileReader); + } catch (IOException e) { + throw new QueueException("if an error occurred when reading from: " + checkpointPath, e); + } + return checkpointProps; + } + + private void loadQueueDefinitions(Properties checkpointProps) throws QueueException { + // structure of queues definitions in properties file: + // queues.0.name = bla bla + // queues.0.segments = head (id_page, offset), (id_page, offset), ... tail + // queues.0.head_offset = bytes offset from the start of the page where last data was written + // queues.0.tail_offset = bytes offset from the start of the page where first data could be read + boolean noMoreQueues = false; + int queueId = 0; + while (!noMoreQueues) { + final String queueKey = String.format("queues.%d.name", queueId); + if (!checkpointProps.containsKey(queueKey)) { + noMoreQueues = true; + continue; + } + final QueueName queueName = new QueueName(checkpointProps.getProperty(queueKey)); + LinkedList segmentRefs = decodeSegments(checkpointProps.getProperty(String.format("queues.%d.segments", queueId))); + final int numSegments = segmentRefs.size(); + queueSegments.put(queueName, segmentRefs); + + final long headOffset = Long.parseLong(checkpointProps.getProperty(String.format("queues.%d.head_offset", queueId))); + final SegmentRef headSegmentRef = segmentRefs.get(0); + final SegmentPointer currentHead = new SegmentPointer(headSegmentRef.pageId, headOffset); + // TODO this reopen could be done in lazy way during getOrCreate method. + Segment headSegment = allocator.reopenSegment(headSegmentRef.pageId, headSegmentRef.offset); + + final long tailOffset = Long.parseLong(checkpointProps.getProperty(String.format("queues.%d.tail_offset", queueId))); + final SegmentRef tailSegmentRef = segmentRefs.getLast(); + final SegmentPointer currentTail = new SegmentPointer(tailSegmentRef.pageId, tailOffset); + Segment tailSegment = allocator.reopenSegment(tailSegmentRef.pageId, tailSegmentRef.offset); + + // Create relative positioned head and tail pointers + // Tail is an offset relative to start of the first segment in the list + // Head is n-1 full segments plus the offset of the physical head + final VirtualPointer logicalTail = new VirtualPointer(currentTail.offset()); + final VirtualPointer logicalHead = new VirtualPointer((long) (numSegments - 1) * segmentSize + currentHead.offset()); + final Queue queue = new Queue(queueName.name, headSegment, logicalHead, tailSegment, logicalTail, + allocator, callback, this); + queues.put(queueName, queue); + + queueId++; + } + } + + private void loadRecycledSegments(Properties checkpointProps) throws QueueException { + TreeSet usedSegments = new TreeSet<>(); + + boolean noMoreQueues = false; + int queueId = 0; + + // load all queues definitions from checkpoint file + // TODO second use of this, extract as an iterator + while (!noMoreQueues) { + final String queueKey = String.format("queues.%d.name", queueId); + if (!checkpointProps.containsKey(queueKey)) { + noMoreQueues = true; + continue; + } + LinkedList segmentRefs = decodeSegments(checkpointProps.getProperty(String.format("queues.%d.segments", queueId))); + usedSegments.addAll(segmentRefs); + + queueId++; + } + + if (usedSegments.isEmpty()) { + // no queue definitions were loaded + return; + } + + final List recreatedSegments = recreateSegmentHoles(usedSegments); + + segmentsAllocationLock.lock(); + try { + recycledSegments.addAll(recreatedSegments); + } finally { + segmentsAllocationLock.unlock(); + } + } + + /** + * @param usedSegments sorted set of used segments + * */ + // package-private for testing + List recreateSegmentHoles(TreeSet usedSegments) throws QueueException { + // find the holes in the list of used segments + if (usedSegments.isEmpty()) { + throw new QueueException("Status error, expected to find at least one segment"); + } + + // prev point to the last examined segment. + // recreates segments on left of current segment. + SegmentRef prev = null; + final List recreatedSegments = new LinkedList<>(); + for (SegmentRef current : usedSegments) { + if (prev == null) { + // recreate recycled segments before first used segment + recreatedSegments.addAll(recreateRecycledSegmentsBetween(current)); + prev = current; + continue; + } + if (isAdjacent(prev, current)) { + // contiguous, skip it + prev = current; + continue; + } + if (prev.pageId == current.pageId) { + recreatedSegments.addAll(recreateRecycledSegments(prev.offset + segmentSize, current.offset, prev.pageId)); + } else { + // recreate recycled segments between 2 used segments + recreatedSegments.addAll(recreateRecycledSegmentsBetween(prev, current)); + } + } + return recreatedSegments; + } + + private boolean isAdjacent(SegmentRef prev, SegmentRef segment) { + if (prev.pageId == segment.pageId) { + // same page + if (prev.offset + segmentSize == segment.offset) { + // contiguous, skip it + return true; + } + } else if (prev.pageId + 1 == segment.pageId) { + // adjacent pages, last segment in one and first in the other + if (prev.offset == allocator.getPageSize() - segmentSize && segment.offset == 0) { + return true; + } + } + return false; + } + + private List recreateRecycledSegmentsBetween(SegmentRef toSegment) { + return recreateRecycledSegmentsBetween(null, toSegment); + } + + private List recreateRecycledSegmentsBetween(SegmentRef fromSegment, SegmentRef toSegment) { + final List recreatedSegments = new LinkedList<>(); + int prevPageId = 0; + if (fromSegment != null) { + prevPageId = fromSegment.pageId; + // holes after previous segment, to complete the page + recreatedSegments.addAll(recreateRecycledSegments(fromSegment.offset + segmentSize, allocator.getPageSize(), fromSegment.pageId)); + prevPageId++; + } + + // all the intermediate pages + for (; prevPageId < toSegment.pageId; prevPageId++) { + recreatedSegments.addAll(recreateRecycledSegments(0, allocator.getPageSize(), prevPageId)); + } + + // holes before the current segment + recreatedSegments.addAll(recreateRecycledSegments(0, toSegment.offset, toSegment.pageId)); + return recreatedSegments; + } + + private List recreateRecycledSegments(int fromOffset, int toOffset, int pageId) { + final List recreatedSegments = new LinkedList<>(); + while (fromOffset != toOffset) { + recreatedSegments.add(new SegmentRef(pageId, fromOffset)); + fromOffset = fromOffset + segmentSize; + } + return recreatedSegments; + } + + private LinkedList decodeSegments(String s) { + final String[] segments = s.substring(s.indexOf("(") + 1, s.lastIndexOf(")")) + .split("\\), \\("); + + LinkedList acc = new LinkedList<>(); + for (String segment : segments) { + final String[] split = segment.split(","); + final int idPage = Integer.parseInt(split[0].trim()); + final int offset = Integer.parseInt(split[1].trim()); + + acc.offer(new SegmentRef(idPage, offset)); + } + return acc; + } + + public Queue getOrCreate(String queueName) throws QueueException { + final QueueName queueN = new QueueName(queueName); + if (queues.containsKey(queueN)) { + return queues.get(queueN); + } else { + // create new queue with first empty segment + final Segment segment = nextFreeSegment(); + //notify segment creation for queue in queue pool + segmentedCreated(queueName, segment); + + // When a segment is freshly created the head must the last occupied byte, + // so can't be the start of a segment, but one position before, or in case + // of a new page, -1 + final Queue queue = new Queue(queueName, segment, VirtualPointer.buildUntouched(), segment, VirtualPointer.buildUntouched(), + this.allocator, callback, this); + queues.put(queueN, queue); + return queue; + } + } + + /** + * Free mapped files + * */ + public void close() throws QueueException { + allocator.close(); + + //save all into the checkpoint file + Properties checkpoint = new Properties(); + allocator.dumpState(checkpoint); + + int queueCounter = 0; + for (Map.Entry> entry : queueSegments.entrySet()) { + // queues.0.name = bla bla + final QueueName queueName = entry.getKey(); + checkpoint.setProperty("queues." + queueCounter + ".name", queueName.name); + + // queues.0.segments = head (id_page, offset), (id_page, offset), ... tail + final LinkedList segmentRefs = entry.getValue(); + final String segmentsDef = segmentRefs.stream() + .map(SegmentRef::toString) + .collect(Collectors.joining(", ")); + checkpoint.setProperty("queues." + queueCounter + ".segments", segmentsDef); + + // queues.0.head_offset = bytes offset from the start of the page where last data was written + final Queue queue = queues.get(queueName); + checkpoint.setProperty("queues." + queueCounter + ".head_offset", String.valueOf(queue.currentHead().segmentOffset(segmentSize))); + checkpoint.setProperty("queues." + queueCounter + ".tail_offset", String.valueOf(queue.currentTail().segmentOffset(segmentSize))); + } + + final File propertiesFile = dataPath.resolve("checkpoint.properties").toFile(); + final FileWriter fileWriter; + try { + fileWriter = new FileWriter(propertiesFile); + } catch (IOException ex) { + throw new QueueException("Problem opening checkpoint.properties file", ex); + } + try { + checkpoint.store(fileWriter, "DON'T EDIT, AUTOGENERATED"); + } catch (IOException ex) { + throw new QueueException("Problem writing checkpoint.properties file", ex); + } + } + + Optional openNextTailSegment(String name) throws QueueException { + // definition from QueuePool.queueSegments + final QueueName queueName = new QueueName(name); + final LinkedList segmentRefs = queueSegments.get(queueName); + + final SegmentRef pollSegment = segmentRefs.peekLast(); + if (pollSegment == null) { + return Optional.empty(); + } + + final Path pageFile = dataPath.resolve(String.format("%d.page", pollSegment.pageId)); + if (!Files.exists(pageFile)) { + throw new QueueException("Can't find file for page file" + pageFile); + } + + final MappedByteBuffer tailPage; + try (FileChannel fileChannel = FileChannel.open(pageFile, StandardOpenOption.READ, StandardOpenOption.WRITE)) { + tailPage = fileChannel.map(FileChannel.MapMode.READ_WRITE/*READ_ONLY*/, 0, allocator.getPageSize()); + } catch (IOException ex) { + throw new QueueException("Can't open page file " + pageFile, ex); + } + + final SegmentPointer begin = new SegmentPointer(pollSegment.pageId, pollSegment.offset); + final SegmentPointer end = new SegmentPointer(pollSegment.pageId, pollSegment.offset + segmentSize - 1); + return Optional.of(new Segment(tailPage, begin, end)); + } + + /** + * Notify the actual tail segment was completely read + * */ + void consumedTailSegment(String name) { + final QueueName queueName = new QueueName(name); + final LinkedList segmentRefs = queueSegments.get(queueName); + final SegmentRef segmentRef = segmentRefs.pollLast(); + LOG.debug("Consumed tail segment {} from queue {}", segmentRef, queueName); + segmentsAllocationLock.lock(); + try { + recycledSegments.add(segmentRef); + } finally { + segmentsAllocationLock.unlock(); + } + } + + Segment nextFreeSegment() throws QueueException { + segmentsAllocationLock.lock(); + try { + if (recycledSegments.isEmpty()) { + LOG.debug("no recycled segments available, request the creation of new one"); + return allocator.nextFreeSegment(); + } + final SegmentRef recycledSegment = recycledSegments.pollFirst(); + if (recycledSegment == null) { + throw new QueueException("Invalid state, expected available recycled segment"); + } + LOG.debug("Reusing recycled segment from page: {} at page offset: {}", recycledSegment.pageId, recycledSegment.offset); + return allocator.reopenSegment(recycledSegment.pageId, recycledSegment.offset); + } finally { + segmentsAllocationLock.unlock(); + } + } +} diff --git a/moquette-0.17/broker/src/main/java/io/moquette/broker/unsafequeues/Segment.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/unsafequeues/Segment.java new file mode 100644 index 00000000..ff154104 --- /dev/null +++ b/moquette-0.17/broker/src/main/java/io/moquette/broker/unsafequeues/Segment.java @@ -0,0 +1,161 @@ +package io.moquette.broker.unsafequeues; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; +import java.nio.MappedByteBuffer; + +final class Segment { + private static final Logger LOG = LoggerFactory.getLogger(Segment.class); + + final int segmentSize; + final SegmentPointer begin; + final SegmentPointer end; + + private final MappedByteBuffer mappedBuffer; + + Segment(MappedByteBuffer page, SegmentPointer begin, SegmentPointer end) { + assert begin.samePage(end); + this.segmentSize = end.offset() - begin.offset() + 1; + this.begin = begin; + this.end = end; + this.mappedBuffer = page; + } + + boolean hasSpace(VirtualPointer mark, long length) { + return bytesAfter(mark) >= length; + } + + /** + * @return number of bytes in segment after the pointer. + * The pointer slot is not counted. + * */ + public long bytesAfter(SegmentPointer mark) { + assert mark.samePage(this.end); + return end.distance(mark); + } + + public long bytesAfter(VirtualPointer mark) { + final int pageOffset = rebasedOffset(mark); + final SegmentPointer physicalMark = new SegmentPointer(this.end.pageId(), pageOffset); + return end.distance(physicalMark); + } + + void write(SegmentPointer offset, ByteBuffer content) { + checkContentStartWith(content); + final int startPos = offset.offset(); + final int endPos = startPos + content.remaining(); + for (int i = startPos; i < endPos; i++) { + mappedBuffer.put(i, content.get()); + } + } + + // fill the segment with value bytes + void fillWith(byte value) { + LOG.debug("Wipe segment {}", this); + final int target = begin.offset() + (int)size(); + for (int i = begin.offset(); i < target; i++) { + mappedBuffer.put(i, value); + } + } + + // debug method + private void checkContentStartWith(ByteBuffer content) { + if (content.get(0) == 0 && content.get(1) == 0 && content.get(2) == 0 && content.get(3) == 0) { + System.out.println("DNADBG content starts with 4 zero"); + } + } + + void write(VirtualPointer offset, ByteBuffer content) { + final int startPos = rebasedOffset(offset); + final int endPos = startPos + content.remaining(); + for (int i = startPos; i < endPos; i++) { + mappedBuffer.put(i, content.get()); + } + } + + /** + * Force flush of memory mapper buffer to disk + * */ + void force() { + mappedBuffer.force(); + } + + /** + * return the int value contained in the 4 bytes after the pointer. + * + * @param pointer virtual pointer to start read from. + * */ + int readHeader(VirtualPointer pointer) { + final int rebasedIndex = rebasedOffset(pointer); + LOG.debug(" {} {} {} {} at {}", Integer.toHexString(mappedBuffer.get(rebasedIndex)), + Integer.toHexString(mappedBuffer.get(rebasedIndex + 1)), + Integer.toHexString(mappedBuffer.get(rebasedIndex + 2)), + Integer.toHexString(mappedBuffer.get(rebasedIndex + 3)), + pointer + ); + return mappedBuffer.getInt(rebasedIndex); + } + + /*private*/ int rebasedOffset(VirtualPointer virtualPtr) { + final int pointerOffset = (int) virtualPtr.segmentOffset(segmentSize); + return this.begin.plus(pointerOffset).offset(); + } + + public ByteBuffer read(VirtualPointer start, int length) { + final int pageOffset = rebasedOffset(start); + byte[] dst = new byte[length]; + + int sourceIdx = pageOffset; + for (int dstIndex = 0; dstIndex < length; dstIndex++, sourceIdx++) { + dst[dstIndex] = mappedBuffer.get(sourceIdx); + } + + return ByteBuffer.wrap(dst); + } + + public ByteBuffer read(SegmentPointer start, int length) { + byte[] dst = new byte[length]; + + if (length > mappedBuffer.remaining() - start.offset()) + throw new BufferUnderflowException(); + + int sourceIdx = start.offset(); + for (int dstIndex = 0; dstIndex < length; dstIndex++, sourceIdx++) { + dst[dstIndex] = mappedBuffer.get(sourceIdx); + } + + return ByteBuffer.wrap(dst); + } + + private long size() { + return end.distance(begin) + 1; + } + + @Override + public String toString() { + return "Segment{page=" + begin.pageId() + ", begin=" + begin.offset() + ", end=" + end.offset() + ", size=" + size() + "}"; + } + + ByteBuffer readAllBytesAfter(SegmentPointer start) { + // WARN, dataStart points to a byte position to read + // if currentSegment.end is at offset 1023, and data start is 1020, the bytes after are 4 and + // not 1023 - 1020. + final long availableDataLength = bytesAfter(start) + 1; + final ByteBuffer buffer = read(start, (int) availableDataLength); + buffer.rewind(); + return buffer; + } + + ByteBuffer readAllBytesAfter(VirtualPointer start) { + // WARN, dataStart points to a byte position to read + // if currentSegment.end is at offset 1023, and data start is 1020, the bytes after are 4 and + // not 1023 - 1020. + final long availableDataLength = bytesAfter(start) + 1; + final ByteBuffer buffer = read(start, (int) availableDataLength); + buffer.rewind(); + return buffer; + } +} diff --git a/moquette-0.17/broker/src/main/java/io/moquette/broker/unsafequeues/SegmentAllocator.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/unsafequeues/SegmentAllocator.java new file mode 100644 index 00000000..bb712abe --- /dev/null +++ b/moquette-0.17/broker/src/main/java/io/moquette/broker/unsafequeues/SegmentAllocator.java @@ -0,0 +1,35 @@ +package io.moquette.broker.unsafequeues; + +import java.util.Properties; + +interface SegmentAllocator { + + /** + * Return the next free segment in the current page, or create a new Page if necessary. + * + * This method has to be invoked inside a lock, it's not thread safe. + * + * @throws QueueException if any IO error happens on the filesystem. + * */ + Segment nextFreeSegment() throws QueueException; + + Segment reopenSegment(int pageId, int beginOffset) throws QueueException; + + void close() throws QueueException; + + void dumpState(Properties checkpoint); + + /** + * Get the size of a page that this allocator uses. + * + * @return the size of a page that this allocator uses. + */ + int getPageSize(); + + /** + * Get the size of a segment that this allocator uses. + * + * @return the size of a segment that this allocator uses. + */ + int getSegmentSize(); +} diff --git a/moquette-0.17/broker/src/main/java/io/moquette/broker/unsafequeues/SegmentPointer.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/unsafequeues/SegmentPointer.java new file mode 100644 index 00000000..86a456dd --- /dev/null +++ b/moquette-0.17/broker/src/main/java/io/moquette/broker/unsafequeues/SegmentPointer.java @@ -0,0 +1,83 @@ +package io.moquette.broker.unsafequeues; + +import java.util.Objects; + +final class SegmentPointer implements Comparable { + private final int idPage; + private final long offset; + + public SegmentPointer(int idPage, long offset) { + this.idPage = idPage; + this.offset = offset; + } + + /** + * Construct using the segment, but changing the offset. + * */ + public SegmentPointer(Segment segment, long offset) { + this.idPage = segment.begin.idPage; + this.offset = offset; + } + + /** + * Copy constructor + * */ + public SegmentPointer copy() { + return new SegmentPointer(idPage, offset); + } + + @Override + public int compareTo(SegmentPointer other) { + if (idPage == other.idPage) { + return Long.compare(offset, other.offset); + } else { + return Integer.compare(idPage, other.idPage); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + SegmentPointer that = (SegmentPointer) o; + return idPage == that.idPage && offset == that.offset; + } + + @Override + public int hashCode() { + return Objects.hash(idPage, offset); + } + + boolean samePage(SegmentPointer other) { + return idPage == other.idPage; + } + + SegmentPointer moveForward(long length) { + return new SegmentPointer(idPage, offset + length); + } + + @Override + public String toString() { + return "SegmentPointer{idPage=" + idPage + ", offset=" + offset + '}'; + } + + /** + * Calculate the distance in bytes inside the same segment + * */ + public long distance(SegmentPointer other) { + assert idPage == other.idPage; + return offset - other.offset; + } + + int offset() { + return (int) offset; + } + + public SegmentPointer plus(int delta) { + return moveForward(delta); + } + + int pageId() { + return this.idPage; + } +} diff --git a/moquette-0.17/broker/src/main/java/io/moquette/broker/unsafequeues/VirtualPointer.java b/moquette-0.17/broker/src/main/java/io/moquette/broker/unsafequeues/VirtualPointer.java new file mode 100644 index 00000000..c157a202 --- /dev/null +++ b/moquette-0.17/broker/src/main/java/io/moquette/broker/unsafequeues/VirtualPointer.java @@ -0,0 +1,51 @@ +package io.moquette.broker.unsafequeues; + +public class VirtualPointer implements Comparable { + final long logicalOffset; + + public static VirtualPointer buildUntouched() { + return new VirtualPointer(-1); + } + + public VirtualPointer(long logicalOffset) { + this.logicalOffset = logicalOffset; + } + + @Override + public int compareTo(VirtualPointer other) { + return Long.compare(logicalOffset, other.logicalOffset); + } + + public long segmentOffset(int segmentSize) { + return logicalOffset % segmentSize; + } + + public long logicalOffset() { + return logicalOffset; + } + + public VirtualPointer moveForward(long delta) { + return new VirtualPointer(logicalOffset + delta); + } + + public VirtualPointer plus(int i) { + return new VirtualPointer(logicalOffset + i); + } + + public boolean isGreaterThan(VirtualPointer other) { + return this.compareTo(other) > 0; + } + + public boolean isUntouched() { + return logicalOffset == -1; + } + + public VirtualPointer copy() { + return new VirtualPointer(this.logicalOffset); + } + + @Override + public String toString() { + return "VirtualPointer{logicalOffset=" + logicalOffset + '}'; + } +} diff --git a/moquette-0.16/broker/src/main/java/io/moquette/interception/AbstractInterceptHandler.java b/moquette-0.17/broker/src/main/java/io/moquette/interception/AbstractInterceptHandler.java similarity index 96% rename from moquette-0.16/broker/src/main/java/io/moquette/interception/AbstractInterceptHandler.java rename to moquette-0.17/broker/src/main/java/io/moquette/interception/AbstractInterceptHandler.java index bd6e11f9..2af961d0 100644 --- a/moquette-0.16/broker/src/main/java/io/moquette/interception/AbstractInterceptHandler.java +++ b/moquette-0.17/broker/src/main/java/io/moquette/interception/AbstractInterceptHandler.java @@ -25,7 +25,7 @@ import io.moquette.interception.messages.InterceptUnsubscribeMessage; /** - * Basic abstract class usefull to avoid empty methods creation in subclasses. + * Basic abstract class useful to avoid empty methods creation in subclasses. */ public abstract class AbstractInterceptHandler implements InterceptHandler { diff --git a/moquette-0.16/broker/src/main/java/io/moquette/interception/BrokerInterceptor.java b/moquette-0.17/broker/src/main/java/io/moquette/interception/BrokerInterceptor.java similarity index 95% rename from moquette-0.16/broker/src/main/java/io/moquette/interception/BrokerInterceptor.java rename to moquette-0.17/broker/src/main/java/io/moquette/interception/BrokerInterceptor.java index ffcf2b1f..2168db46 100644 --- a/moquette-0.16/broker/src/main/java/io/moquette/interception/BrokerInterceptor.java +++ b/moquette-0.17/broker/src/main/java/io/moquette/interception/BrokerInterceptor.java @@ -128,8 +128,8 @@ public void notifyTopicPublished(final MqttPublishMessage msg, final String clie int messageId = msg.variableHeader().messageId(); String topic = msg.variableHeader().topicName(); for (InterceptHandler handler : handlers.get(InterceptPublishMessage.class)) { - LOG.debug("Notifying MQTT PUBLISH message to interceptor. CId={}, messageId={}, topic={}, " - + "interceptorId={}", clientID, messageId, topic, handler.getID()); + LOG.debug("Notifying unexpected MQTT client disconnection to interceptor CId={}, " + + "interceptorId={}", clientID, handler.getID()); // Sending to the outside, make a retainedDuplicate. handler.onPublish(new InterceptPublishMessage(msg.retainedDuplicate(), clientID, username)); } @@ -166,6 +166,13 @@ public void notifyMessageAcknowledged(final InterceptAcknowledgedMessage msg) { } } + @Override + public void notifyLoopException(InterceptExceptionMessage msg) { + for (final InterceptHandler handler : this.handlers.get(InterceptExceptionMessage.class)) { + handler.onSessionLoopError(msg.getError()); + } + } + @Override public void addInterceptHandler(InterceptHandler interceptHandler) { Class[] interceptedMessageTypes = getInterceptedMessageTypes(interceptHandler); diff --git a/moquette-0.16/broker/src/main/java/io/moquette/interception/InterceptHandler.java b/moquette-0.17/broker/src/main/java/io/moquette/interception/InterceptHandler.java similarity index 93% rename from moquette-0.16/broker/src/main/java/io/moquette/interception/InterceptHandler.java rename to moquette-0.17/broker/src/main/java/io/moquette/interception/InterceptHandler.java index c5dde510..141456d4 100644 --- a/moquette-0.16/broker/src/main/java/io/moquette/interception/InterceptHandler.java +++ b/moquette-0.17/broker/src/main/java/io/moquette/interception/InterceptHandler.java @@ -22,6 +22,7 @@ /** * This interface is used to inject code for intercepting broker events. + * This is part of the API that integrator of Moquette has to implement. *

* The events can act only as observers. *

@@ -33,7 +34,7 @@ public interface InterceptHandler { Class[] ALL_MESSAGE_TYPES = {InterceptConnectMessage.class, InterceptDisconnectMessage.class, InterceptConnectionLostMessage.class, InterceptPublishMessage.class, InterceptSubscribeMessage.class, - InterceptUnsubscribeMessage.class, InterceptAcknowledgedMessage.class}; + InterceptUnsubscribeMessage.class, InterceptAcknowledgedMessage.class, InterceptExceptionMessage.class}; /** * @return the identifier of this intercept handler. @@ -65,4 +66,6 @@ public interface InterceptHandler { void onUnsubscribe(InterceptUnsubscribeMessage msg); void onMessageAcknowledged(InterceptAcknowledgedMessage msg); + + void onSessionLoopError(Throwable error); } diff --git a/moquette-0.16/broker/src/main/java/io/moquette/interception/Interceptor.java b/moquette-0.17/broker/src/main/java/io/moquette/interception/Interceptor.java similarity index 93% rename from moquette-0.16/broker/src/main/java/io/moquette/interception/Interceptor.java rename to moquette-0.17/broker/src/main/java/io/moquette/interception/Interceptor.java index e6950981..7055bcea 100644 --- a/moquette-0.16/broker/src/main/java/io/moquette/interception/Interceptor.java +++ b/moquette-0.17/broker/src/main/java/io/moquette/interception/Interceptor.java @@ -18,6 +18,7 @@ import io.moquette.interception.messages.InterceptAcknowledgedMessage; import io.moquette.broker.subscriptions.Subscription; +import io.moquette.interception.messages.InterceptExceptionMessage; import io.netty.handler.codec.mqtt.MqttConnectMessage; import io.netty.handler.codec.mqtt.MqttPublishMessage; @@ -47,6 +48,8 @@ public interface Interceptor { void notifyMessageAcknowledged(InterceptAcknowledgedMessage msg); + void notifyLoopException(InterceptExceptionMessage th); + void addInterceptHandler(InterceptHandler interceptHandler); void removeInterceptHandler(InterceptHandler interceptHandler); diff --git a/moquette-0.16/broker/src/main/java/io/moquette/interception/messages/InterceptAbstractMessage.java b/moquette-0.17/broker/src/main/java/io/moquette/interception/messages/InterceptAbstractMessage.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/interception/messages/InterceptAbstractMessage.java rename to moquette-0.17/broker/src/main/java/io/moquette/interception/messages/InterceptAbstractMessage.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/interception/messages/InterceptAcknowledgedMessage.java b/moquette-0.17/broker/src/main/java/io/moquette/interception/messages/InterceptAcknowledgedMessage.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/interception/messages/InterceptAcknowledgedMessage.java rename to moquette-0.17/broker/src/main/java/io/moquette/interception/messages/InterceptAcknowledgedMessage.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/interception/messages/InterceptConnectMessage.java b/moquette-0.17/broker/src/main/java/io/moquette/interception/messages/InterceptConnectMessage.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/interception/messages/InterceptConnectMessage.java rename to moquette-0.17/broker/src/main/java/io/moquette/interception/messages/InterceptConnectMessage.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/interception/messages/InterceptConnectionLostMessage.java b/moquette-0.17/broker/src/main/java/io/moquette/interception/messages/InterceptConnectionLostMessage.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/interception/messages/InterceptConnectionLostMessage.java rename to moquette-0.17/broker/src/main/java/io/moquette/interception/messages/InterceptConnectionLostMessage.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/interception/messages/InterceptDisconnectMessage.java b/moquette-0.17/broker/src/main/java/io/moquette/interception/messages/InterceptDisconnectMessage.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/interception/messages/InterceptDisconnectMessage.java rename to moquette-0.17/broker/src/main/java/io/moquette/interception/messages/InterceptDisconnectMessage.java diff --git a/moquette-0.17/broker/src/main/java/io/moquette/interception/messages/InterceptExceptionMessage.java b/moquette-0.17/broker/src/main/java/io/moquette/interception/messages/InterceptExceptionMessage.java new file mode 100644 index 00000000..fda676ec --- /dev/null +++ b/moquette-0.17/broker/src/main/java/io/moquette/interception/messages/InterceptExceptionMessage.java @@ -0,0 +1,13 @@ +package io.moquette.interception.messages; + +public class InterceptExceptionMessage implements InterceptMessage { + private Throwable error; + + public InterceptExceptionMessage(Throwable error) { + this.error = error; + } + + public Throwable getError() { + return error; + } +} diff --git a/moquette-0.16/broker/src/main/java/io/moquette/interception/messages/InterceptMessage.java b/moquette-0.17/broker/src/main/java/io/moquette/interception/messages/InterceptMessage.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/interception/messages/InterceptMessage.java rename to moquette-0.17/broker/src/main/java/io/moquette/interception/messages/InterceptMessage.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/interception/messages/InterceptPublishMessage.java b/moquette-0.17/broker/src/main/java/io/moquette/interception/messages/InterceptPublishMessage.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/interception/messages/InterceptPublishMessage.java rename to moquette-0.17/broker/src/main/java/io/moquette/interception/messages/InterceptPublishMessage.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/interception/messages/InterceptSubscribeMessage.java b/moquette-0.17/broker/src/main/java/io/moquette/interception/messages/InterceptSubscribeMessage.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/interception/messages/InterceptSubscribeMessage.java rename to moquette-0.17/broker/src/main/java/io/moquette/interception/messages/InterceptSubscribeMessage.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/interception/messages/InterceptUnsubscribeMessage.java b/moquette-0.17/broker/src/main/java/io/moquette/interception/messages/InterceptUnsubscribeMessage.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/interception/messages/InterceptUnsubscribeMessage.java rename to moquette-0.17/broker/src/main/java/io/moquette/interception/messages/InterceptUnsubscribeMessage.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/logging/LoggingUtils.java b/moquette-0.17/broker/src/main/java/io/moquette/logging/LoggingUtils.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/logging/LoggingUtils.java rename to moquette-0.17/broker/src/main/java/io/moquette/logging/LoggingUtils.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/persistence/ByteBufDataType.java b/moquette-0.17/broker/src/main/java/io/moquette/persistence/ByteBufDataType.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/persistence/ByteBufDataType.java rename to moquette-0.17/broker/src/main/java/io/moquette/persistence/ByteBufDataType.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/persistence/EnqueuedMessageValueType.java b/moquette-0.17/broker/src/main/java/io/moquette/persistence/EnqueuedMessageValueType.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/persistence/EnqueuedMessageValueType.java rename to moquette-0.17/broker/src/main/java/io/moquette/persistence/EnqueuedMessageValueType.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/persistence/H2Builder.java b/moquette-0.17/broker/src/main/java/io/moquette/persistence/H2Builder.java similarity index 64% rename from moquette-0.16/broker/src/main/java/io/moquette/persistence/H2Builder.java rename to moquette-0.17/broker/src/main/java/io/moquette/persistence/H2Builder.java index fd72e757..641c5cd6 100644 --- a/moquette-0.16/broker/src/main/java/io/moquette/persistence/H2Builder.java +++ b/moquette-0.17/broker/src/main/java/io/moquette/persistence/H2Builder.java @@ -1,14 +1,18 @@ package io.moquette.persistence; -import io.moquette.BrokerConstants; import io.moquette.broker.IQueueRepository; import io.moquette.broker.IRetainedRepository; +import io.moquette.broker.ISessionsRepository; import io.moquette.broker.ISubscriptionsRepository; -import io.moquette.broker.config.IConfig; import org.h2.mvstore.MVStore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Clock; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -19,21 +23,31 @@ public class H2Builder { private final String storePath; private final int autosaveInterval; // in seconds private final ScheduledExecutorService scheduler; + private final Clock clock; private MVStore mvStore; - public H2Builder(IConfig props, ScheduledExecutorService scheduler) { - this.storePath = props.getProperty(BrokerConstants.PERSISTENT_STORE_PROPERTY_NAME, ""); - final String autosaveProp = props.getProperty(BrokerConstants.AUTOSAVE_INTERVAL_PROPERTY_NAME, "30"); - this.autosaveInterval = Integer.parseInt(autosaveProp); + public H2Builder(ScheduledExecutorService scheduler, Path storePath, int autosaveInterval, Clock clock) { + this.storePath = storePath.resolve("moquette_store.h2").toAbsolutePath().toString(); + this.autosaveInterval = autosaveInterval; this.scheduler = scheduler; + this.clock = clock; } @SuppressWarnings("FutureReturnValueIgnored") public H2Builder initStore() { - LOG.info("Initializing H2 store"); + LOG.info("Initializing H2 store to {}", storePath); if (storePath == null || storePath.isEmpty()) { throw new IllegalArgumentException("H2 store path can't be null or empty"); } + + if (!Files.exists(Paths.get(storePath))) { + try { + Files.createFile(Paths.get(storePath)); + } catch (IOException ex) { + throw new IllegalArgumentException("Error creating " + storePath + " file", ex); + } + } + mvStore = new MVStore.Builder() .fileName(storePath) .autoCommitDisabled() @@ -62,4 +76,8 @@ public IQueueRepository queueRepository() { public IRetainedRepository retainedRepository() { return new H2RetainedRepository(mvStore); } + + public ISessionsRepository sessionsRepository() { + return new H2SessionsRepository(mvStore, clock); + } } diff --git a/moquette-0.16/broker/src/main/java/io/moquette/persistence/H2PersistentQueue.java b/moquette-0.17/broker/src/main/java/io/moquette/persistence/H2PersistentQueue.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/persistence/H2PersistentQueue.java rename to moquette-0.17/broker/src/main/java/io/moquette/persistence/H2PersistentQueue.java diff --git a/moquette-0.16/broker/src/main/java/io/moquette/persistence/H2QueueRepository.java b/moquette-0.17/broker/src/main/java/io/moquette/persistence/H2QueueRepository.java similarity index 93% rename from moquette-0.16/broker/src/main/java/io/moquette/persistence/H2QueueRepository.java rename to moquette-0.17/broker/src/main/java/io/moquette/persistence/H2QueueRepository.java index 90eb967c..8f27b92c 100644 --- a/moquette-0.16/broker/src/main/java/io/moquette/persistence/H2QueueRepository.java +++ b/moquette-0.17/broker/src/main/java/io/moquette/persistence/H2QueueRepository.java @@ -20,11 +20,7 @@ import io.moquette.broker.SessionRegistry.EnqueuedMessage; import org.h2.mvstore.MVStore; -import java.util.HashMap; -import java.util.Map; -import java.util.Queue; import java.util.Set; -import java.util.concurrent.ConcurrentLinkedQueue; import java.util.stream.Collectors; public class H2QueueRepository implements IQueueRepository { @@ -52,4 +48,9 @@ public boolean containsQueue(String queueName) { public SessionMessageQueue getOrCreateQueue(String clientId) { return new H2PersistentQueue(mvStore, clientId); } + + @Override + public void close() { + // No-op + } } diff --git a/moquette-0.16/broker/src/main/java/io/moquette/persistence/H2RetainedRepository.java b/moquette-0.17/broker/src/main/java/io/moquette/persistence/H2RetainedRepository.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/persistence/H2RetainedRepository.java rename to moquette-0.17/broker/src/main/java/io/moquette/persistence/H2RetainedRepository.java diff --git a/moquette-0.17/broker/src/main/java/io/moquette/persistence/H2SessionsRepository.java b/moquette-0.17/broker/src/main/java/io/moquette/persistence/H2SessionsRepository.java new file mode 100644 index 00000000..e9667eca --- /dev/null +++ b/moquette-0.17/broker/src/main/java/io/moquette/persistence/H2SessionsRepository.java @@ -0,0 +1,104 @@ +package io.moquette.persistence; + +import io.moquette.broker.ISessionsRepository; +import io.netty.handler.codec.mqtt.MqttVersion; +import org.h2.mvstore.MVMap; +import org.h2.mvstore.MVStore; +import org.h2.mvstore.WriteBuffer; +import org.h2.mvstore.type.BasicDataType; +import org.h2.mvstore.type.StringDataType; + +import java.nio.ByteBuffer; +import java.time.Clock; +import java.time.Instant; +import java.util.Collection; + +class H2SessionsRepository implements ISessionsRepository { + + private static final byte SESSION_DATA_SERDES_V1 = 1; + private static final long UNDEFINED_INSTANT = -1; + + private final MVMap sessionMap; + private final Clock clock; + + public H2SessionsRepository(MVStore mvStore, Clock clock) { + this.clock = clock; + final MVMap.Builder sessionTypeBuilder = + new MVMap.Builder() + .valueType(new SessionDataValueType()); + + this.sessionMap = mvStore.openMap("sessions_store", sessionTypeBuilder); + } + + @Override + public Collection list() { + return sessionMap.values(); + } + + @Override + public void saveSession(SessionData session) { + sessionMap.put(session.clientId(), session); + } + + @Override + public void delete(SessionData session) { + sessionMap.remove(session.clientId()); + } + + /** + * Codec data type to load and store SessionData instances + */ + private final class SessionDataValueType extends BasicDataType { + + private final StringDataType stringDataType = new StringDataType(); + + @Override + public int getMemory(SessionData obj) { + return stringDataType.getMemory(obj.clientId()) + 8 + 1 + 4; + } + + @Override + public void write(WriteBuffer buff, SessionData obj) { + buff.put(SESSION_DATA_SERDES_V1); + stringDataType.write(buff, obj.clientId()); + buff.putLong(obj.expiryInstant().orElse(UNDEFINED_INSTANT)); + buff.put(obj.protocolVersion().protocolLevel()); + buff.putInt(obj.expiryInterval()); + } + + @Override + public SessionData read(ByteBuffer buff) { + final byte serDesVersion = buff.get(); + if (serDesVersion != SESSION_DATA_SERDES_V1) { + throw new IllegalArgumentException("Unrecognized serialization version " + serDesVersion); + } + final String clientId = stringDataType.read(buff); + final long expiresAt = buff.getLong(); + final MqttVersion version = readMQTTVersion(buff.get()); + final int expiryInterval = buff.getInt(); + + if (expiresAt == UNDEFINED_INSTANT) { + return new SessionData(clientId, version, expiryInterval, clock); + } else { + return new SessionData(clientId, Instant.ofEpochMilli(expiresAt), version, expiryInterval, clock); + } + } + + @Override + public SessionData[] createStorage(int i) { + return new SessionData[i]; + } + } + + private MqttVersion readMQTTVersion(byte rawVersion) { + final MqttVersion version; + switch (rawVersion) { + case 3: version = MqttVersion.MQTT_3_1; break; + case 4: version = MqttVersion.MQTT_3_1_1; break; + case 5: version = MqttVersion.MQTT_5; break; + default: + throw new IllegalArgumentException("Unrecognized MQTT version value " + rawVersion); + } + return version; + } +} diff --git a/moquette-0.16/broker/src/main/java/io/moquette/persistence/H2SubscriptionsRepository.java b/moquette-0.17/broker/src/main/java/io/moquette/persistence/H2SubscriptionsRepository.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/persistence/H2SubscriptionsRepository.java rename to moquette-0.17/broker/src/main/java/io/moquette/persistence/H2SubscriptionsRepository.java diff --git a/moquette-0.17/broker/src/main/java/io/moquette/persistence/MemorySessionsRepository.java b/moquette-0.17/broker/src/main/java/io/moquette/persistence/MemorySessionsRepository.java new file mode 100644 index 00000000..25f0b04e --- /dev/null +++ b/moquette-0.17/broker/src/main/java/io/moquette/persistence/MemorySessionsRepository.java @@ -0,0 +1,27 @@ +package io.moquette.persistence; + +import io.moquette.broker.ISessionsRepository; + +import java.util.Collection; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +public class MemorySessionsRepository implements ISessionsRepository { + + private final ConcurrentMap sessions = new ConcurrentHashMap<>(); + + @Override + public Collection list() { + return sessions.values(); + } + + @Override + public void saveSession(SessionData session) { + sessions.put(session.clientId(), session); + } + + @Override + public void delete(SessionData session) { + sessions.remove(session.clientId()); + } +} diff --git a/moquette-0.16/broker/src/main/java/io/moquette/persistence/MemorySubscriptionsRepository.java b/moquette-0.17/broker/src/main/java/io/moquette/persistence/MemorySubscriptionsRepository.java similarity index 100% rename from moquette-0.16/broker/src/main/java/io/moquette/persistence/MemorySubscriptionsRepository.java rename to moquette-0.17/broker/src/main/java/io/moquette/persistence/MemorySubscriptionsRepository.java diff --git a/moquette-0.17/broker/src/main/java/io/moquette/persistence/SegmentPersistentQueue.java b/moquette-0.17/broker/src/main/java/io/moquette/persistence/SegmentPersistentQueue.java new file mode 100644 index 00000000..0d802045 --- /dev/null +++ b/moquette-0.17/broker/src/main/java/io/moquette/persistence/SegmentPersistentQueue.java @@ -0,0 +1,160 @@ +package io.moquette.persistence; + +import io.moquette.broker.AbstractSessionMessageQueue; +import io.moquette.broker.SessionRegistry; +import io.moquette.broker.subscriptions.Topic; +import io.moquette.broker.unsafequeues.Queue; +import io.moquette.broker.unsafequeues.QueueException; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.handler.codec.mqtt.MqttQoS; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Optional; + +public class SegmentPersistentQueue extends AbstractSessionMessageQueue { + + private static class SerDes { + + private enum MessageType {PUB_REL_MARKER, PUBLISHED_MESSAGE} + + public ByteBuffer toBytes(SessionRegistry.EnqueuedMessage message) { + final int memorySize = getMemory(message); + final ByteBuffer payload = ByteBuffer.allocate(memorySize); + payload.mark(); + write(message, payload); + payload.reset(); + return payload; + } + + private void write(SessionRegistry.EnqueuedMessage obj, ByteBuffer buff) { + if (obj instanceof SessionRegistry.PublishedMessage) { + buff.put((byte) MessageType.PUBLISHED_MESSAGE.ordinal()); + + final SessionRegistry.PublishedMessage casted = (SessionRegistry.PublishedMessage) obj; + buff.put((byte) casted.getPublishingQos().value()); + + final String topic = casted.getTopic().toString(); + + writeTopic(buff, topic); + writePayload(buff, casted.getPayload()); + } else if (obj instanceof SessionRegistry.PubRelMarker) { + buff.put((byte) MessageType.PUB_REL_MARKER.ordinal()); + } else { + throw new IllegalArgumentException("Unrecognized message class " + obj.getClass()); + } + } + + private void writePayload(ByteBuffer target, ByteBuf source) { + final int payloadSize = source.readableBytes(); + byte[] rawBytes = new byte[payloadSize]; + final int pinPoint = source.readerIndex(); + source.readBytes(rawBytes).release(); + source.readerIndex(pinPoint); + target.putInt(payloadSize); + target.put(rawBytes); + } + + private void writeTopic(ByteBuffer buff, String topic) { + final byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8); + buff.putInt(topicBytes.length).put(topicBytes); + } + + private int getMemory(SessionRegistry.EnqueuedMessage obj) { + if (obj instanceof SessionRegistry.PubRelMarker) { + return 1; + } + final SessionRegistry.PublishedMessage casted = (SessionRegistry.PublishedMessage) obj; + return 1 + // message type + 1 + // qos + topicMemorySize(casted.getTopic()) + + payloadMemorySize(casted.getPayload()); + } + + private int payloadMemorySize(ByteBuf payload) { + return 4 + // size + payload.readableBytes(); + } + + private int topicMemorySize(Topic topic) { + return 4 + // size + topic.toString().getBytes(StandardCharsets.UTF_8).length; + } + + public SessionRegistry.EnqueuedMessage fromBytes(ByteBuffer buff) { + final byte messageType = buff.get(); + if (messageType == MessageType.PUB_REL_MARKER.ordinal()) { + return new SessionRegistry.PubRelMarker(); + } else if (messageType == MessageType.PUBLISHED_MESSAGE.ordinal()) { + final MqttQoS qos = MqttQoS.valueOf(buff.get()); + final String topicStr = readTopic(buff); + final ByteBuf payload = readPayload(buff); + return new SessionRegistry.PublishedMessage(Topic.asTopic(topicStr), qos, payload, false); + } else { + throw new IllegalArgumentException("Can't recognize record of type: " + messageType); + } + } + + private String readTopic(ByteBuffer buff) { + final int stringLen = buff.getInt(); + final byte[] rawString = new byte[stringLen]; + buff.get(rawString); + return new String(rawString, StandardCharsets.UTF_8); + } + + private ByteBuf readPayload(ByteBuffer buff) { + final int payloadSize = buff.getInt(); + byte[] payload = new byte[payloadSize]; + buff.get(payload); + return Unpooled.wrappedBuffer(payload); + } + } + + private final Queue segmentedQueue; + private final SerDes serdes = new SerDes(); + + public SegmentPersistentQueue(Queue segmentedQueue) { + this.segmentedQueue = segmentedQueue; + } + + @Override + public void enqueue(SessionRegistry.EnqueuedMessage message) { + checkEnqueuePreconditions(message); + + final ByteBuffer payload = serdes.toBytes(message); + try { + segmentedQueue.enqueue(payload); + } catch (QueueException e) { + throw new RuntimeException(e); + } + } + + @Override + public SessionRegistry.EnqueuedMessage dequeue() { + checkDequeuePreconditions(); + + final Optional dequeue; + try { + dequeue = segmentedQueue.dequeue(); + } catch (QueueException e) { + throw new RuntimeException(e); + } + if (!dequeue.isPresent()) { + return null; + } + + final ByteBuffer content = dequeue.get(); + return serdes.fromBytes(content); + } + + @Override + public boolean isEmpty() { + return segmentedQueue.isEmpty(); + } + + @Override + public void closeAndPurge() { + closed = true; + } +} diff --git a/moquette-0.17/broker/src/main/java/io/moquette/persistence/SegmentQueueRepository.java b/moquette-0.17/broker/src/main/java/io/moquette/persistence/SegmentQueueRepository.java new file mode 100644 index 00000000..5722c880 --- /dev/null +++ b/moquette-0.17/broker/src/main/java/io/moquette/persistence/SegmentQueueRepository.java @@ -0,0 +1,59 @@ +package io.moquette.persistence; + +import io.moquette.broker.IQueueRepository; +import io.moquette.broker.SessionMessageQueue; +import io.moquette.broker.SessionRegistry; +import io.moquette.broker.unsafequeues.Queue; +import io.moquette.broker.unsafequeues.QueueException; +import io.moquette.broker.unsafequeues.QueuePool; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Set; + +public class SegmentQueueRepository implements IQueueRepository { + + private static final Logger LOG = LoggerFactory.getLogger(SegmentQueueRepository.class); + + private final QueuePool queuePool; + + public SegmentQueueRepository(String path, int pageSize, int segmentSize) throws QueueException { + queuePool = QueuePool.loadQueues(Paths.get(path), pageSize, segmentSize); + } + + public SegmentQueueRepository(Path path, int pageSize, int segmentSize) throws QueueException { + queuePool = QueuePool.loadQueues(path, pageSize, segmentSize); + } + + @Override + public Set listQueueNames() { + return queuePool.queueNames(); + } + + @Override + public boolean containsQueue(String clientId) { + return listQueueNames().contains(clientId); + } + + @Override + public SessionMessageQueue getOrCreateQueue(String clientId) { + final Queue segmentedQueue; + try { + segmentedQueue = queuePool.getOrCreate(clientId); + } catch (QueueException e) { + throw new RuntimeException(e); + } + return new SegmentPersistentQueue(segmentedQueue); + } + + @Override + public void close() { + try { + queuePool.close(); + } catch (QueueException e) { + LOG.error("Error saving state of the queue pool", e); + } + } +} diff --git a/moquette-0.16/broker/src/test/java/io/moquette/Utils.java b/moquette-0.17/broker/src/test/java/io/moquette/Utils.java similarity index 100% rename from moquette-0.16/broker/src/test/java/io/moquette/Utils.java rename to moquette-0.17/broker/src/test/java/io/moquette/Utils.java diff --git a/moquette-0.16/broker/src/test/java/io/moquette/broker/BrokerConfigurationTest.java b/moquette-0.17/broker/src/test/java/io/moquette/broker/BrokerConfigurationTest.java similarity index 87% rename from moquette-0.16/broker/src/test/java/io/moquette/broker/BrokerConfigurationTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/broker/BrokerConfigurationTest.java index 9027beda..ba0581ba 100644 --- a/moquette-0.16/broker/src/test/java/io/moquette/broker/BrokerConfigurationTest.java +++ b/moquette-0.17/broker/src/test/java/io/moquette/broker/BrokerConfigurationTest.java @@ -21,6 +21,7 @@ import java.util.Properties; +import static io.moquette.BrokerConstants.IMMEDIATE_BUFFER_FLUSH; import static org.junit.jupiter.api.Assertions.*; public class BrokerConfigurationTest { @@ -32,7 +33,7 @@ public void defaultConfig() { assertTrue(brokerConfiguration.isAllowAnonymous()); assertFalse(brokerConfiguration.isAllowZeroByteClientId()); assertFalse(brokerConfiguration.isReauthorizeSubscriptionsOnConnect()); - assertFalse(brokerConfiguration.isImmediateBufferFlush()); + assertEquals(IMMEDIATE_BUFFER_FLUSH, brokerConfiguration.getBufferFlushMillis(), "Immediate flush by default"); assertFalse(brokerConfiguration.isPeerCertificateAsUsername()); } @@ -45,7 +46,7 @@ public void configureAllowAnonymous() { assertFalse(brokerConfiguration.isAllowAnonymous()); assertFalse(brokerConfiguration.isAllowZeroByteClientId()); assertFalse(brokerConfiguration.isReauthorizeSubscriptionsOnConnect()); - assertFalse(brokerConfiguration.isImmediateBufferFlush()); + assertEquals(IMMEDIATE_BUFFER_FLUSH, brokerConfiguration.getBufferFlushMillis(), "Immediate flush by default"); assertFalse(brokerConfiguration.isPeerCertificateAsUsername()); } @@ -58,7 +59,7 @@ public void configureAllowZeroByteClientId() { assertTrue(brokerConfiguration.isAllowAnonymous()); assertTrue(brokerConfiguration.isAllowZeroByteClientId()); assertFalse(brokerConfiguration.isReauthorizeSubscriptionsOnConnect()); - assertFalse(brokerConfiguration.isImmediateBufferFlush()); + assertEquals(IMMEDIATE_BUFFER_FLUSH, brokerConfiguration.getBufferFlushMillis(), "Immediate flush by default"); assertFalse(brokerConfiguration.isPeerCertificateAsUsername()); } @@ -71,7 +72,7 @@ public void configureReauthorizeSubscriptionsOnConnect() { assertTrue(brokerConfiguration.isAllowAnonymous()); assertFalse(brokerConfiguration.isAllowZeroByteClientId()); assertTrue(brokerConfiguration.isReauthorizeSubscriptionsOnConnect()); - assertFalse(brokerConfiguration.isImmediateBufferFlush()); + assertEquals(IMMEDIATE_BUFFER_FLUSH, brokerConfiguration.getBufferFlushMillis(), "Immediate flush by default"); assertFalse(brokerConfiguration.isPeerCertificateAsUsername()); } @@ -84,7 +85,7 @@ public void configureImmediateBufferFlush() { assertTrue(brokerConfiguration.isAllowAnonymous()); assertFalse(brokerConfiguration.isAllowZeroByteClientId()); assertFalse(brokerConfiguration.isReauthorizeSubscriptionsOnConnect()); - assertTrue(brokerConfiguration.isImmediateBufferFlush()); + assertEquals(IMMEDIATE_BUFFER_FLUSH, brokerConfiguration.getBufferFlushMillis(), "No immediate flush by default"); assertFalse(brokerConfiguration.isPeerCertificateAsUsername()); } @@ -97,7 +98,6 @@ public void configurePeerCertificateAsUsername() { assertTrue(brokerConfiguration.isAllowAnonymous()); assertFalse(brokerConfiguration.isAllowZeroByteClientId()); assertFalse(brokerConfiguration.isReauthorizeSubscriptionsOnConnect()); - assertFalse(brokerConfiguration.isImmediateBufferFlush()); assertTrue(brokerConfiguration.isPeerCertificateAsUsername()); } } diff --git a/moquette-0.16/broker/src/test/java/io/moquette/broker/ConnectionTestUtils.java b/moquette-0.17/broker/src/test/java/io/moquette/broker/ConnectionTestUtils.java similarity index 100% rename from moquette-0.16/broker/src/test/java/io/moquette/broker/ConnectionTestUtils.java rename to moquette-0.17/broker/src/test/java/io/moquette/broker/ConnectionTestUtils.java diff --git a/moquette-0.17/broker/src/test/java/io/moquette/broker/ForwardableClock.java b/moquette-0.17/broker/src/test/java/io/moquette/broker/ForwardableClock.java new file mode 100644 index 00000000..50a2ee03 --- /dev/null +++ b/moquette-0.17/broker/src/test/java/io/moquette/broker/ForwardableClock.java @@ -0,0 +1,38 @@ +package io.moquette.broker; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; + +/** + * Utility class to represent a clock that can be moved in time. + * This is used for tests that needs to verify conditions after a certain amount of time. + * */ +class ForwardableClock extends Clock { + + private Clock currentClock; + + ForwardableClock(Clock clock) { + this.currentClock = clock; + } + + void forward(Duration period) { + currentClock = Clock.offset(currentClock, period); + } + + @Override + public ZoneId getZone() { + return currentClock.getZone(); + } + + @Override + public Clock withZone(ZoneId zone) { + return currentClock.withZone(zone); + } + + @Override + public Instant instant() { + return currentClock.instant(); + } +} diff --git a/moquette-0.16/broker/src/test/java/io/moquette/broker/MQTTConnectionConnectTest.java b/moquette-0.17/broker/src/test/java/io/moquette/broker/MQTTConnectionConnectTest.java similarity index 95% rename from moquette-0.16/broker/src/test/java/io/moquette/broker/MQTTConnectionConnectTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/broker/MQTTConnectionConnectTest.java index 07193f06..cc603ed2 100644 --- a/moquette-0.16/broker/src/test/java/io/moquette/broker/MQTTConnectionConnectTest.java +++ b/moquette-0.17/broker/src/test/java/io/moquette/broker/MQTTConnectionConnectTest.java @@ -30,6 +30,7 @@ import io.netty.handler.codec.mqtt.MqttMessageBuilders; import io.netty.handler.codec.mqtt.MqttVersion; import io.netty.handler.ssl.SslHandler; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -41,10 +42,14 @@ import java.security.cert.CertificateEncodingException; import java.util.HashMap; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import javax.net.ssl.SSLEngine; import javax.net.ssl.SSLPeerUnverifiedException; import javax.net.ssl.SSLSession; +import static io.moquette.BrokerConstants.NO_BUFFER_FLUSH; +import static io.moquette.broker.MQTTConnectionPublishTest.memorySessionsRepository; import static io.moquette.broker.NettyChannelAssertions.assertEqualsConnAck; import static io.netty.handler.codec.mqtt.MqttConnectReturnCode.CONNECTION_ACCEPTED; import static io.netty.handler.codec.mqtt.MqttConnectReturnCode.CONNECTION_REFUSED_BAD_USER_NAME_OR_PASSWORD; @@ -71,13 +76,15 @@ public class MQTTConnectionConnectTest { private EmbeddedChannel channel; private SessionRegistry sessionRegistry; private MqttMessageBuilders.ConnectBuilder connMsg; - private static final BrokerConfiguration CONFIG = new BrokerConfiguration(true, true, false, false); + private static final BrokerConfiguration CONFIG = new BrokerConfiguration(true, true, false, NO_BUFFER_FLUSH); private IAuthenticator mockAuthenticator; private PostOffice postOffice; private MemoryQueueRepository queueRepository; + private ScheduledExecutorService scheduler; @BeforeEach public void setUp() { + scheduler = Executors.newScheduledThreadPool(1); connMsg = MqttMessageBuilders.connect().protocolVersion(MqttVersion.MQTT_3_1).cleanSession(true); mockAuthenticator = new MockAuthenticator(singleton(FAKE_CLIENT_ID), singletonMap(TEST_USER, TEST_PWD)); @@ -89,14 +96,20 @@ public void setUp() { final PermitAllAuthorizatorPolicy authorizatorPolicy = new PermitAllAuthorizatorPolicy(); final Authorizator permitAll = new Authorizator(authorizatorPolicy); - sessionRegistry = new SessionRegistry(subscriptions, queueRepository, permitAll); + final SessionEventLoopGroup loopsGroup = new SessionEventLoopGroup(ConnectionTestUtils.NO_OBSERVERS_INTERCEPTOR, 1024); + sessionRegistry = new SessionRegistry(subscriptions, memorySessionsRepository(), queueRepository, permitAll, scheduler, loopsGroup); postOffice = new PostOffice(subscriptions, new MemoryRetainedRepository(), sessionRegistry, - ConnectionTestUtils.NO_OBSERVERS_INTERCEPTOR, permitAll, 1024); + ConnectionTestUtils.NO_OBSERVERS_INTERCEPTOR, permitAll, loopsGroup); sut = createMQTTConnection(CONFIG); channel = (EmbeddedChannel) sut.channel; } + @AfterEach + public void tearDown() { + scheduler.shutdown(); + } + private MQTTConnection createMQTTConnection(BrokerConfiguration config) { EmbeddedChannel channel = new EmbeddedChannel(); return createMQTTConnection(config, channel, postOffice); @@ -239,7 +252,7 @@ public void noPasswdAuthentication() { public void peerCertAsUsernameAuthentication() throws CertificateEncodingException, IOException, InterruptedException { final byte[] PEER_CERT_BYTES = "PEER_CERT".getBytes(StandardCharsets.UTF_8); MqttConnectMessage msg = connMsg.clientId(FAKE_CLIENT_ID).build(); - BrokerConfiguration config = new BrokerConfiguration(false, false, false, false, true); + BrokerConfiguration config = new BrokerConfiguration(false, false, false, 100, true); Certificate peerCertificate = mock(Certificate.class); // Add SslHandler to channel @@ -265,7 +278,7 @@ public void peerCertAsUsernameAuthentication() throws CertificateEncodingExcepti public void peerCertAsUsernameAuthentication_badCert() throws CertificateEncodingException, IOException, InterruptedException { final byte[] PEER_CERT_BYTES = "BAD_PEER_CERT".getBytes(StandardCharsets.UTF_8); MqttConnectMessage msg = connMsg.clientId(FAKE_CLIENT_ID).build(); - BrokerConfiguration config = new BrokerConfiguration(false, false, false, false, true); + BrokerConfiguration config = new BrokerConfiguration(false, false, false, 100, true); Certificate peerCertificate = mock(Certificate.class); // Add SslHandler to channel @@ -289,7 +302,7 @@ public void peerCertAsUsernameAuthentication_badCert() throws CertificateEncodin @Test public void prohibitAnonymousClient() { MqttConnectMessage msg = connMsg.clientId(FAKE_CLIENT_ID).build(); - BrokerConfiguration config = new BrokerConfiguration(false, true, false, false); + BrokerConfiguration config = new BrokerConfiguration(false, true, false, NO_BUFFER_FLUSH); sut = createMQTTConnection(config); channel = (EmbeddedChannel) sut.channel; @@ -307,7 +320,7 @@ public void prohibitAnonymousClient_providingUsername() { MqttConnectMessage msg = connMsg.clientId(FAKE_CLIENT_ID) .username(TEST_USER + "_fake") .build(); - BrokerConfiguration config = new BrokerConfiguration(false, true, false, false); + BrokerConfiguration config = new BrokerConfiguration(false, true, false, NO_BUFFER_FLUSH); createMQTTConnection(config); @@ -321,7 +334,7 @@ public void prohibitAnonymousClient_providingUsername() { @Test public void testZeroByteClientIdNotAllowed() { - BrokerConfiguration config = new BrokerConfiguration(false, false, false, false); + BrokerConfiguration config = new BrokerConfiguration(false, false, false, NO_BUFFER_FLUSH); sut = createMQTTConnection(config); channel = (EmbeddedChannel) sut.channel; @@ -372,7 +385,7 @@ public void testBindWithSameClientIDBadCredentialsDoesntDropExistingClient() thr EmbeddedChannel evilChannel = new EmbeddedChannel(); // Exercise - BrokerConfiguration config = new BrokerConfiguration(true, true, false, false); + BrokerConfiguration config = new BrokerConfiguration(true, true, false, NO_BUFFER_FLUSH); final MQTTConnection evilConnection = createMQTTConnection(config, evilChannel, postOffice); evilConnection.processConnect(evilClientConnMsg); diff --git a/moquette-0.16/broker/src/test/java/io/moquette/broker/MQTTConnectionPublishTest.java b/moquette-0.17/broker/src/test/java/io/moquette/broker/MQTTConnectionPublishTest.java similarity index 81% rename from moquette-0.16/broker/src/test/java/io/moquette/broker/MQTTConnectionPublishTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/broker/MQTTConnectionPublishTest.java index 7d605de8..3704d382 100644 --- a/moquette-0.16/broker/src/test/java/io/moquette/broker/MQTTConnectionPublishTest.java +++ b/moquette-0.17/broker/src/test/java/io/moquette/broker/MQTTConnectionPublishTest.java @@ -19,6 +19,7 @@ import io.moquette.broker.subscriptions.CTrieSubscriptionDirectory; import io.moquette.broker.subscriptions.ISubscriptionsDirectory; import io.moquette.broker.security.IAuthenticator; +import io.moquette.persistence.MemorySessionsRepository; import io.moquette.persistence.MemorySubscriptionsRepository; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; @@ -28,11 +29,16 @@ import io.netty.handler.codec.mqtt.MqttPublishMessage; import io.netty.handler.codec.mqtt.MqttQoS; import io.netty.handler.codec.mqtt.MqttVersion; +//import org.jetbrains.annotations.NotNull; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import static io.moquette.BrokerConstants.NO_BUFFER_FLUSH; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Collections.singleton; import static java.util.Collections.singletonMap; @@ -49,16 +55,24 @@ public class MQTTConnectionPublishTest { private SessionRegistry sessionRegistry; private MqttMessageBuilders.ConnectBuilder connMsg; private MemoryQueueRepository queueRepository; + private ScheduledExecutorService scheduler; @BeforeEach public void setUp() { connMsg = MqttMessageBuilders.connect().protocolVersion(MqttVersion.MQTT_3_1).cleanSession(true); - BrokerConfiguration config = new BrokerConfiguration(true, true, false, false); + BrokerConfiguration config = new BrokerConfiguration(true, true, false, NO_BUFFER_FLUSH); + + scheduler = Executors.newScheduledThreadPool(1); createMQTTConnection(config); } + @AfterEach + public void tearDown() { + scheduler.shutdown(); + } + private void createMQTTConnection(BrokerConfiguration config) { channel = new EmbeddedChannel(); NettyUtils.clientID(channel, "test_client"); @@ -76,12 +90,18 @@ private MQTTConnection createMQTTConnection(BrokerConfiguration config, Channel final PermitAllAuthorizatorPolicy authorizatorPolicy = new PermitAllAuthorizatorPolicy(); final Authorizator permitAll = new Authorizator(authorizatorPolicy); - sessionRegistry = new SessionRegistry(subscriptions, queueRepository, permitAll); + final SessionEventLoopGroup loopsGroup = new SessionEventLoopGroup(ConnectionTestUtils.NO_OBSERVERS_INTERCEPTOR, 1024); + sessionRegistry = new SessionRegistry(subscriptions, memorySessionsRepository(), queueRepository, permitAll, scheduler, loopsGroup); final PostOffice postOffice = new PostOffice(subscriptions, - new MemoryRetainedRepository(), sessionRegistry, ConnectionTestUtils.NO_OBSERVERS_INTERCEPTOR, permitAll, 1024); + new MemoryRetainedRepository(), sessionRegistry, ConnectionTestUtils.NO_OBSERVERS_INTERCEPTOR, permitAll, loopsGroup); return new MQTTConnection(channel, config, mockAuthenticator, sessionRegistry, postOffice); } +// @NotNull + static ISessionsRepository memorySessionsRepository() { + return new MemorySessionsRepository(); + } + @Test public void dropConnectionOnPublishWithInvalidTopicFormat() throws ExecutionException, InterruptedException { // Connect message with clean session set to true and client id is null. diff --git a/moquette-0.16/broker/src/test/java/io/moquette/broker/MemoryRetainedRepositoryTest.java b/moquette-0.17/broker/src/test/java/io/moquette/broker/MemoryRetainedRepositoryTest.java similarity index 100% rename from moquette-0.16/broker/src/test/java/io/moquette/broker/MemoryRetainedRepositoryTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/broker/MemoryRetainedRepositoryTest.java diff --git a/moquette-0.16/broker/src/test/java/io/moquette/broker/MockAuthenticator.java b/moquette-0.17/broker/src/test/java/io/moquette/broker/MockAuthenticator.java similarity index 100% rename from moquette-0.16/broker/src/test/java/io/moquette/broker/MockAuthenticator.java rename to moquette-0.17/broker/src/test/java/io/moquette/broker/MockAuthenticator.java diff --git a/moquette-0.16/broker/src/test/java/io/moquette/broker/NettyChannelAssertions.java b/moquette-0.17/broker/src/test/java/io/moquette/broker/NettyChannelAssertions.java similarity index 100% rename from moquette-0.16/broker/src/test/java/io/moquette/broker/NettyChannelAssertions.java rename to moquette-0.17/broker/src/test/java/io/moquette/broker/NettyChannelAssertions.java diff --git a/moquette-0.16/broker/src/test/java/io/moquette/broker/PostOfficeInternalPublishTest.java b/moquette-0.17/broker/src/test/java/io/moquette/broker/PostOfficeInternalPublishTest.java similarity index 92% rename from moquette-0.16/broker/src/test/java/io/moquette/broker/PostOfficeInternalPublishTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/broker/PostOfficeInternalPublishTest.java index 2f82d492..e0c27993 100644 --- a/moquette-0.16/broker/src/test/java/io/moquette/broker/PostOfficeInternalPublishTest.java +++ b/moquette-0.17/broker/src/test/java/io/moquette/broker/PostOfficeInternalPublishTest.java @@ -25,18 +25,24 @@ import io.netty.channel.Channel; import io.netty.channel.embedded.EmbeddedChannel; import io.netty.handler.codec.mqtt.*; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.util.Set; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import static io.moquette.broker.MQTTConnectionPublishTest.memorySessionsRepository; +import static io.moquette.BrokerConstants.NO_BUFFER_FLUSH; import static io.moquette.broker.PostOfficeUnsubscribeTest.CONFIG; import static io.netty.handler.codec.mqtt.MqttQoS.*; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Collections.singleton; import static java.util.Collections.singletonMap; +import java.util.List; import static org.junit.jupiter.api.Assertions.*; public class PostOfficeInternalPublishTest { @@ -54,9 +60,10 @@ public class PostOfficeInternalPublishTest { private SessionRegistry sessionRegistry; private MockAuthenticator mockAuthenticator; private static final BrokerConfiguration ALLOW_ANONYMOUS_AND_ZERO_BYTES_CLID = - new BrokerConfiguration(true, true, false, false); + new BrokerConfiguration(true, true, false, NO_BUFFER_FLUSH); private MemoryRetainedRepository retainedRepository; private MemoryQueueRepository queueRepository; + private ScheduledExecutorService scheduler; @BeforeEach public void setUp() throws ExecutionException, InterruptedException { @@ -71,6 +78,11 @@ public void setUp() throws ExecutionException, InterruptedException { ConnectionTestUtils.assertConnectAccepted(channel); } + @AfterEach + public void tearDown() { + scheduler.shutdown(); + } + private MQTTConnection createMQTTConnection(BrokerConfiguration config) { channel = new EmbeddedChannel(); return createMQTTConnection(config, channel); @@ -81,6 +93,7 @@ private MQTTConnection createMQTTConnection(BrokerConfiguration config, Channel } private void initPostOfficeAndSubsystems() { + scheduler = Executors.newScheduledThreadPool(1); subscriptions = new CTrieSubscriptionDirectory(); ISubscriptionsRepository subscriptionsRepository = new MemorySubscriptionsRepository(); subscriptions.init(subscriptionsRepository); @@ -89,9 +102,10 @@ private void initPostOfficeAndSubsystems() { final PermitAllAuthorizatorPolicy authorizatorPolicy = new PermitAllAuthorizatorPolicy(); final Authorizator permitAll = new Authorizator(authorizatorPolicy); - sessionRegistry = new SessionRegistry(subscriptions, queueRepository, permitAll); + final SessionEventLoopGroup loopsGroup = new SessionEventLoopGroup(ConnectionTestUtils.NO_OBSERVERS_INTERCEPTOR, 1024); + sessionRegistry = new SessionRegistry(subscriptions, memorySessionsRepository(), queueRepository, permitAll, scheduler, loopsGroup); sut = new PostOffice(subscriptions, retainedRepository, sessionRegistry, - ConnectionTestUtils.NO_OBSERVERS_INTERCEPTOR, permitAll, 1024); + ConnectionTestUtils.NO_OBSERVERS_INTERCEPTOR, permitAll, loopsGroup); } private void internalPublishNotRetainedTo(String topic) { @@ -324,7 +338,7 @@ protected void subscribe(MQTTConnection connection, String topic, MqttQoS desire final String clientId = connection.getClientId(); Subscription expectedSubscription = new Subscription(clientId, new Topic(topic), desiredQos); - final Set matchedSubscriptions = subscriptions.matchWithoutQosSharpening(new Topic(topic)); + final List matchedSubscriptions = subscriptions.matchWithoutQosSharpening(new Topic(topic)); assertEquals(1, matchedSubscriptions.size()); final Subscription onlyMatchedSubscription = matchedSubscriptions.iterator().next(); assertEquals(expectedSubscription, onlyMatchedSubscription); diff --git a/moquette-0.16/broker/src/test/java/io/moquette/broker/PostOfficePublishTest.java b/moquette-0.17/broker/src/test/java/io/moquette/broker/PostOfficePublishTest.java similarity index 96% rename from moquette-0.16/broker/src/test/java/io/moquette/broker/PostOfficePublishTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/broker/PostOfficePublishTest.java index 6a8ed3bb..3a6355f6 100644 --- a/moquette-0.16/broker/src/test/java/io/moquette/broker/PostOfficePublishTest.java +++ b/moquette-0.17/broker/src/test/java/io/moquette/broker/PostOfficePublishTest.java @@ -34,10 +34,10 @@ import java.nio.charset.Charset; import java.util.*; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; +import java.util.concurrent.*; +import static io.moquette.broker.MQTTConnectionPublishTest.memorySessionsRepository; +import static io.moquette.BrokerConstants.NO_BUFFER_FLUSH; import static io.moquette.broker.PostOfficeUnsubscribeTest.CONFIG; import static io.netty.handler.codec.mqtt.MqttQoS.AT_LEAST_ONCE; import static io.netty.handler.codec.mqtt.MqttQoS.AT_MOST_ONCE; @@ -66,9 +66,10 @@ public class PostOfficePublishTest { private SessionRegistry sessionRegistry; private MockAuthenticator mockAuthenticator; static final BrokerConfiguration ALLOW_ANONYMOUS_AND_ZERO_BYTES_CLID = - new BrokerConfiguration(true, true, false, false); + new BrokerConfiguration(true, true, false, NO_BUFFER_FLUSH); private MemoryRetainedRepository retainedRepository; private MemoryQueueRepository queueRepository; + private ScheduledExecutorService scheduler; @BeforeEach public void setUp() { @@ -83,6 +84,7 @@ public void setUp() { @AfterEach public void tearDown() { + scheduler.shutdown(); sut.terminate(); } @@ -95,6 +97,7 @@ private MQTTConnection createMQTTConnection(BrokerConfiguration config, Channel } private void initPostOfficeAndSubsystems() { + scheduler = Executors.newScheduledThreadPool(1); subscriptions = new CTrieSubscriptionDirectory(); ISubscriptionsRepository subscriptionsRepository = new MemorySubscriptionsRepository(); subscriptions.init(subscriptionsRepository); @@ -103,9 +106,10 @@ private void initPostOfficeAndSubsystems() { final PermitAllAuthorizatorPolicy authorizatorPolicy = new PermitAllAuthorizatorPolicy(); final Authorizator permitAll = new Authorizator(authorizatorPolicy); - sessionRegistry = new SessionRegistry(subscriptions, queueRepository, permitAll); + final SessionEventLoopGroup loopsGroup = new SessionEventLoopGroup(ConnectionTestUtils.NO_OBSERVERS_INTERCEPTOR, 1024); + sessionRegistry = new SessionRegistry(subscriptions, memorySessionsRepository(), queueRepository, permitAll, scheduler, loopsGroup); sut = new PostOffice(subscriptions, retainedRepository, sessionRegistry, - ConnectionTestUtils.NO_OBSERVERS_INTERCEPTOR, permitAll, 1024); + ConnectionTestUtils.NO_OBSERVERS_INTERCEPTOR, permitAll, loopsGroup); } @Test @@ -192,7 +196,7 @@ protected void subscribe(MQTTConnection connection, String topic, MqttQoS desire final String clientId = connection.getClientId(); Subscription expectedSubscription = new Subscription(clientId, new Topic(topic), desiredQos); - final Set matchedSubscriptions = subscriptions.matchWithoutQosSharpening(new Topic(topic)); + final List matchedSubscriptions = subscriptions.matchWithoutQosSharpening(new Topic(topic)); assertEquals(1, matchedSubscriptions.size()); final Subscription onlyMatchedSubscription = matchedSubscriptions.iterator().next(); assertEquals(expectedSubscription, onlyMatchedSubscription); diff --git a/moquette-0.16/broker/src/test/java/io/moquette/broker/PostOfficeSubscribeTest.java b/moquette-0.17/broker/src/test/java/io/moquette/broker/PostOfficeSubscribeTest.java similarity index 92% rename from moquette-0.16/broker/src/test/java/io/moquette/broker/PostOfficeSubscribeTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/broker/PostOfficeSubscribeTest.java index 5318d6ca..dd97da47 100644 --- a/moquette-0.16/broker/src/test/java/io/moquette/broker/PostOfficeSubscribeTest.java +++ b/moquette-0.17/broker/src/test/java/io/moquette/broker/PostOfficeSubscribeTest.java @@ -28,16 +28,17 @@ import io.netty.channel.Channel; import io.netty.channel.embedded.EmbeddedChannel; import io.netty.handler.codec.mqtt.*; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.nio.charset.Charset; import java.util.List; import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; +import java.util.concurrent.*; +import static io.moquette.broker.MQTTConnectionPublishTest.memorySessionsRepository; +import static io.moquette.BrokerConstants.NO_BUFFER_FLUSH; import static io.moquette.broker.PostOfficePublishTest.ALLOW_ANONYMOUS_AND_ZERO_BYTES_CLID; import static io.moquette.broker.PostOfficePublishTest.SUBSCRIBER_ID; import static io.netty.handler.codec.mqtt.MqttConnectReturnCode.CONNECTION_ACCEPTED; @@ -67,8 +68,9 @@ public class PostOfficeSubscribeTest { private MqttConnectMessage connectMessage; private IAuthenticator mockAuthenticator; private SessionRegistry sessionRegistry; - public static final BrokerConfiguration CONFIG = new BrokerConfiguration(true, true, false, false); + public static final BrokerConfiguration CONFIG = new BrokerConfiguration(true, true, false, NO_BUFFER_FLUSH); private MemoryQueueRepository queueRepository; + private ScheduledExecutorService scheduler; @BeforeEach public void setUp() { @@ -80,12 +82,18 @@ public void setUp() { createMQTTConnection(CONFIG); } + @AfterEach + public void tearDown() { + scheduler.shutdown(); + } + private void createMQTTConnection(BrokerConfiguration config) { channel = new EmbeddedChannel(); connection = createMQTTConnection(config, channel); } private void prepareSUT() { + scheduler = Executors.newScheduledThreadPool(1); mockAuthenticator = new MockAuthenticator(singleton(FAKE_CLIENT_ID), singletonMap(TEST_USER, TEST_PWD)); subscriptions = new CTrieSubscriptionDirectory(); @@ -95,9 +103,10 @@ private void prepareSUT() { final PermitAllAuthorizatorPolicy authorizatorPolicy = new PermitAllAuthorizatorPolicy(); final Authorizator permitAll = new Authorizator(authorizatorPolicy); - sessionRegistry = new SessionRegistry(subscriptions, queueRepository, permitAll); + final SessionEventLoopGroup loopsGroup = new SessionEventLoopGroup(ConnectionTestUtils.NO_OBSERVERS_INTERCEPTOR, 1024); + sessionRegistry = new SessionRegistry(subscriptions, memorySessionsRepository(), queueRepository, permitAll, scheduler, loopsGroup); sut = new PostOffice(subscriptions, new MemoryRetainedRepository(), sessionRegistry, - ConnectionTestUtils.NO_OBSERVERS_INTERCEPTOR, permitAll, 1024); + ConnectionTestUtils.NO_OBSERVERS_INTERCEPTOR, permitAll, loopsGroup); } private MQTTConnection createMQTTConnection(BrokerConfiguration config, Channel channel) { @@ -135,7 +144,7 @@ protected void subscribe(EmbeddedChannel channel, String topic, MqttQoS desiredQ final String clientId = NettyUtils.clientID(channel); Subscription expectedSubscription = new Subscription(clientId, new Topic(topic), desiredQos); - final Set matchedSubscriptions = subscriptions.matchWithoutQosSharpening(new Topic(topic)); + final List matchedSubscriptions = subscriptions.matchWithoutQosSharpening(new Topic(topic)); assertEquals(1, matchedSubscriptions.size()); final Subscription onlyMatchedSubscription = matchedSubscriptions.iterator().next(); assertEquals(expectedSubscription, onlyMatchedSubscription); @@ -155,7 +164,7 @@ protected void subscribe(MQTTConnection connection, String topic, MqttQoS desire final String clientId = connection.getClientId(); Subscription expectedSubscription = new Subscription(clientId, new Topic(topic), desiredQos); - final Set matchedSubscriptions = subscriptions.matchWithoutQosSharpening(new Topic(topic)); + final List matchedSubscriptions = subscriptions.matchWithoutQosSharpening(new Topic(topic)); assertEquals(1, matchedSubscriptions.size()); final Subscription onlyMatchedSubscription = matchedSubscriptions.iterator().next(); assertEquals(expectedSubscription, onlyMatchedSubscription); @@ -169,8 +178,9 @@ public void testSubscribedToNotAuthorizedTopic() throws ExecutionException, Inte when(prohibitReadOnNewsTopic.canRead(eq(new Topic(NEWS_TOPIC)), eq(FAKE_USER_NAME), eq(FAKE_CLIENT_ID))) .thenReturn(false); + final SessionEventLoopGroup loopsGroup = new SessionEventLoopGroup(ConnectionTestUtils.NO_OBSERVERS_INTERCEPTOR, 1024); sut = new PostOffice(subscriptions, new MemoryRetainedRepository(), sessionRegistry, - ConnectionTestUtils.NO_OBSERVERS_INTERCEPTOR, new Authorizator(prohibitReadOnNewsTopic), 1024); + ConnectionTestUtils.NO_OBSERVERS_INTERCEPTOR, new Authorizator(prohibitReadOnNewsTopic), loopsGroup); connection.processConnect(connectMessage).completableFuture().get(); ConnectionTestUtils.assertConnectAccepted(channel); diff --git a/moquette-0.16/broker/src/test/java/io/moquette/broker/PostOfficeUnsubscribeTest.java b/moquette-0.17/broker/src/test/java/io/moquette/broker/PostOfficeUnsubscribeTest.java similarity index 95% rename from moquette-0.16/broker/src/test/java/io/moquette/broker/PostOfficeUnsubscribeTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/broker/PostOfficeUnsubscribeTest.java index 579a64fa..91df197b 100644 --- a/moquette-0.16/broker/src/test/java/io/moquette/broker/PostOfficeUnsubscribeTest.java +++ b/moquette-0.17/broker/src/test/java/io/moquette/broker/PostOfficeUnsubscribeTest.java @@ -27,20 +27,21 @@ import io.netty.channel.Channel; import io.netty.channel.embedded.EmbeddedChannel; import io.netty.handler.codec.mqtt.*; -import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.nio.charset.Charset; import java.util.Collections; import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; +import java.util.concurrent.*; +import static io.moquette.broker.MQTTConnectionPublishTest.memorySessionsRepository; +import static io.moquette.BrokerConstants.NO_BUFFER_FLUSH; import static io.moquette.broker.PostOfficePublishTest.PUBLISHER_ID; import static io.netty.handler.codec.mqtt.MqttQoS.*; import static java.util.Collections.*; +import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -59,8 +60,9 @@ public class PostOfficeUnsubscribeTest { private MqttConnectMessage connectMessage; private IAuthenticator mockAuthenticator; private SessionRegistry sessionRegistry; - public static final BrokerConfiguration CONFIG = new BrokerConfiguration(true, true, false, false); + public static final BrokerConfiguration CONFIG = new BrokerConfiguration(true, true, false, NO_BUFFER_FLUSH); private MemoryQueueRepository queueRepository; + private ScheduledExecutorService scheduler; @BeforeEach public void setUp() { @@ -72,12 +74,18 @@ public void setUp() { createMQTTConnection(CONFIG); } + @AfterEach + public void tearDown() { + scheduler.shutdown(); + } + private void createMQTTConnection(BrokerConfiguration config) { channel = new EmbeddedChannel(); connection = createMQTTConnection(config, channel); } private void prepareSUT() { + scheduler = Executors.newScheduledThreadPool(1); mockAuthenticator = new MockAuthenticator(singleton(FAKE_CLIENT_ID), singletonMap(TEST_USER, TEST_PWD)); subscriptions = new CTrieSubscriptionDirectory(); @@ -87,9 +95,10 @@ private void prepareSUT() { final PermitAllAuthorizatorPolicy authorizatorPolicy = new PermitAllAuthorizatorPolicy(); final Authorizator permitAll = new Authorizator(authorizatorPolicy); - sessionRegistry = new SessionRegistry(subscriptions, queueRepository, permitAll); + final SessionEventLoopGroup loopsGroup = new SessionEventLoopGroup(ConnectionTestUtils.NO_OBSERVERS_INTERCEPTOR, 1024); + sessionRegistry = new SessionRegistry(subscriptions, memorySessionsRepository(), queueRepository, permitAll, scheduler, loopsGroup); sut = new PostOffice(subscriptions, new MemoryRetainedRepository(), sessionRegistry, - ConnectionTestUtils.NO_OBSERVERS_INTERCEPTOR, permitAll, 1024); + ConnectionTestUtils.NO_OBSERVERS_INTERCEPTOR, permitAll, loopsGroup); } private MQTTConnection createMQTTConnection(BrokerConfiguration config, Channel channel) { @@ -115,7 +124,7 @@ protected void subscribe(MQTTConnection connection, String topic, MqttQoS desire final String clientId = connection.getClientId(); Subscription expectedSubscription = new Subscription(clientId, new Topic(topic), desiredQos); - final Set matchedSubscriptions = subscriptions.matchQosSharpening(new Topic(topic)); + final List matchedSubscriptions = subscriptions.matchQosSharpening(new Topic(topic)); assertEquals(1, matchedSubscriptions.size()); //assertTrue(matchedSubscriptions.size() >=1); final Subscription onlyMatchedSubscription = matchedSubscriptions.iterator().next(); diff --git a/moquette-0.16/broker/src/test/java/io/moquette/broker/SessionRegistryTest.java b/moquette-0.17/broker/src/test/java/io/moquette/broker/SessionRegistryTest.java similarity index 53% rename from moquette-0.16/broker/src/test/java/io/moquette/broker/SessionRegistryTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/broker/SessionRegistryTest.java index 78d1c9f8..b6186c17 100644 --- a/moquette-0.16/broker/src/test/java/io/moquette/broker/SessionRegistryTest.java +++ b/moquette-0.17/broker/src/test/java/io/moquette/broker/SessionRegistryTest.java @@ -31,44 +31,75 @@ import io.netty.handler.codec.mqtt.MqttMessageBuilders; import io.netty.handler.codec.mqtt.MqttQoS; import io.netty.handler.codec.mqtt.MqttVersion; +import org.awaitility.Awaitility; import org.h2.mvstore.MVMap; import org.h2.mvstore.MVStore; +import org.hamcrest.Matchers; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; import java.nio.charset.StandardCharsets; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; +import java.util.Collection; +import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import static io.moquette.broker.MQTTConnectionPublishTest.memorySessionsRepository; +import static io.moquette.BrokerConstants.NO_BUFFER_FLUSH; import static io.moquette.broker.NettyChannelAssertions.assertEqualsConnAck; import static io.netty.handler.codec.mqtt.MqttConnectReturnCode.CONNECTION_ACCEPTED; import static java.util.Collections.singleton; import static java.util.Collections.singletonMap; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; public class SessionRegistryTest { + public static final boolean ANY_BOOLEAN = false; + private static final Logger LOG = LoggerFactory.getLogger(SessionRegistryTest.class); static final String FAKE_CLIENT_ID = "FAKE_123"; static final String TEST_USER = "fakeuser"; static final String TEST_PWD = "fakepwd"; + public static final int GLOBAL_SESSION_EXPIRY_SECONDS = 100 * 24 * 60 * 60; // 100 days private MQTTConnection connection; private EmbeddedChannel channel; private SessionRegistry sut; private MqttMessageBuilders.ConnectBuilder connMsg; private static final BrokerConfiguration ALLOW_ANONYMOUS_AND_ZEROBYTE_CLIENT_ID = - new BrokerConfiguration(true, true, false, false); + new BrokerConfiguration(true, true, false, NO_BUFFER_FLUSH); private MemoryQueueRepository queueRepository; + private ScheduledExecutorService scheduler; + private final Clock pointInTimeFixedClock = Clock.fixed(Instant.parse("2023-03-26T18:09:30.00Z"), ZoneId.of("Europe/Rome")); + private ForwardableClock slidingClock = new ForwardableClock(pointInTimeFixedClock); + private ISessionsRepository sessionRepository; @BeforeEach public void setUp() { connMsg = MqttMessageBuilders.connect().protocolVersion(MqttVersion.MQTT_3_1).cleanSession(true); + scheduler = Executors.newScheduledThreadPool(1); + createMQTTConnection(ALLOW_ANONYMOUS_AND_ZEROBYTE_CLIENT_ID); } + @AfterEach + public void tearDown() { + scheduler.shutdown(); + } + private void createMQTTConnection(BrokerConfiguration config) { channel = new EmbeddedChannel(); connection = createMQTTConnection(config, channel); @@ -85,15 +116,17 @@ private MQTTConnection createMQTTConnection(BrokerConfiguration config, Channel final PermitAllAuthorizatorPolicy authorizatorPolicy = new PermitAllAuthorizatorPolicy(); final Authorizator permitAll = new Authorizator(authorizatorPolicy); - sut = new SessionRegistry(subscriptions, queueRepository, permitAll); + final SessionEventLoopGroup loopsGroup = new SessionEventLoopGroup(ConnectionTestUtils.NO_OBSERVERS_INTERCEPTOR, 1024); + sessionRepository = memorySessionsRepository(); + sut = new SessionRegistry(subscriptions, sessionRepository, queueRepository, permitAll, scheduler, slidingClock, GLOBAL_SESSION_EXPIRY_SECONDS, loopsGroup); final PostOffice postOffice = new PostOffice(subscriptions, - new MemoryRetainedRepository(), sut, ConnectionTestUtils.NO_OBSERVERS_INTERCEPTOR, permitAll, 1024); + new MemoryRetainedRepository(), sut, ConnectionTestUtils.NO_OBSERVERS_INTERCEPTOR, permitAll, loopsGroup); return new MQTTConnection(channel, config, mockAuthenticator, sut, postOffice); } @Test public void testConnAckContainsSessionPresentFlag() { - System.out.println("testConnAckContainsSessionPresentFlag invoked"); + LOG.info("testConnAckContainsSessionPresentFlag invoked"); MqttConnectMessage msg = connMsg.clientId(FAKE_CLIENT_ID) .protocolVersion(MqttVersion.MQTT_3_1_1) .build(); @@ -118,6 +151,7 @@ public void testConnAckContainsSessionPresentFlag() { @Test public void connectWithCleanSessionUpdateClientSession() throws ExecutionException, InterruptedException { + LOG.info("connectWithCleanSessionUpdateClientSession"); // first connect with clean session true MqttConnectMessage msg = connMsg.clientId(FAKE_CLIENT_ID).cleanSession(true).build(); connection.processConnect(msg).completableFuture().get(); @@ -142,8 +176,50 @@ public void connectWithCleanSessionUpdateClientSession() throws ExecutionExcepti assertFalse(session.isClean()); } + @Test + public void testDropSessionWithNullClientId() { + assertFalse(sut.dropSession(null, ANY_BOOLEAN), "Can't be successful when null clientId is passed"); + } + + @Test + public void testDropSessionWithNotExistingClientId() { + assertFalse(sut.dropSession(FAKE_CLIENT_ID, ANY_BOOLEAN), "Can't be successful when non existing clientId is passed"); + } + + @Test + public void testDropSessionToForceClosingConnectedSessionWithoutCleaning() throws ExecutionException, InterruptedException { + MqttConnectMessage msg = connMsg.clientId(FAKE_CLIENT_ID) + .protocolVersion(MqttVersion.MQTT_3_1_1) + .build(); + connection.processConnect(msg).completableFuture().get(); + assertEqualsConnAck(CONNECTION_ACCEPTED, channel.readOutbound()); + + // Exercise + assertTrue(sut.dropSession(FAKE_CLIENT_ID, false), "Operation must have successfully terminated"); + + // Verify + final Session stillStoredSession = sut.retrieve(FAKE_CLIENT_ID); + assertTrue(stillStoredSession.disconnected(), "session is still present and disconnected"); + } + + @Test + public void testDropSessionToForceClosingConnectedSessionWithCleaning() throws ExecutionException, InterruptedException { + MqttConnectMessage msg = connMsg.clientId(FAKE_CLIENT_ID) + .protocolVersion(MqttVersion.MQTT_3_1_1) + .build(); + connection.processConnect(msg).completableFuture().get(); + assertEqualsConnAck(CONNECTION_ACCEPTED, channel.readOutbound()); + + // Exercise + assertTrue(sut.dropSession(FAKE_CLIENT_ID, true), "Operation must have successfully terminated"); + + // Verify + assertNull(sut.retrieve(FAKE_CLIENT_ID), "Session state can't be present"); + } + @Test public void testSerializabilityOfPublishedMessage() { + LOG.info("testSerializabilityOfPublishedMessage"); MVStore mvStore = new MVStore.Builder() .fileName(BrokerConstants.DEFAULT_PERSISTENT_PATH) .autoCommitDisabled() @@ -181,4 +257,67 @@ public void testSerializabilityOfPublishedMessage() { assertFalse(dbFile.exists()); } } + + @Test + public void givenSessionWithExpireTimeWhenAfterExpirationIsPassedThenSessionIsRemoved() { + LOG.info("givenSessionWithExpireTimeWhenAfterExpirationIsPassedThenSessionIsRemoved"); + + // insert a not clean session that should expire in GLOBAL_SESSION_EXPIRY_SECONDS (100 days) + final String clientId = "client_to_be_removed"; + final SessionRegistry.SessionCreationResult res = sut.createOrReopenSession(connMsg.cleanSession(false).build(), clientId, "User"); + assertEquals(SessionRegistry.CreationModeEnum.CREATED_CLEAN_NEW, res.mode, "Not clean session must be created"); + + // remove it, so that it's tracked in the inner delay queue + sut.connectionClosed(res.session); + assertEquals(1, sessionRepository.list().size(), "Not clean session must be persisted"); + + // move time forward + Duration moreThenSessionExpiration = Duration.ofSeconds(GLOBAL_SESSION_EXPIRY_SECONDS).plusSeconds(10); + slidingClock.forward(moreThenSessionExpiration); + + // check the session has been removed + Awaitility + .await() + .atMost(3 * SessionRegistry.EXPIRED_SESSION_CLEANER_TASK_INTERVAL.toMillis(), TimeUnit.MILLISECONDS) + .until(sessionsList(), Matchers.empty()); + } + + @Test + public void givenSessionThatExpiresWhenReopenIsNotAnymoreTrackedForExpiration() throws InterruptedException { + LOG.info("givenSessionThatExpiresWhenReopenIsNotAnymoreTrackedForExpiration"); + final String clientId = "client_to_be_removed"; + SessionRegistry.SessionCreationResult res = sut.createOrReopenSession(connMsg.cleanSession(false).build(), clientId, "User"); + assertEquals(SessionRegistry.CreationModeEnum.CREATED_CLEAN_NEW, res.mode, "Non clean session must be created"); + res.session.completeConnection(); + + // remove it, so that it's tracked in the inner delay queue + sut.connectionClosed(res.session); + assertEquals(1, sessionRepository.list().size(), "Non clean session must be persisted"); + + // Exercise + // reopen the session + res = sut.createOrReopenSession(connMsg.cleanSession(false).build(), clientId, "User"); + assertEquals(SessionRegistry.CreationModeEnum.REOPEN_EXISTING, res.mode, "Non clean session must be re-opened"); + + // move time forward + Duration moreThenSessionExpiration = Duration.ofSeconds(GLOBAL_SESSION_EXPIRY_SECONDS).plusSeconds(10); + slidingClock.forward(moreThenSessionExpiration); + + // Verify that the session reopened is still listed + final Collection activeSessions = sessionRepository.list(); + assertEquals(1, activeSessions.size(), "There must be active one session"); + final ISessionsRepository.SessionData element = activeSessions.iterator().next(); + assertFalse(element.expireAt().isPresent(), "Shouldn't have an expiration configured"); + + // wait the session expiry thread kicks in for at least one execution + Thread.sleep(3 * SessionRegistry.EXPIRED_SESSION_CLEANER_TASK_INTERVAL.toMillis()); + Awaitility + .await() + .atMost(2 * SessionRegistry.EXPIRED_SESSION_CLEANER_TASK_INTERVAL.toMillis(), TimeUnit.MILLISECONDS) + .until(sessionsList(), Matchers.not(Matchers.empty())); + } + + private Callable> sessionsList() { + return () -> sessionRepository.list(); + } } diff --git a/moquette-0.16/broker/src/test/java/io/moquette/broker/SessionTest.java b/moquette-0.17/broker/src/test/java/io/moquette/broker/SessionTest.java similarity index 89% rename from moquette-0.16/broker/src/test/java/io/moquette/broker/SessionTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/broker/SessionTest.java index e440f6ca..66c6c237 100644 --- a/moquette-0.16/broker/src/test/java/io/moquette/broker/SessionTest.java +++ b/moquette-0.17/broker/src/test/java/io/moquette/broker/SessionTest.java @@ -6,13 +6,21 @@ import io.netty.buffer.UnpooledByteBufAllocator; import io.netty.channel.embedded.EmbeddedChannel; import io.netty.handler.codec.mqtt.MqttQoS; +import io.netty.handler.codec.mqtt.MqttVersion; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import static io.moquette.BrokerConstants.FLIGHT_BEFORE_RESEND_MS; import io.moquette.broker.subscriptions.Subscription; + +import java.time.Clock; import java.util.Arrays; import org.assertj.core.api.Assertions; + +import static io.moquette.broker.Session.INFINITE_EXPIRY; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static io.moquette.BrokerConstants.NO_BUFFER_FLUSH; import static org.junit.jupiter.api.Assertions.*; public class SessionTest { @@ -27,7 +35,9 @@ public class SessionTest { public void setUp() { testChannel = new EmbeddedChannel(); queuedMessages = new InMemoryQueue(); - client = new Session(CLIENT_ID, true, null, queuedMessages); + final Clock clock = Clock.systemDefaultZone(); + final ISessionsRepository.SessionData data = new ISessionsRepository.SessionData(CLIENT_ID, MqttVersion.MQTT_3_1_1, INFINITE_EXPIRY, clock); + client = new Session(data, true, null, queuedMessages); createConnection(client); } @@ -114,7 +124,7 @@ public void testRemoveSubscription() { } private void createConnection(Session client) { - BrokerConfiguration brokerConfiguration = new BrokerConfiguration(true, false, false, false); + BrokerConfiguration brokerConfiguration = new BrokerConfiguration(true, false, false, NO_BUFFER_FLUSH); MQTTConnection mqttConnection = new MQTTConnection(testChannel, brokerConfiguration, null, null, null); client.markConnecting(); client.bind(mqttConnection); diff --git a/moquette-0.16/broker/src/test/java/io/moquette/broker/config/ClasspathResourceLoaderTest.java b/moquette-0.17/broker/src/test/java/io/moquette/broker/config/ClasspathResourceLoaderTest.java similarity index 100% rename from moquette-0.16/broker/src/test/java/io/moquette/broker/config/ClasspathResourceLoaderTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/broker/config/ClasspathResourceLoaderTest.java diff --git a/moquette-0.16/broker/src/test/java/io/moquette/broker/config/ConfigurationParserTest.java b/moquette-0.17/broker/src/test/java/io/moquette/broker/config/ConfigurationParserTest.java similarity index 100% rename from moquette-0.16/broker/src/test/java/io/moquette/broker/config/ConfigurationParserTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/broker/config/ConfigurationParserTest.java diff --git a/moquette-0.17/broker/src/test/java/io/moquette/broker/config/FluentConfigUsageTest.java b/moquette-0.17/broker/src/test/java/io/moquette/broker/config/FluentConfigUsageTest.java new file mode 100644 index 00000000..9dc2e7ba --- /dev/null +++ b/moquette-0.17/broker/src/test/java/io/moquette/broker/config/FluentConfigUsageTest.java @@ -0,0 +1,34 @@ +package io.moquette.broker.config; + +import io.moquette.BrokerConstants; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class FluentConfigUsageTest { + + private void assertPropertyEquals(IConfig config, String expected, String propertyName) { + assertEquals(expected, config.getProperty(propertyName)); + } + + @Test + public void checkTLSSubscopeCanConfigureTheRightProperties() { + // given + IConfig config = new FluentConfig() + .withTLS(tls -> { + tls.sslProvider(FluentConfig.SSLProvider.SSL); + tls.jksPath("/tmp/keystore.jks"); + tls.keyStoreType(FluentConfig.KeyStoreType.JKS); + tls.keyStorePassword("s3cr3t"); + tls.keyManagerPassword("sup3rs3cr3t"); + }).build(); + + // then after config build + // expect the settings are properly set + assertPropertyEquals(config, "ssl", BrokerConstants.SSL_PROVIDER); + assertPropertyEquals(config, "/tmp/keystore.jks", BrokerConstants.JKS_PATH_PROPERTY_NAME); + assertPropertyEquals(config, "jks", BrokerConstants.KEY_STORE_TYPE); + assertPropertyEquals(config, "s3cr3t", BrokerConstants.KEY_STORE_PASSWORD_PROPERTY_NAME); + assertPropertyEquals(config, "sup3rs3cr3t", BrokerConstants.KEY_MANAGER_PASSWORD_PROPERTY_NAME); + } +} diff --git a/moquette-0.16/broker/src/test/java/io/moquette/broker/security/ACLFileParserTest.java b/moquette-0.17/broker/src/test/java/io/moquette/broker/security/ACLFileParserTest.java similarity index 100% rename from moquette-0.16/broker/src/test/java/io/moquette/broker/security/ACLFileParserTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/broker/security/ACLFileParserTest.java diff --git a/moquette-0.16/broker/src/test/java/io/moquette/broker/security/AuthorizationsCollectorTest.java b/moquette-0.17/broker/src/test/java/io/moquette/broker/security/AuthorizationsCollectorTest.java similarity index 100% rename from moquette-0.16/broker/src/test/java/io/moquette/broker/security/AuthorizationsCollectorTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/broker/security/AuthorizationsCollectorTest.java diff --git a/moquette-0.16/broker/src/test/java/io/moquette/broker/security/DBAuthenticatorTest.java b/moquette-0.17/broker/src/test/java/io/moquette/broker/security/DBAuthenticatorTest.java similarity index 100% rename from moquette-0.16/broker/src/test/java/io/moquette/broker/security/DBAuthenticatorTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/broker/security/DBAuthenticatorTest.java diff --git a/moquette-0.16/broker/src/test/java/io/moquette/broker/security/FileAuthenticatorTest.java b/moquette-0.17/broker/src/test/java/io/moquette/broker/security/FileAuthenticatorTest.java similarity index 100% rename from moquette-0.16/broker/src/test/java/io/moquette/broker/security/FileAuthenticatorTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/broker/security/FileAuthenticatorTest.java diff --git a/moquette-0.17/broker/src/test/java/io/moquette/broker/subscriptions/CTrieSpeedTest.java b/moquette-0.17/broker/src/test/java/io/moquette/broker/subscriptions/CTrieSpeedTest.java new file mode 100644 index 00000000..6d696cf0 --- /dev/null +++ b/moquette-0.17/broker/src/test/java/io/moquette/broker/subscriptions/CTrieSpeedTest.java @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2012-2023 The original author or authors + * ------------------------------------------------------ + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Apache License v2.0 which accompanies this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * The Apache License v2.0 is available at + * http://www.opensource.org/licenses/apache2.0.php + * + * You may elect to redistribute this code under either of these licenses. + */ +package io.moquette.broker.subscriptions; + +import io.netty.handler.codec.mqtt.MqttQoS; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import static io.moquette.broker.subscriptions.Topic.asTopic; + +@Disabled("No perf testing") +public class CTrieSpeedTest { + + private static final Logger LOGGER = LoggerFactory.getLogger(CTrieSpeedTest.class.getName()); + + private static final int MAX_DURATION_S = 5 * 60; + private static final int CHECK_INTERVAL = 50_000; + private static final int TOTAL_SUBSCRIPTIONS = 500_000; + + static Subscription clientSubOnTopic(String clientID, String topicName) { + return new Subscription(clientID, asTopic(topicName), null); + } + + @Test + @Timeout(value = MAX_DURATION_S) + public void testManyClientsFewTopics() { + List subscriptionList = prepareSubscriptionsManyClientsFewTopic(); + createSubscriptions(subscriptionList); + } + + @Test + @Timeout(value = MAX_DURATION_S) + public void testFlat() { + List results = prepareSubscriptionsFlat(); + createSubscriptions(results); + } + + @Test + @Timeout(value = MAX_DURATION_S) + public void testDeep() { + List results = prepareSubscriptionsDeep(); + createSubscriptions(results); + } + + public void createSubscriptions(List results) { + int count = 0; + long start = System.currentTimeMillis(); + int log = CHECK_INTERVAL; + CTrie tree = new CTrie(); + for (Subscription result : results) { + tree.addToTree(result); + count++; + log--; + if (log <= 0) { + log = CHECK_INTERVAL; + long end = System.currentTimeMillis(); + long duration = end - start; + LOGGER.info("Added {} subscriptions in {} ms ({}/ms)", count, duration, Math.round(1.0 * count / duration)); + } + if (Thread.currentThread().isInterrupted()) { + return; + } + } + long end = System.currentTimeMillis(); + long duration = end - start; + LOGGER.info("Added " + count + " subscriptions in " + duration + " ms (" + Math.round(1000.0 * count / duration) + "/s)"); + } + + public List prepareSubscriptionsManyClientsFewTopic() { + List subscriptionList = new ArrayList<>(TOTAL_SUBSCRIPTIONS); + for (int i = 0; i < TOTAL_SUBSCRIPTIONS; i++) { + Topic topic = asTopic("topic/test/" + new Random().nextInt(1 + i % 10) + "/test"); + subscriptionList.add(new Subscription("TestClient-" + i, topic, MqttQoS.AT_LEAST_ONCE)); + } + return subscriptionList; + } + + public List prepareSubscriptionsFlat() { + List results = new ArrayList<>(TOTAL_SUBSCRIPTIONS); + int count = 0; + long start = System.currentTimeMillis(); + for (int topicNr = 0; topicNr < TOTAL_SUBSCRIPTIONS / 10; topicNr++) { + for (int clientNr = 0; clientNr < 10; clientNr++) { + count++; + results.add(clientSubOnTopic("Client-" + clientNr, "mainTopic-" + topicNr)); + } + } + long end = System.currentTimeMillis(); + long duration = end - start; + LOGGER.info("Prepared {} subscriptions in {} ms", count, duration); + return results; + } + + public List prepareSubscriptionsDeep() { + List results = new ArrayList<>(TOTAL_SUBSCRIPTIONS); + long countPerLevel = Math.round(Math.pow(TOTAL_SUBSCRIPTIONS, 0.25)); + LOGGER.info("Preparing {} subscriptions, 4 deep with {} per level", TOTAL_SUBSCRIPTIONS, countPerLevel); + int count = 0; + long start = System.currentTimeMillis(); + outerloop: + for (int clientNr = 0; clientNr < countPerLevel; clientNr++) { + for (int firstLevelNr = 0; firstLevelNr < countPerLevel; firstLevelNr++) { + for (int secondLevelNr = 0; secondLevelNr < countPerLevel; secondLevelNr++) { + for (int thirdLevelNr = 0; thirdLevelNr < countPerLevel; thirdLevelNr++) { + count++; + results.add(clientSubOnTopic("Client-" + clientNr, "mainTopic-" + firstLevelNr + "/subTopic-" + secondLevelNr + "/subSubTopic" + thirdLevelNr)); + // Due to the 4th-power-root we don't get exactly the required number of subs. + if (count >= TOTAL_SUBSCRIPTIONS) { + break outerloop; + } + } + } + } + } + long end = System.currentTimeMillis(); + long duration = end - start; + LOGGER.info("Prepared {} subscriptions in {} ms", count, duration); + return results; + } + +} diff --git a/moquette-0.16/broker/src/test/java/io/moquette/broker/subscriptions/CTrieSubscriptionDirectoryMatchingTest.java b/moquette-0.17/broker/src/test/java/io/moquette/broker/subscriptions/CTrieSubscriptionDirectoryMatchingTest.java similarity index 96% rename from moquette-0.16/broker/src/test/java/io/moquette/broker/subscriptions/CTrieSubscriptionDirectoryMatchingTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/broker/subscriptions/CTrieSubscriptionDirectoryMatchingTest.java index a3a57a03..6deddc6c 100644 --- a/moquette-0.16/broker/src/test/java/io/moquette/broker/subscriptions/CTrieSubscriptionDirectoryMatchingTest.java +++ b/moquette-0.17/broker/src/test/java/io/moquette/broker/subscriptions/CTrieSubscriptionDirectoryMatchingTest.java @@ -27,6 +27,7 @@ import static io.moquette.broker.subscriptions.CTrieTest.clientSubOnTopic; import static io.moquette.broker.subscriptions.Topic.asTopic; +import java.util.List; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -203,7 +204,7 @@ public void testOverlappingSubscriptions() { sut.add(specificSub); //Exercise - final Set matchingForSpecific = sut.matchQosSharpening(asTopic("a/b")); + final List matchingForSpecific = sut.matchQosSharpening(asTopic("a/b")); // Verify assertThat(matchingForSpecific.size()).isEqualTo(1); @@ -234,7 +235,7 @@ public void removeSubscription_sameClients_subscribedSameTopic() { sut.removeSubscription(asTopic("/topic"), slashSub.clientId); // Verify - final Set matchingSubscriptions = sut.matchWithoutQosSharpening(asTopic("/topic")); + final List matchingSubscriptions = sut.matchWithoutQosSharpening(asTopic("/topic")); assertThat(matchingSubscriptions).isEmpty(); } @@ -252,7 +253,7 @@ public void duplicatedSubscriptionsWithDifferentQos() { this.sut.add(client1SubQoS2); // Verify - Set subscriptions = this.sut.matchQosSharpening(asTopic("client/test/b")); + List subscriptions = this.sut.matchQosSharpening(asTopic("client/test/b")); assertThat(subscriptions).contains(client1SubQoS2); assertThat(subscriptions).contains(client2Sub); diff --git a/moquette-0.16/broker/src/test/java/io/moquette/broker/subscriptions/CTrieTest.java b/moquette-0.17/broker/src/test/java/io/moquette/broker/subscriptions/CTrieTest.java similarity index 87% rename from moquette-0.16/broker/src/test/java/io/moquette/broker/subscriptions/CTrieTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/broker/subscriptions/CTrieTest.java index 4b7764b1..88aa6394 100644 --- a/moquette-0.16/broker/src/test/java/io/moquette/broker/subscriptions/CTrieTest.java +++ b/moquette-0.17/broker/src/test/java/io/moquette/broker/subscriptions/CTrieTest.java @@ -24,6 +24,7 @@ import java.util.Set; import static io.moquette.broker.subscriptions.Topic.asTopic; +import java.util.List; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -94,7 +95,7 @@ public void testAddNewSubscriptionOnExistingNode() { //Verify final Optional matchedNode = sut.lookup(asTopic("/temp")); assertTrue(matchedNode.isPresent(), "Node on path /temp must be present"); - final Set subscriptions = matchedNode.get().subscriptions; + final List subscriptions = matchedNode.get().subscriptions; assertTrue(subscriptions.contains(newSubscription)); } @@ -109,7 +110,7 @@ public void testAddNewDeepNodes() { //Verify final Optional matchedNode = sut.lookup(asTopic("/italy/happiness")); assertTrue(matchedNode.isPresent(), "Node on path /italy/happiness must be present"); - final Set subscriptions = matchedNode.get().subscriptions; + final List subscriptions = matchedNode.get().subscriptions; assertTrue(subscriptions.contains(happinessSensor)); } @@ -176,7 +177,7 @@ public void givenTreeWithSomeNodeHierarchyWhenRemoveContainedSubscriptionThenNod sut.removeFromTree(asTopic("/temp/1"), "TempSensor1"); sut.removeFromTree(asTopic("/temp/1"), "TempSensor1"); - final Set matchingSubs = sut.recursiveMatch(asTopic("/temp/2")); + final List matchingSubs = sut.recursiveMatch(asTopic("/temp/2")); //Verify final Subscription expectedMatchingsub = new Subscription("TempSensor1", asTopic("/temp/2"), MqttQoS.AT_MOST_ONCE); @@ -191,8 +192,8 @@ public void givenTreeWithSomeNodeHierarchWhenRemoveContainedSubscriptionSmallerT //Exercise sut.removeFromTree(asTopic("/temp"), "TempSensor1"); - final Set matchingSubs1 = sut.recursiveMatch(asTopic("/temp/1")); - final Set matchingSubs2 = sut.recursiveMatch(asTopic("/temp/2")); + final List matchingSubs1 = sut.recursiveMatch(asTopic("/temp/1")); + final List matchingSubs2 = sut.recursiveMatch(asTopic("/temp/2")); //Verify // not clear to me, but I believe /temp unsubscribe should not unsub you from downstream /temp/1 or /temp/2 @@ -218,7 +219,7 @@ public void testMatchSubscriptionNoWildcards() { sut.addToTree(clientSubOnTopic("TempSensor1", "/temp")); //Exercise - final Set matchingSubs = sut.recursiveMatch(asTopic("/temp")); + final List matchingSubs = sut.recursiveMatch(asTopic("/temp")); //Verify final Subscription expectedMatchingsub = new Subscription("TempSensor1", asTopic("/temp"), MqttQoS.AT_MOST_ONCE); @@ -231,8 +232,8 @@ public void testRemovalInnerTopicOffRootSameClient() { sut.addToTree(clientSubOnTopic("TempSensor1", "temp/1")); //Exercise - final Set matchingSubs1 = sut.recursiveMatch(asTopic("temp")); - final Set matchingSubs2 = sut.recursiveMatch(asTopic("temp/1")); + final List matchingSubs1 = sut.recursiveMatch(asTopic("temp")); + final List matchingSubs2 = sut.recursiveMatch(asTopic("temp/1")); //Verify final Subscription expectedMatchingsub1 = new Subscription("TempSensor1", asTopic("temp"), MqttQoS.AT_MOST_ONCE); @@ -244,8 +245,8 @@ public void testRemovalInnerTopicOffRootSameClient() { sut.removeFromTree(asTopic("temp"), "TempSensor1"); //Exercise - final Set matchingSubs3 = sut.recursiveMatch(asTopic("temp")); - final Set matchingSubs4 = sut.recursiveMatch(asTopic("temp/1")); + final List matchingSubs3 = sut.recursiveMatch(asTopic("temp")); + final List matchingSubs4 = sut.recursiveMatch(asTopic("temp/1")); assertThat(matchingSubs3).doesNotContain(expectedMatchingsub1); assertThat(matchingSubs4).contains(expectedMatchingsub2); @@ -257,8 +258,8 @@ public void testRemovalInnerTopicOffRootDiffClient() { sut.addToTree(clientSubOnTopic("TempSensor2", "temp/1")); //Exercise - final Set matchingSubs1 = sut.recursiveMatch(asTopic("temp")); - final Set matchingSubs2 = sut.recursiveMatch(asTopic("temp/1")); + final List matchingSubs1 = sut.recursiveMatch(asTopic("temp")); + final List matchingSubs2 = sut.recursiveMatch(asTopic("temp/1")); //Verify final Subscription expectedMatchingsub1 = new Subscription("TempSensor1", asTopic("temp"), MqttQoS.AT_MOST_ONCE); @@ -270,8 +271,8 @@ public void testRemovalInnerTopicOffRootDiffClient() { sut.removeFromTree(asTopic("temp"), "TempSensor1"); //Exercise - final Set matchingSubs3 = sut.recursiveMatch(asTopic("temp")); - final Set matchingSubs4 = sut.recursiveMatch(asTopic("temp/1")); + final List matchingSubs3 = sut.recursiveMatch(asTopic("temp")); + final List matchingSubs4 = sut.recursiveMatch(asTopic("temp/1")); assertThat(matchingSubs3).doesNotContain(expectedMatchingsub1); assertThat(matchingSubs4).contains(expectedMatchingsub2); @@ -283,8 +284,8 @@ public void testRemovalOuterTopicOffRootDiffClient() { sut.addToTree(clientSubOnTopic("TempSensor2", "temp/1")); //Exercise - final Set matchingSubs1 = sut.recursiveMatch(asTopic("temp")); - final Set matchingSubs2 = sut.recursiveMatch(asTopic("temp/1")); + final List matchingSubs1 = sut.recursiveMatch(asTopic("temp")); + final List matchingSubs2 = sut.recursiveMatch(asTopic("temp/1")); //Verify final Subscription expectedMatchingsub1 = new Subscription("TempSensor1", asTopic("temp"), MqttQoS.AT_MOST_ONCE); @@ -296,8 +297,8 @@ public void testRemovalOuterTopicOffRootDiffClient() { sut.removeFromTree(asTopic("temp/1"), "TempSensor2"); //Exercise - final Set matchingSubs3 = sut.recursiveMatch(asTopic("temp")); - final Set matchingSubs4 = sut.recursiveMatch(asTopic("temp/1")); + final List matchingSubs3 = sut.recursiveMatch(asTopic("temp")); + final List matchingSubs4 = sut.recursiveMatch(asTopic("temp/1")); assertThat(matchingSubs3).contains(expectedMatchingsub1); assertThat(matchingSubs4).doesNotContain(expectedMatchingsub2); diff --git a/moquette-0.16/broker/src/test/java/io/moquette/broker/subscriptions/TopicTest.java b/moquette-0.17/broker/src/test/java/io/moquette/broker/subscriptions/TopicTest.java similarity index 100% rename from moquette-0.16/broker/src/test/java/io/moquette/broker/subscriptions/TopicTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/broker/subscriptions/TopicTest.java diff --git a/moquette-0.17/broker/src/test/java/io/moquette/broker/unsafequeues/DummySegmentAllocator.java b/moquette-0.17/broker/src/test/java/io/moquette/broker/unsafequeues/DummySegmentAllocator.java new file mode 100644 index 00000000..311076c1 --- /dev/null +++ b/moquette-0.17/broker/src/test/java/io/moquette/broker/unsafequeues/DummySegmentAllocator.java @@ -0,0 +1,55 @@ +package io.moquette.broker.unsafequeues; + +import io.moquette.BrokerConstants; +import java.io.IOException; +import java.nio.MappedByteBuffer; +import java.util.Properties; + +public class DummySegmentAllocator implements SegmentAllocator { + + @Override + public Segment nextFreeSegment() { + final MappedByteBuffer pageBuffer = createFreshPageTmpTile(); + final SegmentPointer begin = new SegmentPointer(0, 0); + final SegmentPointer end = new SegmentPointer(0, BrokerConstants.DEFAULT_SEGMENTED_QUEUE_SEGMENT_SIZE); + return new Segment(pageBuffer, begin, end); + } + + @Override + public Segment reopenSegment(int pageId, int beginOffset) throws QueueException { + final MappedByteBuffer pageBuffer = createFreshPageTmpTile(); + final SegmentPointer begin = new SegmentPointer(pageId, beginOffset); + final SegmentPointer end = new SegmentPointer(pageId, beginOffset + BrokerConstants.DEFAULT_SEGMENTED_QUEUE_SEGMENT_SIZE); + return new Segment(pageBuffer, begin, end); + } + + private MappedByteBuffer createFreshPageTmpTile() { + final MappedByteBuffer pageBuffer; + try { + pageBuffer = Utils.createPageFile(); + } catch (IOException ex) { + // used only in tests, so it's safe to fail the test with an untyped exception + throw new RuntimeException(ex); + } + return pageBuffer; + } + + @Override + public void close() throws QueueException { + // TODO, maybe + } + + @Override + public void dumpState(Properties checkpoint) { + } + + @Override + public int getPageSize() { + return BrokerConstants.DEFAULT_SEGMENTED_QUEUE_PAGE_SIZE; + } + + @Override + public int getSegmentSize() { + return BrokerConstants.DEFAULT_SEGMENTED_QUEUE_SEGMENT_SIZE; + } +} diff --git a/moquette-0.17/broker/src/test/java/io/moquette/broker/unsafequeues/QueuePoolTest.java b/moquette-0.17/broker/src/test/java/io/moquette/broker/unsafequeues/QueuePoolTest.java new file mode 100644 index 00000000..86f6ff25 --- /dev/null +++ b/moquette-0.17/broker/src/test/java/io/moquette/broker/unsafequeues/QueuePoolTest.java @@ -0,0 +1,365 @@ +package io.moquette.broker.unsafequeues; + +import io.moquette.BrokerConstants; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; +import org.junit.jupiter.api.io.TempDir; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.Properties; +import java.util.TreeSet; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import static io.moquette.broker.unsafequeues.Queue.LENGTH_HEADER_SIZE; +import static io.moquette.broker.unsafequeues.QueueTest.generatePayload; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@DisabledOnOs(OS.WINDOWS) +class QueuePoolTest { + + private static final Logger LOG = LoggerFactory.getLogger(QueuePoolTest.class); + + private static final int PAGE_SIZE = BrokerConstants.DEFAULT_SEGMENTED_QUEUE_PAGE_SIZE; + private static final int SEGMENT_SIZE = BrokerConstants.DEFAULT_SEGMENTED_QUEUE_SEGMENT_SIZE; + + @TempDir + Path tempQueueFolder; + + @Test + public void checkpointFileContainsCorrectReferences() throws QueueException, IOException { + final QueuePool queuePool = QueuePool.loadQueues(tempQueueFolder, PAGE_SIZE, SEGMENT_SIZE); + final Queue queue = queuePool.getOrCreate("test"); + queue.enqueue((ByteBuffer)ByteBuffer.wrap("AAAA".getBytes(StandardCharsets.UTF_8))); + queue.force(); + queuePool.close(); + + // verify + final Path checkpointPath = tempQueueFolder.resolve("checkpoint.properties"); + final File checkpointFile = checkpointPath.toFile(); + assertTrue(checkpointFile.exists(), "Checkpoint file must be created"); + + final Properties checkpoint = loadCheckpoint(checkpointPath); + final int lastPage = Integer.parseInt(checkpoint.get("segments.last_page").toString()); + assertEquals(0, lastPage); + final int lastSegment = Integer.parseInt(checkpoint.get("segments.last_segment").toString()); + assertEquals(1, lastSegment); + + assertEquals("test", checkpoint.get("queues.0.name"), "Queue name must match"); + } + + private Properties loadCheckpoint(Path checkpointPath) throws IOException { + final FileReader fileReader; + fileReader = new FileReader(checkpointPath.toFile()); + final Properties checkpointProps = new Properties(); + checkpointProps.load(fileReader); + return checkpointProps; + } + + @Test + public void reloadQueuePoolAndCheckRestartFromWhereItLeft() throws QueueException, IOException { + QueuePool queuePool = QueuePool.loadQueues(tempQueueFolder, PAGE_SIZE, SEGMENT_SIZE); + Queue queue = queuePool.getOrCreate("test"); + queue.enqueue(ByteBuffer.wrap("AAAA".getBytes(StandardCharsets.UTF_8))); + queue.force(); + queuePool.close(); + + // reload + queuePool = QueuePool.loadQueues(tempQueueFolder, PAGE_SIZE, SEGMENT_SIZE); + queue = queuePool.getOrCreate("test"); + queue.enqueue(ByteBuffer.wrap("BBBB".getBytes(StandardCharsets.UTF_8))); + queue.force(); + queuePool.close(); + + // verify + final Path checkpointPath = tempQueueFolder.resolve("checkpoint.properties"); + final File checkpointFile = checkpointPath.toFile(); + assertTrue(checkpointFile.exists(), "Checkpoint file must be created"); + + final Properties checkpoint = loadCheckpoint(checkpointPath); + final int lastPage = Integer.parseInt(checkpoint.get("segments.last_page").toString()); + assertEquals(0, lastPage); + final int lastSegment = Integer.parseInt(checkpoint.get("segments.last_segment").toString()); + assertEquals(1, lastSegment); + + assertEquals("test", checkpoint.get("queues.0.name"), "Queue name must match"); + assertEquals("15", checkpoint.get("queues.0.head_offset"), "Queue head must be 16 bytes over the start"); + } + + private TreeSet asTreeSet(QueuePool.SegmentRef... segments) { + final TreeSet usedSegments = new TreeSet<>(); + usedSegments.addAll(Arrays.asList(segments)); + return usedSegments; + } + + @Test + public void checkRecreateHolesAtTheStartOfThePage() throws QueueException { + final QueuePool queuePool = QueuePool.loadQueues(tempQueueFolder, PAGE_SIZE, SEGMENT_SIZE); + final QueuePool.SegmentRef middleSegment = new QueuePool.SegmentRef(0, SEGMENT_SIZE); + + // Exercise + final List holes = queuePool.recreateSegmentHoles(asTreeSet(middleSegment)); + + // Verify + assertEquals(1, holes.size(), "Only the preceding segment should be created"); + QueuePool.SegmentRef singleHole = holes.get(0); + assertEquals(0, singleHole.pageId); + assertEquals(0, singleHole.offset); + } + + @Test + public void checkRecreateHolesAtTheStartOfThePageWith2OccupiedContiguousSegments() throws QueueException { + final QueuePool queuePool = QueuePool.loadQueues(tempQueueFolder, PAGE_SIZE, SEGMENT_SIZE); + final QueuePool.SegmentRef firstSegment = new QueuePool.SegmentRef(0, SEGMENT_SIZE); + final QueuePool.SegmentRef secondSegment = new QueuePool.SegmentRef(0, 2 * SEGMENT_SIZE); + + // Exercise + final List holes = queuePool.recreateSegmentHoles(asTreeSet(firstSegment, secondSegment)); + + // Verify + assertEquals(1, holes.size(), "Only the preceding segment should be created"); + QueuePool.SegmentRef singleHole = holes.get(0); + assertEquals(0, singleHole.pageId); + assertEquals(0, singleHole.offset); + } + + @Test + public void checkRecreateHolesBeforeSecondPage() throws QueueException { + final QueuePool queuePool = QueuePool.loadQueues(tempQueueFolder, PAGE_SIZE, SEGMENT_SIZE); + final QueuePool.SegmentRef middleSegment = new QueuePool.SegmentRef(1, SEGMENT_SIZE); + + // Exercise + final List holes = queuePool.recreateSegmentHoles(asTreeSet(middleSegment)); + + // Verify + final int expectedHolesCount = (int) (PAGE_SIZE / SEGMENT_SIZE) + 1; + assertEquals(expectedHolesCount, holes.size(), "The previous empty page is full of holes"); + for (int i = 0; i < expectedHolesCount - 1; i++) { + final QueuePool.SegmentRef hole = holes.get(i); + assertEquals(0, hole.pageId); + assertEquals(i * SEGMENT_SIZE, hole.offset); + } + QueuePool.SegmentRef singleHole = holes.get(expectedHolesCount - 1); + assertEquals(1, singleHole.pageId); + assertEquals(0, singleHole.offset); + } + + @Test + public void checkRecreateHolesBetweenUsedSegmentsOnSamePage() throws QueueException { + final QueuePool queuePool = QueuePool.loadQueues(tempQueueFolder, PAGE_SIZE, SEGMENT_SIZE); + final QueuePool.SegmentRef initialSegment = new QueuePool.SegmentRef(0, 0); + final QueuePool.SegmentRef middleSegment = new QueuePool.SegmentRef(0, 3 * SEGMENT_SIZE); + + // Exercise + final List holes = queuePool.recreateSegmentHoles(asTreeSet(initialSegment, middleSegment)); + + // Verify + assertEquals(2, holes.size()); + + // first hole + assertEquals(0, holes.get(0).pageId); + assertEquals(SEGMENT_SIZE, holes.get(0).offset); + // second hole + assertEquals(0, holes.get(1).pageId); + assertEquals(2 * SEGMENT_SIZE, holes.get(1).offset); + } + + @Test + public void checkRecreateHolesSpanningMultiplePages() throws QueueException { + final QueuePool queuePool = QueuePool.loadQueues(tempQueueFolder, PAGE_SIZE, SEGMENT_SIZE); + final QueuePool.SegmentRef initialSegment = new QueuePool.SegmentRef(0, 0); + final QueuePool.SegmentRef middleSegment = new QueuePool.SegmentRef(2, 3 * SEGMENT_SIZE); + + // Exercise + final List holes = queuePool.recreateSegmentHoles(asTreeSet(initialSegment, middleSegment)); + + // Verify + final int holesInEmptyPage = (PAGE_SIZE / SEGMENT_SIZE); + final int holesInFirstPage = holesInEmptyPage - 1; + final int holesInLastPage = 3; + final int expectedHolesCount = holesInFirstPage + holesInEmptyPage + holesInLastPage; + assertEquals(expectedHolesCount, holes.size()); + + // first page hole + int i = 0; + int expectedOffset = SEGMENT_SIZE; + for (; i < holesInFirstPage; i++) { + final QueuePool.SegmentRef hole = holes.get(i); + assertEquals(0, hole.pageId); + assertEquals(expectedOffset, hole.offset); + expectedOffset += SEGMENT_SIZE; + } + + // central empty pages + expectedOffset = 0; + for (; i < holesInFirstPage + holesInEmptyPage; i++) { + final QueuePool.SegmentRef hole = holes.get(i); + assertEquals(1, hole.pageId); + assertEquals(expectedOffset, hole.offset); + expectedOffset += SEGMENT_SIZE; + } + + // tail page hole + expectedOffset = 0; + for (; i < expectedHolesCount; i++) { + final QueuePool.SegmentRef hole = holes.get(i); + assertEquals(2, hole.pageId); + assertEquals(expectedOffset, hole.offset); + expectedOffset += SEGMENT_SIZE; + } + } + + @Test + public void checkRecreateHolesWhenSegmentAreAdjacentAndSpanningMultiplePages() throws QueueException { + final QueuePool queuePool = QueuePool.loadQueues(tempQueueFolder, PAGE_SIZE, SEGMENT_SIZE); + final QueuePool.SegmentRef initialSegment = new QueuePool.SegmentRef(0, PAGE_SIZE - SEGMENT_SIZE); + final QueuePool.SegmentRef adjacentSegment = new QueuePool.SegmentRef(1, 0); + + // Exercise + final List holes = queuePool.recreateSegmentHoles(asTreeSet(initialSegment, adjacentSegment)); + + // Verify + assertEquals((PAGE_SIZE - SEGMENT_SIZE) / SEGMENT_SIZE, holes.size()); + } + + @Test + @Disabled + public void verifySingleReaderSingleWriterOnSingleQueuePool_with_955157_size_packet() throws QueueException, ExecutionException, InterruptedException, TimeoutException { + templateSingleReaderSingleWriterOnSingleQueuePool(955157); + } + + @Test + @Disabled + public void verifySingleReaderSingleWriterOnSingleQueuePool_with_random_size_packet() throws QueueException, ExecutionException, InterruptedException, TimeoutException, NoSuchAlgorithmException { + final int payloadSize = SecureRandom.getInstanceStrong().nextInt(SEGMENT_SIZE / 2 - LENGTH_HEADER_SIZE); + templateSingleReaderSingleWriterOnSingleQueuePool(payloadSize); + + } + + private void templateSingleReaderSingleWriterOnSingleQueuePool(int payloadSize) throws QueueException, InterruptedException, ExecutionException, TimeoutException { + final QueuePool queuePool = QueuePool.loadQueues(tempQueueFolder, PAGE_SIZE, SEGMENT_SIZE); + Queue queue = queuePool.getOrCreate("single_writer_single_reader"); + ExecutorService pool = Executors.newCachedThreadPool(); + int messagesToSend = 1000; + + LOG.info("Payload size: " + payloadSize); + ByteBuffer payload = ByteBuffer.wrap(generatePayload(payloadSize, (byte) 'a')); + + Future senderFuture = pool.submit(createMessageSender(queue, payload, messagesToSend)); + Future readerFuture = pool.submit(createMessageReader(queue, messagesToSend)); + + senderFuture.get(60, TimeUnit.SECONDS); + final int readBytes = readerFuture.get(60, TimeUnit.SECONDS); + assertEquals(messagesToSend * payloadSize, readBytes); + } + + private Runnable createMessageSender(Queue queue, ByteBuffer payload, int count) { + return () -> { + int payloadSize = payload.remaining(); + int sentBytes = 0; + for (int i = 0; i < count; i++) { + try { + ByteBuffer duplicate = payload.duplicate(); + sentBytes += duplicate.remaining(); + queue.enqueue(duplicate); + } catch (QueueException e) { + throw new RuntimeException(e); + } + } + LOG.debug("Finish to send data, " + count + " messages of size: " + payloadSize + " for a total of: " + sentBytes); + }; + } + + private Callable createMessageReader(Queue queue, int expectedMessages) { + return () -> { + try { + int readBytes = 0; + int receivedMessages = 0; + do { + LOG.debug("receivedMessages {} ({} expected)", receivedMessages, expectedMessages); + Optional readPayload = queue.dequeue(); + if (readPayload.isPresent()) { + readBytes += readPayload.get().remaining(); + receivedMessages++; + } + } while (receivedMessages < expectedMessages); + LOG.debug("Received messages: " + receivedMessages); + return readBytes; + } catch (QueueException e) { + throw new RuntimeException(e); + } + }; + } + + int result = 0; + + @Test + @Disabled + public void testMultipleWritersSingleReader() throws QueueException, NoSuchAlgorithmException, ExecutionException, InterruptedException, TimeoutException { + final int payloadSize = 152433/*SecureRandom.getInstanceStrong().nextInt(Segment.SIZE / 2 - LENGTH_HEADER_SIZE)*/; + LOG.info("Payload size: " + payloadSize); + final QueuePool queuePool = QueuePool.loadQueues(/*tempQueueFolder*/Paths.get("/tmp/test_dir/"), PAGE_SIZE, SEGMENT_SIZE); + Queue queue = queuePool.getOrCreate("multiple_writers_single_reader"); + ExecutorService pool = Executors.newCachedThreadPool(); + int messagesToSend = 1000; + + ByteBuffer payloadA = ByteBuffer.wrap(generatePayload(payloadSize, (byte) 'a')); + ByteBuffer payloadB = ByteBuffer.wrap(generatePayload(payloadSize, (byte) 'b')); + + final Thread threadWriterA = new Thread(createMessageSender(queue, payloadA, messagesToSend / 2)); + threadWriterA.setName("Writer-A"); + + final Thread threadWriterB = new Thread(createMessageSender(queue, payloadB, messagesToSend / 2)); + threadWriterB.setName("Writer-B"); + + final Thread threadReader = new Thread(new Runnable() { + @Override + public void run() { + try { + result = createMessageReader(queue, messagesToSend).call(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + }); + threadReader.setName("Reader"); + + threadWriterA.start(); + threadWriterB.start(); + threadReader.start(); + threadWriterA.join(60_000); + threadWriterB.join(60_000); + threadReader.join(60_000); + + +// Future senderFutureA = pool.submit(createMessageSender(queue, payloadA, messagesToSend / 2)); +// Future senderFutureB = pool.submit(createMessageSender(queue, payloadB, messagesToSend / 2)); +// Future readerFuture = pool.submit(createMessageReader(queue, messagesToSend)); + +// senderFutureA.get(60, TimeUnit.SECONDS); +// senderFutureB.get(60, TimeUnit.SECONDS); +// final int readBytes = readerFuture.get(60, TimeUnit.SECONDS); + assertEquals(messagesToSend * payloadSize, result); + } +} diff --git a/moquette-0.17/broker/src/test/java/io/moquette/broker/unsafequeues/QueueTest.java b/moquette-0.17/broker/src/test/java/io/moquette/broker/unsafequeues/QueueTest.java new file mode 100644 index 00000000..2934910f --- /dev/null +++ b/moquette-0.17/broker/src/test/java/io/moquette/broker/unsafequeues/QueueTest.java @@ -0,0 +1,578 @@ +package io.moquette.broker.unsafequeues; + +import io.moquette.BrokerConstants; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Optional; +import java.util.Properties; +import java.util.Random; +import java.util.function.Consumer; + +import static io.moquette.broker.unsafequeues.Queue.LENGTH_HEADER_SIZE; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +@DisabledOnOs(OS.WINDOWS) +class QueueTest { + + private static final int PAGE_SIZE = BrokerConstants.DEFAULT_SEGMENTED_QUEUE_PAGE_SIZE; + private static final int SEGMENT_SIZE = BrokerConstants.DEFAULT_SEGMENTED_QUEUE_SEGMENT_SIZE; + + @TempDir + Path tempQueueFolder; + + private void assertContainsOnly(char expectedChar, byte[] verify) { + for (int i = 0; i < verify.length; i++) { + if (verify[i] != expectedChar) { + fail(String.format("Expected %c but found %c in %c%c%c", expectedChar, verify[i], verify[i-1], verify[i], verify[i+1])); + } + } + } + + private void assertContainsOnly(char expectedChar, ByteBuffer verify) { + int pos = verify.position(); + while (verify.hasRemaining()) { + final byte readChar = verify.get(); + pos++; + if (readChar != expectedChar) { + fail(String.format("Expected %c but found %c at position %d", expectedChar, readChar, pos)); + } + } + } + + private void assertContainsOnly(char expectedChar, ByteBuffer verify, int expectedSize) { + int pos = verify.position(); + int countChars = 0; + while (verify.hasRemaining()) { + final byte readChar = verify.get(); + pos++; + countChars++; + if (readChar != expectedChar) { + fail(String.format("Expected %c but found %c at position %d", expectedChar, readChar, pos)); + } + } + assertEquals(expectedSize, countChars); + } + + + static byte[] generatePayload(int numBytes) { + return generatePayload(numBytes, (byte) 'A'); + } + + static byte[] generatePayload(int numBytes, byte c) { + final byte[] payload = new byte[numBytes]; + for (int i = 0; i < numBytes; i++) { + payload[i] = c; + } + return payload; + } + + @Test + public void testSizesFitTogether() { + Assertions.assertThrows(IllegalArgumentException.class, () -> {new PagedFilesAllocator(null, 1000, 499, 0, 0);}); + } + + @Test + public void basicNoBlockEnqueue() throws QueueException, IOException { + final MappedByteBuffer pageBuffer = Utils.createPageFile(); + + final Segment head = new Segment(pageBuffer, new SegmentPointer(0, 0), new SegmentPointer(0, 1024)); + final VirtualPointer currentHead = VirtualPointer.buildUntouched(); + final Queue queue = new Queue("test", head, currentHead, head, currentHead, new DummySegmentAllocator(), (name, segment) -> { + // NOOP + }, null); + + // generate byte array to insert. + ByteBuffer payload = randomPayload(128); + + queue.enqueue(payload); + } + + private ByteBuffer randomPayload(int dataSize) { + byte[] payload = new byte[dataSize]; + new Random().nextBytes(payload); + + return (ByteBuffer) ByteBuffer.wrap(payload); + } + + @Test + public void newlyQueueIsEmpty() throws QueueException { + final QueuePool queuePool = QueuePool.loadQueues(tempQueueFolder, PAGE_SIZE, SEGMENT_SIZE); + final Queue queue = queuePool.getOrCreate("test"); + + assertTrue(queue.isEmpty(), "Freshly created queue must be empty"); + } + + @Test + public void consumedQueueIsEmpty() throws QueueException { + final QueuePool queuePool = QueuePool.loadQueues(tempQueueFolder, PAGE_SIZE, SEGMENT_SIZE); + final Queue queue = queuePool.getOrCreate("test"); + queue.enqueue(ByteBuffer.wrap("AAAA".getBytes(StandardCharsets.UTF_8))); + Optional data = queue.dequeue(); + assertTrue(data.isPresent(), "Some payload is retrieved"); + + assertEquals(4, data.get().remaining(), "Payload contains what's expected"); + + assertTrue(queue.isEmpty(), "Queue must be empty after consuming it"); + } + + @Test + public void insertSomeDataIntoNewQueue() throws QueueException, IOException { + final QueuePool queuePool = QueuePool.loadQueues(tempQueueFolder, PAGE_SIZE, SEGMENT_SIZE); + final Queue queue = queuePool.getOrCreate("test"); + queue.enqueue(ByteBuffer.wrap("AAAA".getBytes(StandardCharsets.UTF_8))); + + // verify + final HashSet fileset = new HashSet<>(Arrays.asList(tempQueueFolder.toFile().list())); + assertEquals(2, fileset.size()); + assertTrue(fileset.contains("checkpoint.properties"), "Checkpoint file must be created"); + assertTrue(fileset.contains("0.page"), "One page file must be created"); + + final Path pageFile = tempQueueFolder.resolve("0.page"); + verifyFile(pageFile, 9, rawContent -> { + assertEquals(4, rawContent.getInt(), "First 4 bytes contains the length"); + assertEquals('A', rawContent.get()); + assertEquals('A', rawContent.get()); + assertEquals('A', rawContent.get()); + assertEquals('A', rawContent.get()); + assertEquals(0, rawContent.get()); + }); + } + + private void verifyFile(Path file, int bytesToRead, Consumer verifier) throws IOException { + final FileChannel fc = FileChannel.open(file, StandardOpenOption.READ); + final ByteBuffer rawContent = ByteBuffer.allocate(bytesToRead); + final int read = fc.read(rawContent); + assertEquals(bytesToRead, read); + rawContent.flip(); + + verifier.accept(rawContent); + } + + @Test + public void insertDataTriggerCreationOfNewPageFile() throws QueueException { + final QueuePool queuePool = QueuePool.loadQueues(tempQueueFolder, PAGE_SIZE, SEGMENT_SIZE); + final Queue queue = queuePool.getOrCreate("test"); + + // one page is 64 MB so the loop count to fill it is 64 * 1024 + // 4 bytes are left for length so that each time are inserted 1024 bytes, 4 header and 1020 payload + ByteBuffer payload = ByteBuffer.wrap(generatePayload(1024 - LENGTH_HEADER_SIZE)); + for (int i = 0; i < 64; i++) { + writeMessages(queue, payload, 1024); + } + + // check the 2 files are created + HashSet fileset = new HashSet<>(Arrays.asList(tempQueueFolder.toFile().list())); + assertEquals(2, fileset.size()); + assertTrue(fileset.contains("checkpoint.properties"), "Checkpoint file must be created"); + assertTrue(fileset.contains("0.page"), + "One page file must be created"); + + // Exercise + // some data to force create a new page + final ByteBuffer crossingPayload = ByteBuffer.wrap(generatePayload(10, (byte) 'B')); + queue.enqueue(crossingPayload); + + // Verify + fileset = new HashSet<>(Arrays.asList(tempQueueFolder.toFile().list())); + assertEquals(3, fileset.size()); + assertTrue(fileset.contains("checkpoint.properties"), "Checkpoint file must be created"); + assertTrue(fileset.contains("0.page"), "First page file must be created"); + assertTrue(fileset.contains("1.page"), "Second page file must be created"); + } + + @Test + public void insertWithAnHeaderThatCrossSegments() throws QueueException, IOException { + final QueuePool queuePool = QueuePool.loadQueues(tempQueueFolder, PAGE_SIZE, SEGMENT_SIZE); + final Queue queue = queuePool.getOrCreate("test"); + + // fill the segment, inserting last message crossing the boundary + ByteBuffer payload = ByteBuffer.wrap(generatePayload(1024 - LENGTH_HEADER_SIZE)); + writeMessages(queue, payload, (4 * 1024) - 1); + // at the end we have 1024 bytes free, so fill only 1022 bytes of that + payload = ByteBuffer.wrap(generatePayload(1022 - LENGTH_HEADER_SIZE)); + payload.rewind(); + queue.enqueue(payload); + + // Exercise + ByteBuffer crossingPayload = ByteBuffer.wrap(generatePayload(1024 - LENGTH_HEADER_SIZE, (byte) 'B')); + queue.enqueue(crossingPayload); + + // Verify + final MappedByteBuffer page = Utils.openPageFile(tempQueueFolder.resolve("0.page"), PAGE_SIZE); + final int beforeLastMessagePayload = 4 * 1024 * 1024 + 2; + final ByteBuffer crossingSegment = (ByteBuffer) page.position(beforeLastMessagePayload); +// final int msgLength = crossingSegment.getInt(); + +// assertEquals(1028 - LENGTH_HEADER_SIZE, msgLength); +// byte[] probe = new byte[msgLength]; + byte[] probe = new byte[1020]; + crossingSegment.get(probe); + assertContainsOnly('B', probe); + } + + @Test + public void insertDataCrossingSegmentBoundary() throws QueueException, IOException { + final QueuePool queuePool = QueuePool.loadQueues(tempQueueFolder, PAGE_SIZE, SEGMENT_SIZE); + final Queue queue = queuePool.getOrCreate("test"); + + // one segment is 4MB 4 * 1024 * payload + // so send (4 * 1024) - 1 payloads of 1024 and then send + // a payload of 1028 (4 bytes over remaining space) + + // 4 bytes are left for length so that each time are inserted 1024 bytes, 4 header and 1020 payload + ByteBuffer payload = ByteBuffer.wrap(generatePayload(1024 - LENGTH_HEADER_SIZE)); + writeMessages(queue, payload, (4 * 1024) - 1); + + // Experiment + ByteBuffer crossingPayload = ByteBuffer.wrap(generatePayload(1028 - LENGTH_HEADER_SIZE, (byte) 'B')); + queue.enqueue(crossingPayload); + queue.force(); + queuePool.close(); + + // Verify + final MappedByteBuffer page = Utils.openPageFile(tempQueueFolder.resolve("0.page"), PAGE_SIZE); + final int beforeLastMessage = ((4 * 1024) - 1) * 1024; + final ByteBuffer crossingSegment = (ByteBuffer) page.position(beforeLastMessage); + final int msgLength = crossingSegment.getInt(); + + assertEquals(1028 - LENGTH_HEADER_SIZE, msgLength); + byte[] probe = new byte[msgLength]; + crossingSegment.get(probe); + assertContainsOnly('B', probe); + } + + @Test + public void insertDataBiggerThanASegment() throws QueueException, IOException { + final QueuePool queuePool = QueuePool.loadQueues(tempQueueFolder, PAGE_SIZE, SEGMENT_SIZE); + final Queue queue = queuePool.getOrCreate("test"); + + // one segment is 4MB 4 * 1024 * payload + // so send (4 * 1024) - 1 payloads of 1024 and then send + // a payload of 1028 (4 bytes over remaining space) + // 4 bytes are left for length so that each time are inserted 1024 bytes, 4 header and 1020 payload + ByteBuffer payload = ByteBuffer.wrap(generatePayload(1024 - LENGTH_HEADER_SIZE)); + writeMessages(queue, payload, (4 * 1024) - 1); + + // Experiment + // 1024 + 4 * 1024 * 1024 + 16 bytes + int moreThanOneSegment = 1024 + 4 * 1024 * 1024 + 16; + ByteBuffer crossingMultipleSegmentPayload = ByteBuffer.wrap(generatePayload(moreThanOneSegment, (byte) 'B')); + queue.enqueue(crossingMultipleSegmentPayload); + queue.force(); + queuePool.close(); + + // Verify + final MappedByteBuffer page = Utils.openPageFile(tempQueueFolder.resolve("0.page"), PAGE_SIZE); + final int beforeLastMessage = ((4 * 1024) - 1) * 1024; + final ByteBuffer crossingSegment = (ByteBuffer) page.position(beforeLastMessage); + final int msgLength = crossingSegment.getInt(); + + assertEquals(moreThanOneSegment, msgLength); + byte[] probe = new byte[msgLength]; + crossingSegment.get(probe); + assertContainsOnly('B', probe); + } + + @Test + public void readFromEmptyQueue() throws QueueException { + final QueuePool queuePool = QueuePool.loadQueues(tempQueueFolder, PAGE_SIZE, SEGMENT_SIZE); + final Queue queue = queuePool.getOrCreate("test"); + + assertFalse(queue.dequeue().isPresent(), "Pulling from empty queue MUST return null value"); + } + + @Test + public void readInSameSegment() throws QueueException { + final QueuePool queuePool = QueuePool.loadQueues(tempQueueFolder, PAGE_SIZE, SEGMENT_SIZE); + final Queue queue = queuePool.getOrCreate("test"); + + final ByteBuffer message = ByteBuffer.wrap("Hello World!".getBytes(StandardCharsets.UTF_8)); + queue.enqueue(message); + + //Exercise + final ByteBuffer result = queue.dequeue().get(); + final String readMessage = Utils.bufferToString(result); + assertEquals("Hello World!", readMessage, "Read the same message tha was enqueued"); + } + + @Test + public void readCrossingSegment() throws QueueException { + final QueuePool queuePool = QueuePool.loadQueues(tempQueueFolder, PAGE_SIZE, SEGMENT_SIZE); + final Queue queue = queuePool.getOrCreate("test"); + + // fill the segment, inserting last message crossing the boundary + ByteBuffer payload = ByteBuffer.wrap(generatePayload(1024 - LENGTH_HEADER_SIZE)); + for (int i = 0; i < (4 * 1024) - 1; i++) { + payload.rewind(); + queue.enqueue(payload); + queue.dequeue(); + } + + ByteBuffer crossingPayload = ByteBuffer.wrap(generatePayload(1028 - LENGTH_HEADER_SIZE, (byte) 'B')); + queue.enqueue(crossingPayload); + + //Exercise + final ByteBuffer message = queue.dequeue().get(); + assertEquals(1028 - LENGTH_HEADER_SIZE, message.remaining(), "There must be 1024 'B' letters"); + assertContainsOnly('B', message); + } + + @Test + public void readWithHeaderCrossingSegments() throws QueueException { + final QueuePool queuePool = QueuePool.loadQueues(tempQueueFolder, PAGE_SIZE, SEGMENT_SIZE); + final Queue queue = queuePool.getOrCreate("test"); + + // fill the segment, inserting last message crossing the boundary + ByteBuffer payload = ByteBuffer.wrap(generatePayload(1024 - LENGTH_HEADER_SIZE)); + for (int i = 0; i < (4 * 1024) - 1; i++) { + payload.rewind(); + queue.enqueue(payload); + queue.dequeue(); + } + // at the end we have 1024 bytes free, so fill only 1022 bytes of that + payload = ByteBuffer.wrap(generatePayload(1022 - LENGTH_HEADER_SIZE)); + payload.rewind(); + queue.enqueue(payload); + queue.dequeue(); + + // write a payload's header with 2 bytes in previous and 2 in next segment + ByteBuffer crossingPayload = ByteBuffer.wrap(generatePayload(1024 - LENGTH_HEADER_SIZE, (byte) 'B')); + queue.enqueue(crossingPayload); + + //Exercise + final ByteBuffer message = queue.dequeue().get(); + assertEquals(1024 - LENGTH_HEADER_SIZE, message.remaining(), "There must be 1020 'B' letters"); + assertContainsOnly('B', message); + } + + @Test + public void readCrossingPages() throws QueueException, IOException { + final QueuePool queuePool = QueuePool.loadQueues(tempQueueFolder, PAGE_SIZE, SEGMENT_SIZE); + final Queue queue = queuePool.getOrCreate("test"); + + // fill all segments less one in a page + ByteBuffer payload = ByteBuffer.wrap(generatePayload(1024 - LENGTH_HEADER_SIZE)); + int messageSize = payload.remaining() + LENGTH_HEADER_SIZE; + final int loopToFill = PAGE_SIZE / messageSize; + writeMessages(queue, payload, loopToFill - 1); + + assertEquals(1, countPages(tempQueueFolder)); + assertEquals(PAGE_SIZE - messageSize, queue.currentHead().logicalOffset() + 1, + "head must be one message size (1024) from the end of the segment"); + + // Exercise + payload = ByteBuffer.wrap(generatePayload(2048 - LENGTH_HEADER_SIZE, (byte) 'B')); + queue.enqueue(payload); + + // Verify + assertEquals(2, countPages(tempQueueFolder)); + } + + private long countPages(Path tempQueueFolder) throws IOException { + return Files.list(tempQueueFolder).filter(p -> p.toString().endsWith(".page")).count(); + } + + @Test + public void interleavedQueueSegments() throws QueueException, IOException { + // first segment queue A, second segment queue B and so on, in stripped fashion. + // writes and read pass single page borders, checking everything is fine + final QueuePool queuePool = QueuePool.loadQueues(tempQueueFolder, PAGE_SIZE, SEGMENT_SIZE); + final Queue queueA = queuePool.getOrCreate("testA"); + final Queue queueB = queuePool.getOrCreate("testB"); + + ByteBuffer payloadQueueA = ByteBuffer.wrap(generatePayload(1024 - LENGTH_HEADER_SIZE, (byte) 'A')); + ByteBuffer payloadQueueB = ByteBuffer.wrap(generatePayload(1024 - LENGTH_HEADER_SIZE, (byte) 'B')); + int messageSize = payloadQueueA.remaining() + LENGTH_HEADER_SIZE; + + // Exercise + final int numPages = 2; + final int segmentsToFill = numPages * PAGE_SIZE / SEGMENT_SIZE; + final int messagesInSegment = SEGMENT_SIZE / messageSize; + for (int i = 0; i < segmentsToFill; i++) { + if (isEven(i)) { + writeMessages(queueA, payloadQueueA, messagesInSegment); + } else { + writeMessages(queueB, payloadQueueB, messagesInSegment); + } + } + + // Verify + assertEquals(numPages, countPages(tempQueueFolder)); + final int numMessagesInQueue = PAGE_SIZE / messageSize; + verifyReadingFromQueue(numMessagesInQueue, queueA, 'A', 1024 - LENGTH_HEADER_SIZE); + verifyReadingFromQueue(numMessagesInQueue, queueB, 'B', 1024 - LENGTH_HEADER_SIZE); + } + + private void verifyReadingFromQueue(int numMessagesInQueue, Queue queue, char ch, int expectedPayloadSize) throws QueueException { + for (int i = 0; i < numMessagesInQueue; i++) { + final ByteBuffer payload = queue.dequeue().get(); + assertContainsOnly(ch, payload, expectedPayloadSize); + } + } + + private void writeMessages(Queue targetQueue, ByteBuffer payload, int messagesToWrite) throws QueueException { + for (int i = 0; i < messagesToWrite; i++) { + payload.rewind(); + targetQueue.enqueue(payload); + } + } + + private boolean isEven(int i) { + return i % 2 == 0; + } + + @Test + public void physicalBackwardSegment() throws IOException, QueueException { + // Artificially create a queue composed of segment(2) and segment(1), inverted in order, verify + // if read and write properly.pageBuffer. + final Path pageFile = this.tempQueueFolder.resolve("0.page"); + final OpenOption[] openOptions = {StandardOpenOption.CREATE_NEW, StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING}; + FileChannel fileChannel = FileChannel.open(pageFile, openOptions); + final MappedByteBuffer pageBuffer = fileChannel.map(FileChannel.MapMode.READ_WRITE, 0, PAGE_SIZE); + // segment with only one message of B letters + pageBuffer.putInt(SEGMENT_SIZE - LENGTH_HEADER_SIZE); + pageBuffer.put(generatePayload(SEGMENT_SIZE - LENGTH_HEADER_SIZE, (byte) 'B')); + // segment with only one message of A letters + pageBuffer.putInt(SEGMENT_SIZE - LENGTH_HEADER_SIZE); + pageBuffer.put(generatePayload(SEGMENT_SIZE - LENGTH_HEADER_SIZE, (byte) 'A')); + pageBuffer.force(); + fileChannel.close(); + + // create the checkpoint file with the inverted segments + Properties checkpoint = new Properties(); +// checkpoint.properties + checkpoint.put("queues.0.name", "test_inverted"); + checkpoint.put("queues.0.segments", "(0, 0), (0, " + SEGMENT_SIZE + ")"); + checkpoint.put("queues.0.head_offset", Integer.toString(SEGMENT_SIZE - 1)); + checkpoint.put("queues.0.tail_offset", Integer.toString(-1)); + checkpoint.put("segments.last_page", Integer.toString(0)); + checkpoint.put("segments.last_segment", Integer.toString(2)); + File propsFile = this.tempQueueFolder.resolve("checkpoint.properties").toFile(); + final FileWriter propsWriter = new FileWriter(propsFile); + checkpoint.store(propsWriter, "test checkpoint file to verify loading of inverted segments"); + + final QueuePool queuePool = QueuePool.loadQueues(tempQueueFolder, PAGE_SIZE, SEGMENT_SIZE); + final Queue queue = queuePool.getOrCreate("test_inverted"); + + assertContainsOnly('A', queue.dequeue().get(), SEGMENT_SIZE - LENGTH_HEADER_SIZE); + assertContainsOnly('B', queue.dequeue().get(), SEGMENT_SIZE - LENGTH_HEADER_SIZE); + } + + @Test + public void reopenQueueWithSomeDataInto() throws QueueException { + // given a queue wth some data split across multiple segments + final QueuePool queuePoolA = QueuePool.loadQueues(tempQueueFolder, PAGE_SIZE, SEGMENT_SIZE); + final Queue queueA = queuePoolA.getOrCreate("testA"); + queueA.enqueue(ByteBuffer.wrap(generatePayload(SEGMENT_SIZE / 2 - LENGTH_HEADER_SIZE, (byte) 'a'))); + queueA.enqueue(ByteBuffer.wrap(generatePayload(SEGMENT_SIZE / 2 - LENGTH_HEADER_SIZE, (byte) 'A'))); + queueA.enqueue(ByteBuffer.wrap(generatePayload(SEGMENT_SIZE / 2 - LENGTH_HEADER_SIZE, (byte) 'b'))); + queueA.enqueue(ByteBuffer.wrap(generatePayload(SEGMENT_SIZE / 2 - LENGTH_HEADER_SIZE, (byte) 'B'))); + // when it's closed and reopened + queueA.force(); + queuePoolA.close(); + + // then the consumption must happen in the same order + final Queue reopened = queuePoolA.getOrCreate("testA"); + assertContainsOnly('a', reopened.dequeue().get(), SEGMENT_SIZE / 2 - LENGTH_HEADER_SIZE); + assertContainsOnly('A', reopened.dequeue().get(), SEGMENT_SIZE / 2 - LENGTH_HEADER_SIZE); + assertContainsOnly('b', reopened.dequeue().get(), SEGMENT_SIZE / 2 - LENGTH_HEADER_SIZE); + assertContainsOnly('B', reopened.dequeue().get(), SEGMENT_SIZE / 2 - LENGTH_HEADER_SIZE); + } + + @Test + public void writeTestSuiteToVerifyPagedFilesAllocatorDoesntCreateExternalFragmentation() throws QueueException, IOException { + // write 2 segments, consume one segment, next segment allocated should be one just freed.0 + final QueuePool queuePool = QueuePool.loadQueues(tempQueueFolder, PAGE_SIZE, SEGMENT_SIZE); + final Queue queue = queuePool.getOrCreate("test_external_fragmentation"); + + // fill first segment (0, 0) + queue.enqueue(ByteBuffer.wrap(generatePayload(SEGMENT_SIZE / 2 - LENGTH_HEADER_SIZE, (byte) 'a'))); + queue.enqueue(ByteBuffer.wrap(generatePayload(SEGMENT_SIZE / 2 - LENGTH_HEADER_SIZE, (byte) 'A'))); + + // fill second segment (0, 4194304) + queue.enqueue(ByteBuffer.wrap(generatePayload(SEGMENT_SIZE / 2 - LENGTH_HEADER_SIZE, (byte) 'b'))); + queue.enqueue(ByteBuffer.wrap(generatePayload(SEGMENT_SIZE / 2 - LENGTH_HEADER_SIZE, (byte) 'B'))); + + // consume first segment + assertContainsOnly('a', queue.dequeue().get(), SEGMENT_SIZE / 2 - LENGTH_HEADER_SIZE); + assertContainsOnly('A', queue.dequeue().get(), SEGMENT_SIZE / 2 - LENGTH_HEADER_SIZE); + + // Exercise + // write new data, should go in first freed segment + queue.enqueue(ByteBuffer.wrap(generatePayload(SEGMENT_SIZE / 2 - LENGTH_HEADER_SIZE, (byte) 'c'))); + queuePool.close(); + + // Verify + // checkpoint contains che correct order, (0,0), (0, 4194304) + final Properties checkpointProps = loadCheckpointFile(tempQueueFolder); + + final String segmentRefs = checkpointProps.getProperty("queues.0.segments"); + assertEquals("(0, 0), (0, 4194304)", segmentRefs); + } + + @Test + public void reopenQueueWithFragmentation() throws QueueException, IOException { + // write 2 segments, consume one segment, next segment allocated should be one just freed.0 + final QueuePool queuePool = QueuePool.loadQueues(tempQueueFolder, PAGE_SIZE, SEGMENT_SIZE); + final Queue queue = queuePool.getOrCreate("test_external_fragmentation"); + + // fill first segment (0, 0) + queue.enqueue(ByteBuffer.wrap(generatePayload(SEGMENT_SIZE / 2 - LENGTH_HEADER_SIZE, (byte) 'a'))); + queue.enqueue(ByteBuffer.wrap(generatePayload(SEGMENT_SIZE / 2 - LENGTH_HEADER_SIZE, (byte) 'A'))); + + // fill second segment (0, 4194304) + queue.enqueue(ByteBuffer.wrap(generatePayload(SEGMENT_SIZE / 2 - LENGTH_HEADER_SIZE, (byte) 'b'))); + queue.enqueue(ByteBuffer.wrap(generatePayload(SEGMENT_SIZE / 2 - LENGTH_HEADER_SIZE, (byte) 'B'))); + + // consume first segment + assertContainsOnly('a', queue.dequeue().get(), SEGMENT_SIZE / 2 - LENGTH_HEADER_SIZE); + assertContainsOnly('A', queue.dequeue().get(), SEGMENT_SIZE / 2 - LENGTH_HEADER_SIZE); + + queue.force(); + queuePool.close(); + + // Exercise + // reopen the queue + final QueuePool recreatedQueuePool = QueuePool.loadQueues(tempQueueFolder, PAGE_SIZE, SEGMENT_SIZE); + final Queue reopened = recreatedQueuePool.getOrCreate("test_external_fragmentation"); + // write new data, should go in first freed segment + reopened.enqueue(ByteBuffer.wrap(generatePayload(SEGMENT_SIZE / 2 - LENGTH_HEADER_SIZE, (byte) 'c'))); + recreatedQueuePool.close(); + + // Verify + // checkpoint contains che correct order, (0,0), (0, 4194304) + final Properties checkpointProps = loadCheckpointFile(tempQueueFolder); + + final String segmentRefs = checkpointProps.getProperty("queues.0.segments"); + assertEquals("(0, 0), (0, 4194304)", segmentRefs); + } + + private Properties loadCheckpointFile(Path dir) throws IOException { + final Path checkpointPath = dir.resolve("checkpoint.properties"); + final FileReader fileReader = new FileReader(checkpointPath.toFile()); + final Properties checkpointProps = new Properties(); + checkpointProps.load(fileReader); + return checkpointProps; + } +} diff --git a/moquette-0.17/broker/src/test/java/io/moquette/broker/unsafequeues/SegmentPointerTest.java b/moquette-0.17/broker/src/test/java/io/moquette/broker/unsafequeues/SegmentPointerTest.java new file mode 100644 index 00000000..f5afdbe3 --- /dev/null +++ b/moquette-0.17/broker/src/test/java/io/moquette/broker/unsafequeues/SegmentPointerTest.java @@ -0,0 +1,30 @@ +package io.moquette.broker.unsafequeues; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +class SegmentPointerTest { + + @Test + public void testCompareInSameSegment() { + final SegmentPointer minor = new SegmentPointer(1, 10); + final SegmentPointer otherMinor = new SegmentPointer(1, 10); + final SegmentPointer major = new SegmentPointer(1, 12); + + assertEquals(-1, minor.compareTo(major), "minor is less than major"); + assertEquals(1, major.compareTo(minor), "major is greater than minor"); + assertEquals(0, minor.compareTo(otherMinor), "minor equals itself"); + } + + @Test + public void testCompareInDifferentSegments() { + final SegmentPointer minor = new SegmentPointer(1, 10); + final SegmentPointer otherMinor = new SegmentPointer(1, 10); + final SegmentPointer major = new SegmentPointer(2, 4); + + assertEquals(-1, minor.compareTo(major), "minor is less than major"); + assertEquals(1, major.compareTo(minor), "major is greater than minor"); + assertEquals(0, minor.compareTo(otherMinor), "minor equals itself"); + } +} diff --git a/moquette-0.17/broker/src/test/java/io/moquette/broker/unsafequeues/SegmentTest.java b/moquette-0.17/broker/src/test/java/io/moquette/broker/unsafequeues/SegmentTest.java new file mode 100644 index 00000000..4113010b --- /dev/null +++ b/moquette-0.17/broker/src/test/java/io/moquette/broker/unsafequeues/SegmentTest.java @@ -0,0 +1,39 @@ +package io.moquette.broker.unsafequeues; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.MappedByteBuffer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class SegmentTest { + + @Test + public void testHasSpace() throws IOException { + final MappedByteBuffer pageBuffer = Utils.createPageFile(); + + final Segment segment = new Segment(pageBuffer, new SegmentPointer(0, 0), new SegmentPointer(0, 1023)); + + final VirtualPointer current = new VirtualPointer(511); + final VirtualPointer otherCurrent = current.moveForward(pageBuffer.capacity()); // pointer in next page + + assertTrue(segment.hasSpace(current, 512)); + assertFalse(segment.hasSpace(current, 513)); + assertFalse(segment.hasSpace(otherCurrent, 513)); + } + + @Test + public void testBytesAfter() throws IOException { + final MappedByteBuffer pageBuffer = Utils.createPageFile(); + + final SegmentPointer begin = new SegmentPointer(0, 0); + final SegmentPointer end = new SegmentPointer(0, 1023); + final Segment segment = new Segment(pageBuffer, begin, end); + + assertEquals(0, segment.bytesAfter(end)); + assertEquals(1023, segment.bytesAfter(begin)); + } +} diff --git a/moquette-0.17/broker/src/test/java/io/moquette/broker/unsafequeues/Utils.java b/moquette-0.17/broker/src/test/java/io/moquette/broker/unsafequeues/Utils.java new file mode 100644 index 00000000..74966c45 --- /dev/null +++ b/moquette-0.17/broker/src/test/java/io/moquette/broker/unsafequeues/Utils.java @@ -0,0 +1,37 @@ +package io.moquette.broker.unsafequeues; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; + +class Utils { + + static MappedByteBuffer createPageFile() throws IOException { + return createPageFile(1024); + } + + static MappedByteBuffer createPageFile(int size) throws IOException { + final Path pageFile = File.createTempFile("test_queue", ".page").toPath(); + final OpenOption[] openOptions = {StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING}; + FileChannel fileChannel = FileChannel.open(pageFile, openOptions); + return fileChannel.map(FileChannel.MapMode.READ_WRITE, 0, size); + } + + static MappedByteBuffer openPageFile(Path pageFile, int pageSize) throws IOException { + final OpenOption[] openOptions = {StandardOpenOption.READ, StandardOpenOption.TRUNCATE_EXISTING}; + FileChannel fileChannel = FileChannel.open(pageFile, openOptions); + return fileChannel.map(FileChannel.MapMode.READ_ONLY, 0, pageSize); + } + + static String bufferToString(ByteBuffer buffer) { + byte[] data = new byte[buffer.remaining()]; + buffer.get(data); + + return new String(data); + } +} diff --git a/moquette-0.17/broker/src/test/java/io/moquette/broker/unsafequeues/VirtualPointerTest.java b/moquette-0.17/broker/src/test/java/io/moquette/broker/unsafequeues/VirtualPointerTest.java new file mode 100644 index 00000000..80babada --- /dev/null +++ b/moquette-0.17/broker/src/test/java/io/moquette/broker/unsafequeues/VirtualPointerTest.java @@ -0,0 +1,18 @@ +package io.moquette.broker.unsafequeues; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +class VirtualPointerTest { + + @Test + public void testCompare() { + final VirtualPointer lower = new VirtualPointer(100L); + final VirtualPointer higher = new VirtualPointer(200L); + + assertEquals(1, higher.compareTo(lower), higher.logicalOffset + " must be greater than " + lower.logicalOffset); + assertEquals(-1, lower.compareTo(higher), lower.logicalOffset + " must be lower than " + higher.logicalOffset); + assertEquals(0, lower.compareTo(lower), "identity must be equal"); + } +} diff --git a/moquette-0.16/broker/src/test/java/io/moquette/integration/AbstractIntegration.java b/moquette-0.17/broker/src/test/java/io/moquette/integration/AbstractIntegration.java similarity index 100% rename from moquette-0.16/broker/src/test/java/io/moquette/integration/AbstractIntegration.java rename to moquette-0.17/broker/src/test/java/io/moquette/integration/AbstractIntegration.java diff --git a/moquette-0.16/broker/src/test/java/io/moquette/integration/ConfigurationClassLoaderTest.java b/moquette-0.17/broker/src/test/java/io/moquette/integration/ConfigurationClassLoaderTest.java similarity index 86% rename from moquette-0.16/broker/src/test/java/io/moquette/integration/ConfigurationClassLoaderTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/integration/ConfigurationClassLoaderTest.java index ea32ea2b..6935cb83 100644 --- a/moquette-0.16/broker/src/test/java/io/moquette/integration/ConfigurationClassLoaderTest.java +++ b/moquette-0.17/broker/src/test/java/io/moquette/integration/ConfigurationClassLoaderTest.java @@ -32,7 +32,6 @@ import java.nio.file.Path; import java.util.Properties; -import static io.moquette.BrokerConstants.ENABLE_TELEMETRY_NAME; import static org.junit.jupiter.api.Assertions.assertTrue; public class ConfigurationClassLoaderTest implements IAuthenticator, IAuthorizatorPolicy { @@ -63,8 +62,8 @@ public void tearDown() { @Test public void loadAuthenticator() throws Exception { Properties props = new Properties(IntegrationUtils.prepareTestProperties(dbPath)); - props.setProperty(BrokerConstants.AUTHENTICATOR_CLASS_NAME, getClass().getName()); - props.setProperty(BrokerConstants.ENABLE_TELEMETRY_NAME, "false"); + props.setProperty(IConfig.AUTHENTICATOR_CLASS_NAME, getClass().getName()); + props.setProperty(IConfig.ENABLE_TELEMETRY_NAME, "false"); startServer(props); assertTrue(true); } @@ -72,8 +71,8 @@ public void loadAuthenticator() throws Exception { @Test public void loadAuthorizator() throws Exception { Properties props = new Properties(IntegrationUtils.prepareTestProperties(dbPath)); - props.setProperty(BrokerConstants.AUTHORIZATOR_CLASS_NAME, getClass().getName()); - props.setProperty(BrokerConstants.ENABLE_TELEMETRY_NAME, "false"); + props.setProperty(IConfig.AUTHORIZATOR_CLASS_NAME, getClass().getName()); + props.setProperty(IConfig.ENABLE_TELEMETRY_NAME, "false"); startServer(props); assertTrue(true); } diff --git a/moquette-0.16/broker/src/test/java/io/moquette/integration/FastPublisherSlowSubscriberTest.java b/moquette-0.17/broker/src/test/java/io/moquette/integration/FastPublisherSlowSubscriberTest.java similarity index 100% rename from moquette-0.16/broker/src/test/java/io/moquette/integration/FastPublisherSlowSubscriberTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/integration/FastPublisherSlowSubscriberTest.java diff --git a/moquette-0.16/broker/src/test/java/io/moquette/integration/IntegrationUtils.java b/moquette-0.17/broker/src/test/java/io/moquette/integration/IntegrationUtils.java similarity index 82% rename from moquette-0.16/broker/src/test/java/io/moquette/integration/IntegrationUtils.java rename to moquette-0.17/broker/src/test/java/io/moquette/integration/IntegrationUtils.java index de15253f..0cb30a2d 100644 --- a/moquette-0.16/broker/src/test/java/io/moquette/integration/IntegrationUtils.java +++ b/moquette-0.17/broker/src/test/java/io/moquette/integration/IntegrationUtils.java @@ -16,7 +16,6 @@ package io.moquette.integration; -import io.moquette.BrokerConstants; import org.eclipse.paho.client.mqttv3.IMqttActionListener; import org.eclipse.paho.client.mqttv3.IMqttAsyncClient; import org.eclipse.paho.client.mqttv3.IMqttClient; @@ -27,10 +26,12 @@ import java.nio.file.Path; import java.util.Properties; +import static io.moquette.broker.config.IConfig.DATA_PATH_PROPERTY_NAME; import static io.moquette.BrokerConstants.DEFAULT_MOQUETTE_STORE_H2_DB_FILENAME; -import static io.moquette.BrokerConstants.ENABLE_TELEMETRY_NAME; -import static io.moquette.BrokerConstants.PERSISTENT_STORE_PROPERTY_NAME; -import static io.moquette.BrokerConstants.PORT_PROPERTY_NAME; +import static io.moquette.broker.config.IConfig.ENABLE_TELEMETRY_NAME; +import static io.moquette.broker.config.IConfig.PERSISTENCE_ENABLED_PROPERTY_NAME; +import static io.moquette.broker.config.IConfig.PERSISTENT_QUEUE_TYPE_PROPERTY_NAME; +import static io.moquette.broker.config.IConfig.PORT_PROPERTY_NAME; /** * Used to carry integration configurations. @@ -58,9 +59,11 @@ public static String tempH2Path(Path tempFolder) { public static Properties prepareTestProperties(String dbPath) { Properties testProperties = new Properties(); - testProperties.put(PERSISTENT_STORE_PROPERTY_NAME, dbPath); + testProperties.put(DATA_PATH_PROPERTY_NAME, dbPath); + testProperties.put(PERSISTENCE_ENABLED_PROPERTY_NAME, "true"); testProperties.put(PORT_PROPERTY_NAME, "1883"); testProperties.put(ENABLE_TELEMETRY_NAME, "false"); + testProperties.put(PERSISTENT_QUEUE_TYPE_PROPERTY_NAME, "h2"); return testProperties; } diff --git a/moquette-0.16/broker/src/test/java/io/moquette/integration/MQTTWebSocket.java b/moquette-0.17/broker/src/test/java/io/moquette/integration/MQTTWebSocket.java similarity index 100% rename from moquette-0.16/broker/src/test/java/io/moquette/integration/MQTTWebSocket.java rename to moquette-0.17/broker/src/test/java/io/moquette/integration/MQTTWebSocket.java diff --git a/moquette-0.16/broker/src/test/java/io/moquette/integration/MessageCollector.java b/moquette-0.17/broker/src/test/java/io/moquette/integration/MessageCollector.java similarity index 100% rename from moquette-0.16/broker/src/test/java/io/moquette/integration/MessageCollector.java rename to moquette-0.17/broker/src/test/java/io/moquette/integration/MessageCollector.java diff --git a/moquette-0.16/broker/src/test/java/io/moquette/integration/PublishToManySubscribersUseCaseTest.java b/moquette-0.17/broker/src/test/java/io/moquette/integration/PublishToManySubscribersUseCaseTest.java similarity index 97% rename from moquette-0.16/broker/src/test/java/io/moquette/integration/PublishToManySubscribersUseCaseTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/integration/PublishToManySubscribersUseCaseTest.java index 684f1747..f9082697 100644 --- a/moquette-0.16/broker/src/test/java/io/moquette/integration/PublishToManySubscribersUseCaseTest.java +++ b/moquette-0.17/broker/src/test/java/io/moquette/integration/PublishToManySubscribersUseCaseTest.java @@ -1,6 +1,5 @@ package io.moquette.integration; -import io.moquette.BrokerConstants; import io.moquette.broker.Server; import io.moquette.broker.config.IConfig; import io.moquette.broker.config.MemoryConfig; @@ -54,7 +53,7 @@ public class PublishToManySubscribersUseCaseTest extends AbstractIntegration { protected void startServer(String dbPath) throws IOException { broker = new Server(); final Properties configProps = IntegrationUtils.prepareTestProperties(dbPath); - configProps.put(BrokerConstants.SESSION_QUEUE_SIZE, Integer.toString(COMMAND_QUEUE_SIZE)); + configProps.put(IConfig.SESSION_QUEUE_SIZE, Integer.toString(COMMAND_QUEUE_SIZE)); IConfig brokerConfig = new MemoryConfig(configProps); broker.startServer(brokerConfig); } @@ -163,7 +162,7 @@ void onePublishTriggerManySubscriptionsNotifications() throws MqttException, Int } private void segmentedParallelSubscriptions(BiConsumer biConsumer) throws InterruptedException { - int openSlotCount = COMMAND_QUEUE_SIZE; + int openSlotCount = COMMAND_QUEUE_SIZE / 2; Semaphore openSlots = new Semaphore(openSlotCount); IMqttActionListener completionCallback = createMqttCallback(openSlots); for (IMqttAsyncClient subscriber : this.subscribers) { diff --git a/moquette-0.16/broker/src/test/java/io/moquette/integration/ServerIntegrationDBAuthenticatorTest.java b/moquette-0.17/broker/src/test/java/io/moquette/integration/ServerIntegrationDBAuthenticatorTest.java similarity index 92% rename from moquette-0.16/broker/src/test/java/io/moquette/integration/ServerIntegrationDBAuthenticatorTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/integration/ServerIntegrationDBAuthenticatorTest.java index fe4f0251..10a9b670 100644 --- a/moquette-0.16/broker/src/test/java/io/moquette/integration/ServerIntegrationDBAuthenticatorTest.java +++ b/moquette-0.17/broker/src/test/java/io/moquette/integration/ServerIntegrationDBAuthenticatorTest.java @@ -17,12 +17,17 @@ package io.moquette.integration; import io.moquette.BrokerConstants; -import io.moquette.broker.config.IConfig; import io.moquette.broker.Server; +import io.moquette.broker.config.IConfig; import io.moquette.broker.config.MemoryConfig; import io.moquette.broker.security.DBAuthenticator; import io.moquette.broker.security.DBAuthenticatorTest; -import org.eclipse.paho.client.mqttv3.*; +import org.eclipse.paho.client.mqttv3.IMqttClient; +import org.eclipse.paho.client.mqttv3.MqttClient; +import org.eclipse.paho.client.mqttv3.MqttClientPersistence; +import org.eclipse.paho.client.mqttv3.MqttConnectOptions; +import org.eclipse.paho.client.mqttv3.MqttException; +import org.eclipse.paho.client.mqttv3.MqttSecurityException; import org.eclipse.paho.client.mqttv3.persist.MqttDefaultFilePersistence; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; @@ -39,7 +44,6 @@ import java.sql.SQLException; import java.util.Properties; -import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -77,7 +81,7 @@ private void stopServer() { } private Properties addDBAuthenticatorConf(Properties properties) { - properties.put(BrokerConstants.AUTHENTICATOR_CLASS_NAME, DBAuthenticator.class.getCanonicalName()); + properties.put(IConfig.AUTHENTICATOR_CLASS_NAME, DBAuthenticator.class.getCanonicalName()); properties.put(BrokerConstants.DB_AUTHENTICATOR_DRIVER, DBAuthenticatorTest.ORG_H2_DRIVER); properties.put(BrokerConstants.DB_AUTHENTICATOR_URL, DBAuthenticatorTest.JDBC_H2_MEM_TEST); properties.put(BrokerConstants.DB_AUTHENTICATOR_QUERY, "SELECT PASSWORD FROM ACCOUNT WHERE LOGIN=?"); diff --git a/moquette-0.16/broker/src/test/java/io/moquette/integration/ServerIntegrationFuseTest.java b/moquette-0.17/broker/src/test/java/io/moquette/integration/ServerIntegrationFuseTest.java similarity index 95% rename from moquette-0.16/broker/src/test/java/io/moquette/integration/ServerIntegrationFuseTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/integration/ServerIntegrationFuseTest.java index 01ab2fed..eeb9ad37 100644 --- a/moquette-0.16/broker/src/test/java/io/moquette/integration/ServerIntegrationFuseTest.java +++ b/moquette-0.17/broker/src/test/java/io/moquette/integration/ServerIntegrationFuseTest.java @@ -103,8 +103,8 @@ public void checkWillTestamentIsPublishedOnConnectionKill_noRetain() throws Exce // Exercise, kill the publisher connection m_publisher.kill(); - // Verify, that the testament is fired (wait the flush interval (1 sec) + small buffer) - Message msg = m_subscriber.receive(1500, TimeUnit.MILLISECONDS); + // Verify, that the testament is fired + Message msg = m_subscriber.receive(1, TimeUnit.SECONDS); // wait the flush interval (1 sec) assertNotNull(msg, "We should get notified with 'Will' message"); msg.ack(); assertEquals(willTestamentMsg, new String(msg.getPayload(), UTF_8)); diff --git a/moquette-0.16/broker/src/test/java/io/moquette/integration/ServerIntegrationMultiConnectTest.java b/moquette-0.17/broker/src/test/java/io/moquette/integration/ServerIntegrationMultiConnectTest.java similarity index 100% rename from moquette-0.16/broker/src/test/java/io/moquette/integration/ServerIntegrationMultiConnectTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/integration/ServerIntegrationMultiConnectTest.java diff --git a/moquette-0.16/broker/src/test/java/io/moquette/integration/ServerIntegrationOpenSSLTest.java b/moquette-0.17/broker/src/test/java/io/moquette/integration/ServerIntegrationOpenSSLTest.java similarity index 74% rename from moquette-0.16/broker/src/test/java/io/moquette/integration/ServerIntegrationOpenSSLTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/integration/ServerIntegrationOpenSSLTest.java index 940e08c7..f07dc45d 100644 --- a/moquette-0.16/broker/src/test/java/io/moquette/integration/ServerIntegrationOpenSSLTest.java +++ b/moquette-0.17/broker/src/test/java/io/moquette/integration/ServerIntegrationOpenSSLTest.java @@ -16,8 +16,8 @@ package io.moquette.integration; -import io.moquette.BrokerConstants; import io.moquette.broker.Server; +import io.moquette.broker.config.IConfig; import io.netty.handler.ssl.OpenSsl; import io.netty.handler.ssl.SslProvider; import org.junit.jupiter.api.Assumptions; @@ -37,7 +37,7 @@ public class ServerIntegrationOpenSSLTest extends ServerIntegrationSSLTest { private static final Logger LOG = LoggerFactory.getLogger(ServerIntegrationOpenSSLTest.class); @BeforeAll - public static void beforeTests() { + public static void checkForOpenSSL() { Assumptions.assumeTrue(new OpensslChecker(), "OpenSSL is available"); } @@ -48,16 +48,18 @@ protected void startServer() throws IOException { m_server = new Server(); Properties sslProps = new Properties(); - sslProps.put(BrokerConstants.SSL_PROVIDER, SslProvider.OPENSSL.name()); + sslProps.put(IConfig.SSL_PROVIDER, SslProvider.OPENSSL.name()); // sslProps.put(BrokerConstants.NEED_CLIENT_AUTH, "true"); - sslProps.put(BrokerConstants.SSL_PORT_PROPERTY_NAME, "8883"); - sslProps.put(BrokerConstants.JKS_PATH_PROPERTY_NAME, "src/test/resources/serverkeystore.jks"); - sslProps.put(BrokerConstants.KEY_STORE_PASSWORD_PROPERTY_NAME, "passw0rdsrv"); - sslProps.put(BrokerConstants.KEY_MANAGER_PASSWORD_PROPERTY_NAME, "passw0rdsrv"); - sslProps.put(BrokerConstants.PERSISTENT_STORE_PROPERTY_NAME, dbPath); + sslProps.put(IConfig.SSL_PORT_PROPERTY_NAME, "8883"); + sslProps.put(IConfig.JKS_PATH_PROPERTY_NAME, "src/test/resources/serverkeystore.jks"); + sslProps.put(IConfig.KEY_STORE_PASSWORD_PROPERTY_NAME, "passw0rdsrv"); + sslProps.put(IConfig.KEY_MANAGER_PASSWORD_PROPERTY_NAME, "passw0rdsrv"); + sslProps.put(IConfig.DATA_PATH_PROPERTY_NAME, dbPath); + sslProps.put(IConfig.PERSISTENCE_ENABLED_PROPERTY_NAME, "true"); + sslProps.put(IConfig.PERSISTENT_QUEUE_TYPE_PROPERTY_NAME, "h2"); - sslProps.put(BrokerConstants.ENABLE_TELEMETRY_NAME, "false"); + sslProps.put(IConfig.ENABLE_TELEMETRY_NAME, "false"); m_server.startServer(sslProps); } diff --git a/moquette-0.16/broker/src/test/java/io/moquette/integration/ServerIntegrationPahoCanPublishOnReadBlockedTopicTest.java b/moquette-0.17/broker/src/test/java/io/moquette/integration/ServerIntegrationPahoCanPublishOnReadBlockedTopicTest.java similarity index 92% rename from moquette-0.16/broker/src/test/java/io/moquette/integration/ServerIntegrationPahoCanPublishOnReadBlockedTopicTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/integration/ServerIntegrationPahoCanPublishOnReadBlockedTopicTest.java index 088645a9..b7e2bde5 100644 --- a/moquette-0.16/broker/src/test/java/io/moquette/integration/ServerIntegrationPahoCanPublishOnReadBlockedTopicTest.java +++ b/moquette-0.17/broker/src/test/java/io/moquette/integration/ServerIntegrationPahoCanPublishOnReadBlockedTopicTest.java @@ -29,12 +29,7 @@ import io.netty.handler.codec.mqtt.MqttQoS; import org.awaitility.Awaitility; import org.awaitility.Durations; -import org.eclipse.paho.client.mqttv3.IMqttClient; -import org.eclipse.paho.client.mqttv3.MqttClient; -import org.eclipse.paho.client.mqttv3.MqttClientPersistence; -import org.eclipse.paho.client.mqttv3.MqttConnectOptions; -import org.eclipse.paho.client.mqttv3.MqttException; -import org.eclipse.paho.client.mqttv3.MqttMessage; +import org.eclipse.paho.client.mqttv3.*; import org.eclipse.paho.client.mqttv3.persist.MqttDefaultFilePersistence; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -44,13 +39,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.nio.file.Path; import java.util.Properties; import static io.moquette.broker.ConnectionTestUtils.EMPTY_OBSERVERS; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.fail; public class ServerIntegrationPahoCanPublishOnReadBlockedTopicTest { @@ -73,11 +68,12 @@ public static void beforeTests() { Awaitility.setDefaultTimeout(Durations.ONE_SECOND); } - protected void startServer(String dbPath) { + protected void startServer(String dbPath) throws IOException { m_server = new Server(); final Properties configProps = IntegrationUtils.prepareTestProperties(dbPath); configProps.setProperty(BrokerConstants.REAUTHORIZE_SUBSCRIPTIONS_ON_CONNECT, "true"); - configProps.setProperty(BrokerConstants.ENABLE_TELEMETRY_NAME, "false"); + configProps.setProperty(IConfig.ENABLE_TELEMETRY_NAME, "false"); + configProps.setProperty(IConfig.PERSISTENT_QUEUE_TYPE_PROPERTY_NAME, "h2"); m_config = new MemoryConfig(configProps); canRead = true; diff --git a/moquette-0.16/broker/src/test/java/io/moquette/integration/ServerIntegrationPahoTest.java b/moquette-0.17/broker/src/test/java/io/moquette/integration/ServerIntegrationPahoTest.java similarity index 100% rename from moquette-0.16/broker/src/test/java/io/moquette/integration/ServerIntegrationPahoTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/integration/ServerIntegrationPahoTest.java diff --git a/moquette-0.16/broker/src/test/java/io/moquette/integration/ServerIntegrationQoSValidationTest.java b/moquette-0.17/broker/src/test/java/io/moquette/integration/ServerIntegrationQoSValidationTest.java similarity index 100% rename from moquette-0.16/broker/src/test/java/io/moquette/integration/ServerIntegrationQoSValidationTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/integration/ServerIntegrationQoSValidationTest.java diff --git a/moquette-0.16/broker/src/test/java/io/moquette/integration/ServerIntegrationRestartTest.java b/moquette-0.17/broker/src/test/java/io/moquette/integration/ServerIntegrationRestartTest.java similarity index 94% rename from moquette-0.16/broker/src/test/java/io/moquette/integration/ServerIntegrationRestartTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/integration/ServerIntegrationRestartTest.java index 705a855c..8a5e25ae 100644 --- a/moquette-0.16/broker/src/test/java/io/moquette/integration/ServerIntegrationRestartTest.java +++ b/moquette-0.17/broker/src/test/java/io/moquette/integration/ServerIntegrationRestartTest.java @@ -46,7 +46,11 @@ public class ServerIntegrationRestartTest { private static final Logger LOG = LoggerFactory.getLogger(ServerIntegrationRestartTest.class); - static MqttConnectOptions CLEAN_SESSION_OPT = new MqttConnectOptions(); + static MqttConnectOptions NOT_CLEAN_SESSION_OPT; + static { + NOT_CLEAN_SESSION_OPT = new MqttConnectOptions(); + NOT_CLEAN_SESSION_OPT.setCleanSession(false); + } Server m_server; IMqttClient m_subscriber; @@ -69,7 +73,6 @@ protected void startServer(String dbPath) throws IOException { @BeforeAll public static void beforeTests() { - CLEAN_SESSION_OPT.setCleanSession(false); Awaitility.setDefaultTimeout(Durations.ONE_SECOND); } @@ -105,7 +108,7 @@ public void tearDown() throws Exception { @Test public void testNotCleanSessionIsVisibleAfterServerRestart() throws Exception { LOG.info("*** testNotCleanSessionIsVisibleAfterServerRestart ***"); - m_subscriber.connect(CLEAN_SESSION_OPT); + m_subscriber.connect(NOT_CLEAN_SESSION_OPT); m_subscriber.subscribe("/topic", 1); m_subscriber.disconnect(); @@ -117,9 +120,9 @@ public void testNotCleanSessionIsVisibleAfterServerRestart() throws Exception { m_publisher.publish("/topic", "Hello world MQTT!!".getBytes(UTF_8), 1, false); //reconnect subscriber and topic should be sent - m_subscriber.connect(CLEAN_SESSION_OPT); + m_subscriber.connect(NOT_CLEAN_SESSION_OPT); - // verify the sent message while offline is read + // verify the sent message while it was offline, is read Awaitility.await().until(m_messageCollector::isMessageReceived); MqttMessage msg = m_messageCollector.retrieveMessage(); assertEquals("Hello world MQTT!!", new String(msg.getPayload(), UTF_8)); @@ -129,7 +132,7 @@ public void testNotCleanSessionIsVisibleAfterServerRestart() throws Exception { public void checkRestartCleanSubscriptionTree() throws Exception { LOG.info("*** checkRestartCleanSubscriptionTree ***"); // subscribe to /topic - m_subscriber.connect(CLEAN_SESSION_OPT); + m_subscriber.connect(NOT_CLEAN_SESSION_OPT); m_subscriber.subscribe("/topic", 1); m_subscriber.disconnect(); @@ -140,11 +143,11 @@ public void checkRestartCleanSubscriptionTree() throws Exception { m_server.startServer(IntegrationUtils.prepareTestProperties(dbPath)); // reconnect the Subscriber subscribing to the same /topic but different QoS - m_subscriber.connect(CLEAN_SESSION_OPT); + m_subscriber.connect(NOT_CLEAN_SESSION_OPT); m_subscriber.subscribe("/topic", 2); // should be just one registration so a publisher receive one notification - m_publisher.connect(CLEAN_SESSION_OPT); + m_publisher.connect(NOT_CLEAN_SESSION_OPT); m_publisher.publish("/topic", "Hello world MQTT!!".getBytes(UTF_8), 1, false); // read the messages diff --git a/moquette-0.16/broker/src/test/java/io/moquette/integration/ServerIntegrationRetainTest.java b/moquette-0.17/broker/src/test/java/io/moquette/integration/ServerIntegrationRetainTest.java similarity index 100% rename from moquette-0.16/broker/src/test/java/io/moquette/integration/ServerIntegrationRetainTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/integration/ServerIntegrationRetainTest.java diff --git a/moquette-0.16/broker/src/test/java/io/moquette/integration/ServerIntegrationSSLClientAuthTest.java b/moquette-0.17/broker/src/test/java/io/moquette/integration/ServerIntegrationSSLClientAuthTest.java similarity index 88% rename from moquette-0.16/broker/src/test/java/io/moquette/integration/ServerIntegrationSSLClientAuthTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/integration/ServerIntegrationSSLClientAuthTest.java index f03ed420..8234b258 100644 --- a/moquette-0.16/broker/src/test/java/io/moquette/integration/ServerIntegrationSSLClientAuthTest.java +++ b/moquette-0.17/broker/src/test/java/io/moquette/integration/ServerIntegrationSSLClientAuthTest.java @@ -16,41 +16,27 @@ package io.moquette.integration; +import io.moquette.BrokerConstants; import io.moquette.broker.Server; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; +import io.moquette.broker.config.IConfig; +import org.eclipse.paho.client.mqttv3.*; +import org.eclipse.paho.client.mqttv3.persist.MqttDefaultFilePersistence; +import org.junit.jupiter.api.*; +import org.junit.jupiter.api.io.TempDir; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import javax.net.ssl.*; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; -import java.security.KeyManagementException; -import java.security.KeyStore; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.UnrecoverableKeyException; +import java.security.*; import java.security.cert.CertificateException; import java.util.Properties; -import javax.net.ssl.KeyManagerFactory; -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLSocketFactory; -import javax.net.ssl.TrustManager; -import javax.net.ssl.TrustManagerFactory; -import io.moquette.BrokerConstants; -import org.eclipse.paho.client.mqttv3.IMqttClient; -import org.eclipse.paho.client.mqttv3.MqttClient; -import org.eclipse.paho.client.mqttv3.MqttClientPersistence; -import org.eclipse.paho.client.mqttv3.MqttConnectOptions; -import org.eclipse.paho.client.mqttv3.MqttException; -import org.eclipse.paho.client.mqttv3.persist.MqttDefaultFilePersistence; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; /** * Check that Moquette could also handle SSL with client authentication. @@ -177,13 +163,15 @@ protected void startServer(String dbPath) throws IOException { m_server = new Server(); Properties sslProps = new Properties(); - sslProps.put(BrokerConstants.SSL_PORT_PROPERTY_NAME, "8883"); - sslProps.put(BrokerConstants.JKS_PATH_PROPERTY_NAME, "src/test/resources/serverkeystore.jks"); - sslProps.put(BrokerConstants.KEY_STORE_PASSWORD_PROPERTY_NAME, "passw0rdsrv"); - sslProps.put(BrokerConstants.KEY_MANAGER_PASSWORD_PROPERTY_NAME, "passw0rdsrv"); - sslProps.put(BrokerConstants.PERSISTENT_STORE_PROPERTY_NAME, dbPath); + sslProps.put(IConfig.SSL_PORT_PROPERTY_NAME, "8883"); + sslProps.put(IConfig.JKS_PATH_PROPERTY_NAME, "src/test/resources/serverkeystore.jks"); + sslProps.put(IConfig.KEY_STORE_PASSWORD_PROPERTY_NAME, "passw0rdsrv"); + sslProps.put(IConfig.KEY_MANAGER_PASSWORD_PROPERTY_NAME, "passw0rdsrv"); + sslProps.put(IConfig.DATA_PATH_PROPERTY_NAME, dbPath); + sslProps.put(IConfig.PERSISTENCE_ENABLED_PROPERTY_NAME, "true"); + sslProps.put(IConfig.PERSISTENT_QUEUE_TYPE_PROPERTY_NAME, "h2"); sslProps.put(BrokerConstants.NEED_CLIENT_AUTH, "true"); - sslProps.put(BrokerConstants.ENABLE_TELEMETRY_NAME, "false"); + sslProps.put(IConfig.ENABLE_TELEMETRY_NAME, "false"); m_server.startServer(sslProps); } diff --git a/moquette-0.16/broker/src/test/java/io/moquette/integration/ServerIntegrationSSLTest.java b/moquette-0.17/broker/src/test/java/io/moquette/integration/ServerIntegrationSSLTest.java similarity index 81% rename from moquette-0.16/broker/src/test/java/io/moquette/integration/ServerIntegrationSSLTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/integration/ServerIntegrationSSLTest.java index 2732e489..39851e51 100644 --- a/moquette-0.16/broker/src/test/java/io/moquette/integration/ServerIntegrationSSLTest.java +++ b/moquette-0.17/broker/src/test/java/io/moquette/integration/ServerIntegrationSSLTest.java @@ -17,41 +17,26 @@ package io.moquette.integration; import io.moquette.broker.Server; +import io.moquette.broker.config.IConfig; +import org.eclipse.paho.client.mqttv3.*; +import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence; +import org.eclipse.paho.client.mqttv3.persist.MqttDefaultFilePersistence; +import org.junit.jupiter.api.*; +import org.junit.jupiter.api.io.TempDir; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.junit.jupiter.api.Assertions.assertFalse; +import javax.net.ssl.*; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; -import java.security.KeyManagementException; -import java.security.KeyStore; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.UnrecoverableKeyException; +import java.security.*; import java.security.cert.CertificateException; import java.util.Properties; -import javax.net.ssl.KeyManagerFactory; -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLSocketFactory; -import javax.net.ssl.TrustManager; -import javax.net.ssl.TrustManagerFactory; -import io.moquette.BrokerConstants; -import org.eclipse.paho.client.mqttv3.IMqttClient; -import org.eclipse.paho.client.mqttv3.MqttClient; -import org.eclipse.paho.client.mqttv3.MqttClientPersistence; -import org.eclipse.paho.client.mqttv3.MqttConnectOptions; -import org.eclipse.paho.client.mqttv3.MqttMessage; -import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence; -import org.eclipse.paho.client.mqttv3.persist.MqttDefaultFilePersistence; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.jupiter.api.Assertions.assertFalse; /** * Check that Moquette could also handle SSL. @@ -109,12 +94,14 @@ protected void startServer() throws IOException { m_server = new Server(); Properties sslProps = new Properties(); - sslProps.put(BrokerConstants.SSL_PORT_PROPERTY_NAME, "8883"); - sslProps.put(BrokerConstants.JKS_PATH_PROPERTY_NAME, "src/test/resources/serverkeystore.jks"); - sslProps.put(BrokerConstants.KEY_STORE_PASSWORD_PROPERTY_NAME, "passw0rdsrv"); - sslProps.put(BrokerConstants.KEY_MANAGER_PASSWORD_PROPERTY_NAME, "passw0rdsrv"); - sslProps.put(BrokerConstants.PERSISTENT_STORE_PROPERTY_NAME, dbPath); - sslProps.put(BrokerConstants.ENABLE_TELEMETRY_NAME, "false"); + sslProps.put(IConfig.SSL_PORT_PROPERTY_NAME, "8883"); + sslProps.put(IConfig.JKS_PATH_PROPERTY_NAME, "src/test/resources/serverkeystore.jks"); + sslProps.put(IConfig.KEY_STORE_PASSWORD_PROPERTY_NAME, "passw0rdsrv"); + sslProps.put(IConfig.KEY_MANAGER_PASSWORD_PROPERTY_NAME, "passw0rdsrv"); + sslProps.put(IConfig.DATA_PATH_PROPERTY_NAME, dbPath); + sslProps.put(IConfig.PERSISTENCE_ENABLED_PROPERTY_NAME, "true"); + sslProps.put(IConfig.PERSISTENT_QUEUE_TYPE_PROPERTY_NAME, "h2"); + sslProps.put(IConfig.ENABLE_TELEMETRY_NAME, "false"); m_server.startServer(sslProps); } diff --git a/moquette-0.16/broker/src/test/java/io/moquette/integration/ServerIntegrationWebSocketTest.java b/moquette-0.17/broker/src/test/java/io/moquette/integration/ServerIntegrationWebSocketTest.java similarity index 80% rename from moquette-0.16/broker/src/test/java/io/moquette/integration/ServerIntegrationWebSocketTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/integration/ServerIntegrationWebSocketTest.java index 3c95a5e5..43a12373 100644 --- a/moquette-0.16/broker/src/test/java/io/moquette/integration/ServerIntegrationWebSocketTest.java +++ b/moquette-0.17/broker/src/test/java/io/moquette/integration/ServerIntegrationWebSocketTest.java @@ -16,23 +16,24 @@ package io.moquette.integration; -import io.moquette.broker.Server; import io.moquette.BrokerConstants; -import io.moquette.broker.config.MemoryConfig; +import io.moquette.broker.Server; +import io.moquette.broker.config.FluentConfig; +import io.moquette.broker.config.IConfig; import org.eclipse.jetty.websocket.client.ClientUpgradeRequest; import org.eclipse.jetty.websocket.client.WebSocketClient; -import io.moquette.broker.config.IConfig; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import java.io.IOException; import java.net.URI; import java.nio.file.Path; -import java.util.Properties; import java.util.concurrent.TimeUnit; + import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -50,12 +51,23 @@ public class ServerIntegrationWebSocketTest { Path tempFolder; protected void startServer(String dbPath) throws IOException { - m_server = new Server(); - final Properties configProps = IntegrationUtils.prepareTestProperties(dbPath); - configProps.put(BrokerConstants.WEB_SOCKET_PORT_PROPERTY_NAME, Integer.toString(BrokerConstants.WEBSOCKET_PORT)); - configProps.put(BrokerConstants.PERSISTENT_STORE_PROPERTY_NAME, dbPath); - m_config = new MemoryConfig(configProps); - m_server.startServer(m_config); +// m_server = new Server(); +// m_config = new FluentConfig() +// .dataPath(dbPath) +// .enablePersistence() +// .disableTelemetry() +// .websocketPort(BrokerConstants.WEBSOCKET_PORT) +// .build(); +// m_server.startServer(m_config); + + m_server = new Server() + .withConfig() + .dataPath(dbPath) + .enablePersistence() + .persistentQueueType(FluentConfig.PersistentQueueType.H2) + .disableTelemetry() + .websocketPort(BrokerConstants.WEBSOCKET_PORT) + .startServer(); } @BeforeEach diff --git a/moquette-0.16/broker/src/test/java/io/moquette/integration/ServerLowlevelMessagesIntegrationTest.java b/moquette-0.17/broker/src/test/java/io/moquette/integration/ServerLowlevelMessagesIntegrationTest.java similarity index 100% rename from moquette-0.16/broker/src/test/java/io/moquette/integration/ServerLowlevelMessagesIntegrationTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/integration/ServerLowlevelMessagesIntegrationTest.java diff --git a/moquette-0.16/broker/src/test/java/io/moquette/interception/BrokerInterceptorTest.java b/moquette-0.17/broker/src/test/java/io/moquette/interception/BrokerInterceptorTest.java similarity index 97% rename from moquette-0.16/broker/src/test/java/io/moquette/interception/BrokerInterceptorTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/interception/BrokerInterceptorTest.java index 42c52a86..48cbf3f2 100644 --- a/moquette-0.16/broker/src/test/java/io/moquette/interception/BrokerInterceptorTest.java +++ b/moquette-0.17/broker/src/test/java/io/moquette/interception/BrokerInterceptorTest.java @@ -89,6 +89,11 @@ public void onUnsubscribe(InterceptUnsubscribeMessage msg) { public void onMessageAcknowledged(InterceptAcknowledgedMessage msg) { n.set(90); } + + @Override + public void onSessionLoopError(Throwable error) { + throw new RuntimeException(error); + } } private static final BrokerInterceptor interceptor = new BrokerInterceptor( diff --git a/moquette-0.16/broker/src/test/java/io/moquette/persistence/H2PersistentQueueTest.java b/moquette-0.17/broker/src/test/java/io/moquette/persistence/H2PersistentQueueTest.java similarity index 100% rename from moquette-0.16/broker/src/test/java/io/moquette/persistence/H2PersistentQueueTest.java rename to moquette-0.17/broker/src/test/java/io/moquette/persistence/H2PersistentQueueTest.java diff --git a/moquette-0.17/broker/src/test/java/io/moquette/persistence/SegmentPersistentQueueTest.java b/moquette-0.17/broker/src/test/java/io/moquette/persistence/SegmentPersistentQueueTest.java new file mode 100644 index 00000000..25bb6e85 --- /dev/null +++ b/moquette-0.17/broker/src/test/java/io/moquette/persistence/SegmentPersistentQueueTest.java @@ -0,0 +1,273 @@ +/* + * Copyright (c) 2012-2023 The original author or authors + * ------------------------------------------------------ + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Apache License v2.0 which accompanies this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * The Apache License v2.0 is available at + * http://www.opensource.org/licenses/apache2.0.php + * + * You may elect to redistribute this code under either of these licenses. + */ +package io.moquette.persistence; + +import io.moquette.broker.SessionMessageQueue; +import io.moquette.broker.SessionRegistry; +import io.moquette.broker.SessionRegistry.EnqueuedMessage; +import io.moquette.broker.SessionRegistry.PublishedMessage; +import io.moquette.broker.subscriptions.Topic; +import io.moquette.broker.unsafequeues.QueueException; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.handler.codec.mqtt.MqttQoS; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; +import org.junit.jupiter.api.io.TempDir; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@DisabledOnOs(OS.WINDOWS) +public class SegmentPersistentQueueTest { + + private static final Logger LOGGER = LoggerFactory.getLogger(SegmentPersistentQueueTest.class.getName()); + + private static final int PAGE_SIZE = 5000; + private static final int SEGMENT_SIZE = 1000; + private static SegmentQueueRepository queueRepository; + List> queues = new ArrayList<>(); + + private int queueIndex = 0; + private static final String QUEUE_NAME = "test"; + + @TempDir + static Path tempQueueFolder; + + @BeforeAll + public static void beforeAll() throws IOException, QueueException { + System.setProperty("moquette.queue.debug", "false"); + queueRepository = new SegmentQueueRepository(tempQueueFolder.toFile().getAbsolutePath(), PAGE_SIZE, SEGMENT_SIZE); + } + + @AfterEach + public void tearDown() { + for (SessionMessageQueue queue : queues) { + queue.closeAndPurge(); + } + queues.clear(); + } + + @AfterAll + public static void afterAll() { + queueRepository.close(); + + } + + private SessionMessageQueue createQueue() { + SessionMessageQueue queue = queueRepository.getOrCreateQueue(QUEUE_NAME + (queueIndex++)); + queues.add(queue); + return queue; + } + + private static void createAndAddToQueue(SessionMessageQueue queue, String topic, int totalSize) { + final PublishedMessage msg1 = createMessage(topic, totalSize); + msg1.retain(); + queue.enqueue(msg1); + msg1.release(); + } + + private void createAndAddToQueues(String topic, int totalSize) { + final PublishedMessage msg = createMessage(topic, totalSize); + for (SessionMessageQueue queue : queues) { + msg.retain(); + queue.enqueue(msg); + } + msg.release(); + } + + private void assertAllEmpty(String message) { + for (SessionMessageQueue queue : queues) { + assertTrue(queue.isEmpty(), message); + } + } + + private void assertAllNonEmpty(String message) { + for (SessionMessageQueue queue : queues) { + assertFalse(queue.isEmpty(), message); + } + } + + private void dequeueFromAll(String expected) { + for (SessionMessageQueue queue : queues) { + final SessionRegistry.PublishedMessage mesg = (PublishedMessage) queue.dequeue(); + assertEquals(expected, mesg.getTopic().toString()); + } + } + + @Test + public void testAdd() { + LOGGER.info("testAdd"); + SessionMessageQueue queue = queueRepository.getOrCreateQueue("testAdd"); + queues.add(queue); + assertTrue(queue.isEmpty(), "Queue must start empty."); + createAndAddToQueue(queue, "Hello", 100); + assertFalse(queue.isEmpty(), "Queue must not be empty after adding."); + createAndAddToQueue(queue, "world", 100); + assertFalse(queue.isEmpty(), "Queue must not be empty after adding."); + + assertEquals("Hello", ((PublishedMessage) queue.dequeue()).getTopic().toString()); + assertEquals("world", ((PublishedMessage) queue.dequeue()).getTopic().toString()); + assertAllEmpty("After dequeueing all, queue must be empty"); + + createAndAddToQueue(queue, "Hello", 100); + assertFalse(queue.isEmpty(), "Queue must not be empty after adding."); + assertEquals("Hello", ((PublishedMessage) queue.dequeue()).getTopic().toString()); + assertAllEmpty("After dequeueing all, queue must be empty"); + } + + @Test + public void testAdd2() { + LOGGER.info("testAdd2"); + testAddX(2); + } + + @Test + public void testAdd10() { + LOGGER.info("testAdd10"); + testAddX(10); + } + + public void testAddX(int x) { + for (int i = 0; i < x; i++) { + createQueue(); + } + assertAllEmpty("Queue must start empty."); + + createAndAddToQueues("Hello", 100); + assertAllNonEmpty("Queue must not be empty after adding."); + + createAndAddToQueues("world", 100); + assertAllNonEmpty("Queue must not be empty after adding."); + + dequeueFromAll("Hello"); + assertAllNonEmpty("After dequeueing one, queue must not be empty"); + + createAndAddToQueues("crazy", 100); + assertAllNonEmpty("Queue must not be empty after adding."); + + dequeueFromAll("world"); + assertAllNonEmpty("Queue must not be empty after adding."); + + dequeueFromAll("crazy"); + assertAllEmpty("After dequeueing all, queue must be empty"); + } + private static String body; + + private static String getBody(int bodySize) { + if (body == null || body.length() != bodySize) { + char a = 'A'; + char z = 'Z'; + char curChar = a; + StringBuilder bodyString = new StringBuilder(); + for (int i = 0; i < bodySize; i++) { + bodyString.append(curChar); + if (curChar == z) { + curChar = a; + } else { + curChar++; + } + } + body = bodyString.toString(); + } + return body; + } + + private static void checkMessage(PublishedMessage message, String expTopic) { + assertEquals(expTopic, message.getTopic().toString()); + final String receivedBody = message.getPayload().toString(UTF_8); + final String expectedBody = getBody(receivedBody.length()); + assertEquals(expectedBody, receivedBody); + } + + private static PublishedMessage createMessage(String topic, int totalMessageSize) { + // 4 totalSize + 1 msgType + 1 qos + 4 topicSize + 4 bodySize = 14 + int bodySize = totalMessageSize - 14 - topic.getBytes(UTF_8).length; + final ByteBuf payload = Unpooled.wrappedBuffer(getBody(bodySize).getBytes(StandardCharsets.UTF_8)); + return new PublishedMessage(Topic.asTopic(topic), MqttQoS.AT_LEAST_ONCE, payload, false); + } + + @Test + public void testPerformance() { + LOGGER.info("testPerformance"); + SessionMessageQueue queue = queueRepository.getOrCreateQueue("testPerformance"); + queues.add(queue); + final String topic = "Hello"; + final int numIterations = 10_000; + final int perIteration = 3; + // With a total (in-queue) message size of 201, and a segment size of 1000 + // we can be sure we hit all corner cases. + final int totalMessageSize = 201; + int countPush = 0; + int countPull = 0; + int j = 0; + final String message = "Queue should have contained " + perIteration + " items"; + try { + for (int i = 0; i < numIterations; i++) { + for (j = 0; j < perIteration; j++) { + countPush++; + LOGGER.debug("push {}, {}", countPush, j); + createAndAddToQueue(queue, topic, totalMessageSize); + } + j = 0; + while (!queue.isEmpty()) { + countPull++; + j++; + LOGGER.debug("pull {}, {}", countPull, j); + final PublishedMessage msg = (PublishedMessage) queue.dequeue(); + checkMessage(msg, topic); + } + assertEquals(perIteration, j, message); + } + } catch (Exception ex) { + LOGGER.error("", ex); + Assertions.fail("Failed on push count " + countPush + ", pull count " + countPull + ", j " + j, ex); + } + assertTrue(queue.isEmpty(), "should be empty"); + } + + @Test + public void testReloadFromPersistedState() { + LOGGER.info("testReloadFromPersistedState"); + SessionMessageQueue queue = queueRepository.getOrCreateQueue("testReloadFromPersistedState"); + queues.add(queue); + createAndAddToQueue(queue, "Hello", 100); + createAndAddToQueue(queue, "crazy", 100); + createAndAddToQueue(queue, "world", 100); + assertEquals("Hello", ((PublishedMessage) queue.dequeue()).getTopic().toString()); + + queue = queueRepository.getOrCreateQueue("testReloadFromPersistedState"); + + assertEquals("crazy", ((PublishedMessage) queue.dequeue()).getTopic().toString()); + assertEquals("world", ((PublishedMessage) queue.dequeue()).getTopic().toString()); + assertTrue(queue.isEmpty(), "should be empty"); + } +} diff --git a/moquette-0.16/broker/src/test/java/io/moquette/spec/v3_1_1/connection/ConnectionIT.java b/moquette-0.17/broker/src/test/java/io/moquette/spec/v3_1_1/connection/ConnectionIT.java similarity index 100% rename from moquette-0.16/broker/src/test/java/io/moquette/spec/v3_1_1/connection/ConnectionIT.java rename to moquette-0.17/broker/src/test/java/io/moquette/spec/v3_1_1/connection/ConnectionIT.java diff --git a/moquette-0.16/broker/src/test/java/io/moquette/testclient/Client.java b/moquette-0.17/broker/src/test/java/io/moquette/testclient/Client.java similarity index 100% rename from moquette-0.16/broker/src/test/java/io/moquette/testclient/Client.java rename to moquette-0.17/broker/src/test/java/io/moquette/testclient/Client.java diff --git a/moquette-0.16/broker/src/test/java/io/moquette/testclient/ClientNettyMQTTHandler.java b/moquette-0.17/broker/src/test/java/io/moquette/testclient/ClientNettyMQTTHandler.java similarity index 100% rename from moquette-0.16/broker/src/test/java/io/moquette/testclient/ClientNettyMQTTHandler.java rename to moquette-0.17/broker/src/test/java/io/moquette/testclient/ClientNettyMQTTHandler.java diff --git a/moquette-0.16/broker/src/test/java/io/moquette/testclient/RawClient.java b/moquette-0.17/broker/src/test/java/io/moquette/testclient/RawClient.java similarity index 100% rename from moquette-0.16/broker/src/test/java/io/moquette/testclient/RawClient.java rename to moquette-0.17/broker/src/test/java/io/moquette/testclient/RawClient.java diff --git a/moquette-0.16/broker/src/test/resources/clientkeystore.jks b/moquette-0.17/broker/src/test/resources/clientkeystore.jks similarity index 100% rename from moquette-0.16/broker/src/test/resources/clientkeystore.jks rename to moquette-0.17/broker/src/test/resources/clientkeystore.jks diff --git a/moquette-0.16/broker/src/test/resources/config/moquette.conf b/moquette-0.17/broker/src/test/resources/config/moquette.conf similarity index 100% rename from moquette-0.16/broker/src/test/resources/config/moquette.conf rename to moquette-0.17/broker/src/test/resources/config/moquette.conf diff --git a/moquette-0.16/broker/src/test/resources/config/password_file.conf b/moquette-0.17/broker/src/test/resources/config/password_file.conf similarity index 100% rename from moquette-0.16/broker/src/test/resources/config/password_file.conf rename to moquette-0.17/broker/src/test/resources/config/password_file.conf diff --git a/moquette-0.16/broker/src/test/resources/log4j.properties b/moquette-0.17/broker/src/test/resources/log4j.properties similarity index 95% rename from moquette-0.16/broker/src/test/resources/log4j.properties rename to moquette-0.17/broker/src/test/resources/log4j.properties index ddc004c1..fe693b8b 100644 --- a/moquette-0.16/broker/src/test/resources/log4j.properties +++ b/moquette-0.17/broker/src/test/resources/log4j.properties @@ -1,12 +1,13 @@ #log4j.rootLogger=ERROR, stdout, file log4j.rootLogger=ERROR, stdout -log4j.logger.io.moquette=INFO +log4j.logger.io.moquette=WARN #log4j.logger.io.moquette.broker=DEBUG #log4j.logger.io.moquette.broker.MQTTConnection=DEBUG #log4j.logger.io.moquette.broker.SessionRegistry=DEBUG #log4j.logger.io.moquette.broker.PostOffice=DEBUG #log4j.logger.io.moquette.broker=WARN +log4j.logger.io.moquette.broker.subscriptions.CTrieSpeedTest=INFO # stdout appender is set to be consoleAppender. log4j.appender.stdout=org.apache.log4j.ConsoleAppender diff --git a/moquette-0.16/broker/src/test/resources/password_file.conf b/moquette-0.17/broker/src/test/resources/password_file.conf similarity index 100% rename from moquette-0.16/broker/src/test/resources/password_file.conf rename to moquette-0.17/broker/src/test/resources/password_file.conf diff --git a/moquette-0.16/broker/src/test/resources/serverkeystore.jks b/moquette-0.17/broker/src/test/resources/serverkeystore.jks similarity index 100% rename from moquette-0.16/broker/src/test/resources/serverkeystore.jks rename to moquette-0.17/broker/src/test/resources/serverkeystore.jks diff --git a/moquette-0.16/broker/src/test/resources/signedclientkeystore.jks b/moquette-0.17/broker/src/test/resources/signedclientkeystore.jks similarity index 100% rename from moquette-0.16/broker/src/test/resources/signedclientkeystore.jks rename to moquette-0.17/broker/src/test/resources/signedclientkeystore.jks diff --git a/moquette-0.16/broker/src/test/resources/unsignedclientkeystore.jks b/moquette-0.17/broker/src/test/resources/unsignedclientkeystore.jks similarity index 100% rename from moquette-0.16/broker/src/test/resources/unsignedclientkeystore.jks rename to moquette-0.17/broker/src/test/resources/unsignedclientkeystore.jks diff --git a/moquette-0.16/checkstyle-suppressions.xml b/moquette-0.17/checkstyle-suppressions.xml similarity index 100% rename from moquette-0.16/checkstyle-suppressions.xml rename to moquette-0.17/checkstyle-suppressions.xml diff --git a/moquette-0.16/checkstyle.xml b/moquette-0.17/checkstyle.xml similarity index 100% rename from moquette-0.16/checkstyle.xml rename to moquette-0.17/checkstyle.xml diff --git a/moquette-0.16/distribution/pom.xml b/moquette-0.17/distribution/pom.xml similarity index 98% rename from moquette-0.16/distribution/pom.xml rename to moquette-0.17/distribution/pom.xml index e86792e1..a2f7ae3e 100644 --- a/moquette-0.16/distribution/pom.xml +++ b/moquette-0.17/distribution/pom.xml @@ -5,7 +5,7 @@ ../pom.xml moquette-parent io.moquette - 0.16-gg + 0.17-gg distribution diff --git a/moquette-0.16/distribution/src/main/assembly/assembly.xml b/moquette-0.17/distribution/src/main/assembly/assembly.xml similarity index 100% rename from moquette-0.16/distribution/src/main/assembly/assembly.xml rename to moquette-0.17/distribution/src/main/assembly/assembly.xml diff --git a/moquette-0.16/distribution/src/main/resources/README.txt b/moquette-0.17/distribution/src/main/resources/README.txt similarity index 100% rename from moquette-0.16/distribution/src/main/resources/README.txt rename to moquette-0.17/distribution/src/main/resources/README.txt diff --git a/moquette-0.16/distribution/src/main/resources/acl.conf b/moquette-0.17/distribution/src/main/resources/acl.conf similarity index 100% rename from moquette-0.16/distribution/src/main/resources/acl.conf rename to moquette-0.17/distribution/src/main/resources/acl.conf diff --git a/moquette-0.16/distribution/src/main/resources/log4j.properties b/moquette-0.17/distribution/src/main/resources/log4j.properties similarity index 100% rename from moquette-0.16/distribution/src/main/resources/log4j.properties rename to moquette-0.17/distribution/src/main/resources/log4j.properties diff --git a/moquette-0.16/distribution/src/main/resources/moquette.conf b/moquette-0.17/distribution/src/main/resources/moquette.conf similarity index 78% rename from moquette-0.16/distribution/src/main/resources/moquette.conf rename to moquette-0.17/distribution/src/main/resources/moquette.conf index d32de195..dc873211 100644 --- a/moquette-0.16/distribution/src/main/resources/moquette.conf +++ b/moquette-0.17/distribution/src/main/resources/moquette.conf @@ -11,7 +11,7 @@ websocket_port 8080 #********************************************************************* # Secure Websocket port (wss) -# decommend this to enable wss +# decomment this to enable wss #********************************************************************* # secure_websocket_port 8883 @@ -49,10 +49,24 @@ websocket_port 8080 host 0.0.0.0 #********************************************************************* -# The file for the persistent store, if not specified, use just memory -# no physical persistence +# If enabled store queues and subscription data into data_path #********************************************************************* -#persistent_store ./moquette_store.h2 +# persistence_enabled true + +#********************************************************************* +# The path to store queues and subscriptions data. Used if +# persistence_enabled is enabled. +#********************************************************************* +# data_path data/ + +#********************************************************************* +# Persistent queues type +# +# persistent_queue_type: +# "h2" or "segmented" +# default: h2 +#********************************************************************* +# persistent_queue_type segmented #********************************************************************* # acl_file: @@ -172,3 +186,28 @@ password_file config/password_file.conf # default: true #********************************************************************* # telemetry_enabled true + +#********************************************************************* +# Flush interval between writes +# +# buffer_flush_millis: +# `immediate` or `full` or number. `immediate` forces the flush on +# every socket write while `full` let the underlying system to flush +# when full. If its defined a number it's used a milliseconds between flushes. +# default: immediate +#********************************************************************* +# buffer_flush_millis immediate + +#********************************************************************* +# Duration after which expire persisted sessions. +# +# persistent_client_expiration: +# This option allows the session of persistent clients (those with clean session set to false) that are not +# currently connected to be removed if they do not reconnect within a certain time frame. +# This is a non-standard option in MQTT v3.1. MQTT v3.1.1 and v5.0 allow brokers to remove client sessions. +# The expiration period should be an integer followed by one of s m h d w M y for seconds, minutes, hours, days, weeks, +# months and years respectively. For example: 2m or 14d or 1y +# default: infinite expiry +#********************************************************************* +# persistent_client_expiration 3d + diff --git a/moquette-0.16/distribution/src/main/resources/password_file.conf b/moquette-0.17/distribution/src/main/resources/password_file.conf similarity index 100% rename from moquette-0.16/distribution/src/main/resources/password_file.conf rename to moquette-0.17/distribution/src/main/resources/password_file.conf diff --git a/moquette-0.16/distribution/src/main/scripts/moquette.bat b/moquette-0.17/distribution/src/main/scripts/moquette.bat similarity index 96% rename from moquette-0.16/distribution/src/main/scripts/moquette.bat rename to moquette-0.17/distribution/src/main/scripts/moquette.bat index 1bc89b16..d7289869 100644 --- a/moquette-0.16/distribution/src/main/scripts/moquette.bat +++ b/moquette-0.17/distribution/src/main/scripts/moquette.bat @@ -1,6 +1,6 @@ @ECHO OFF rem # -rem # Copyright (c) 2012-2015 Andrea Selva +rem # Copyright (c) 2012-2023 Andrea Selva rem # echo " " @@ -13,7 +13,7 @@ echo " \_| |_/\___/ \__, |\__,_|\___|\__|\__\___| \_| |_/\_/\_\ \_/ \_/ " echo " | | " echo " |_| " echo " " -echo " version: 0.13-SNAPSHOT " +echo " version: 0.17 " set "CURRENT_DIR=%cd%" if not "%MOQUETTE_HOME%" == "" goto gotHome diff --git a/moquette-0.16/distribution/src/main/scripts/moquette.sh b/moquette-0.17/distribution/src/main/scripts/moquette.sh similarity index 75% rename from moquette-0.16/distribution/src/main/scripts/moquette.sh rename to moquette-0.17/distribution/src/main/scripts/moquette.sh index 0f06c1ba..acf2d958 100644 --- a/moquette-0.16/distribution/src/main/scripts/moquette.sh +++ b/moquette-0.17/distribution/src/main/scripts/moquette.sh @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (c) 2012-2015 Andrea Selva +# Copyright (c) 2012-2023 Andrea Selva # echo " " @@ -13,30 +13,36 @@ echo " \_| |_/\___/ \__, |\__,_|\___|\__|\__\___| \_| |_/\_/\_\ \_/ \_/ " echo " | | " echo " |_| " echo " " -echo " version: 0.16 " - - -cd "$(dirname "$0")" - -# resolve links - $0 may be a softlink -PRG="$0" - -while [ -h "$PRG" ]; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" +echo " version: 0.17 " + + +unset CDPATH +# This unwieldy bit of scripting is to try to catch instances where Moquette +# was launched from a symlink, rather than a full path to the Moquette binary +if [ -L "$0" ]; then + # Launched from a symlink + # --Test for the readlink binary + RL="$(command -v readlink)" + if [ $? -eq 0 ]; then + # readlink exists + SOURCEPATH="$($RL $0)" else - PRG=`dirname "$PRG"`/"$link" + # readlink not found, attempt to parse the output of stat + SOURCEPATH="$(stat -c %N $0 | awk '{print $3}' | sed -e 's/\‘//' -e 's/\’//')" + if [ $? -ne 0 ]; then + # Failed to execute or parse stat + echo "You may need to launch Moquette with a full path instead of a symlink." + exit 1 + fi fi -done - -# Get standard environment variables -PRGDIR=`dirname "$PRG"` +else + # Not a symlink + SOURCEPATH="$0" +fi -# Only set MOQUETTE_HOME if not already set -[ -f "$MOQUETTE_HOME"/bin/moquette.sh ] || MOQUETTE_HOME=`cd "$PRGDIR/.." ; pwd` +MOQUETTE_HOME="$(cd `dirname $SOURCEPATH`/..; pwd)" export MOQUETTE_HOME +MOQUETTE_JARS=${MOQUETTE_HOME}/lib/* # Set JavaHome if it exists if [ -f "${JAVA_HOME}/bin/java" ]; then @@ -94,5 +100,4 @@ JAVA_OPTS="$JAVA_OPTS -Xloggc:$MOQUETTE_HOME/gc.log" #JAVA_OPTS="$JAVA_OPTS -XX:NumberOfGCLogFiles=10" #JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M" -echo '$JAVA -server $JAVA_OPTS $JAVA_OPTS_SCRIPT -Dlog4j.configuration="file:$LOG_FILE" -Dmoquette.path="$MOQUETTE_PATH" -cp "$MOQUETTE_HOME/lib/*" io.moquette.broker.Server' -$JAVA -server $JAVA_OPTS $JAVA_OPTS_SCRIPT -Dlog4j.configuration="file:$LOG_FILE" -Dmoquette.path="$MOQUETTE_PATH" -cp "$MOQUETTE_HOME/lib/*" io.moquette.broker.Server +$JAVA $JAVA_OPTS $JAVA_OPTS_SCRIPT -Dlog4j.configuration="file:$LOG_FILE" -Dmoquette.path="$MOQUETTE_HOME" -cp "$MOQUETTE_HOME/lib/*" io.moquette.broker.Server diff --git a/moquette-0.16/embedding_moquette/pom.xml b/moquette-0.17/embedding_moquette/pom.xml similarity index 95% rename from moquette-0.16/embedding_moquette/pom.xml rename to moquette-0.17/embedding_moquette/pom.xml index 8bdb3139..ce2d87c6 100644 --- a/moquette-0.16/embedding_moquette/pom.xml +++ b/moquette-0.17/embedding_moquette/pom.xml @@ -5,11 +5,11 @@ ../pom.xml moquette-parent io.moquette - 0.16-gg + 0.17-gg embedded_test - pom + jar Moquette - Embedded test diff --git a/moquette-0.16/embedding_moquette/src/main/java/io/moquette/testembedded/EmbeddedLauncher.java b/moquette-0.17/embedding_moquette/src/main/java/io/moquette/testembedded/EmbeddedLauncher.java similarity index 95% rename from moquette-0.16/embedding_moquette/src/main/java/io/moquette/testembedded/EmbeddedLauncher.java rename to moquette-0.17/embedding_moquette/src/main/java/io/moquette/testembedded/EmbeddedLauncher.java index 8dcf98f2..0c0fb629 100644 --- a/moquette-0.16/embedding_moquette/src/main/java/io/moquette/testembedded/EmbeddedLauncher.java +++ b/moquette-0.17/embedding_moquette/src/main/java/io/moquette/testembedded/EmbeddedLauncher.java @@ -51,6 +51,11 @@ public void onPublish(InterceptPublishMessage msg) { final String decodedPayload = msg.getPayload().toString(UTF_8); System.out.println("Received on topic: " + msg.getTopicName() + " content: " + decodedPayload); } + + @Override + public void onSessionLoopError(Throwable error) { + System.out.println("Session event loop reported error: " + error); + } } public static void main(String[] args) throws InterruptedException, IOException { diff --git a/moquette-0.16/embedding_moquette/src/main/resources/config/moquette.conf b/moquette-0.17/embedding_moquette/src/main/resources/config/moquette.conf similarity index 100% rename from moquette-0.16/embedding_moquette/src/main/resources/config/moquette.conf rename to moquette-0.17/embedding_moquette/src/main/resources/config/moquette.conf diff --git a/moquette-0.16/license-eplv10-aslv20.html b/moquette-0.17/license-eplv10-aslv20.html similarity index 100% rename from moquette-0.16/license-eplv10-aslv20.html rename to moquette-0.17/license-eplv10-aslv20.html diff --git a/moquette-0.16/license.txt b/moquette-0.17/license.txt similarity index 100% rename from moquette-0.16/license.txt rename to moquette-0.17/license.txt diff --git a/moquette-0.16/mvnw b/moquette-0.17/mvnw similarity index 100% rename from moquette-0.16/mvnw rename to moquette-0.17/mvnw diff --git a/moquette-0.16/mvnw.cmd b/moquette-0.17/mvnw.cmd similarity index 100% rename from moquette-0.16/mvnw.cmd rename to moquette-0.17/mvnw.cmd diff --git a/moquette-0.16/pom.xml b/moquette-0.17/pom.xml similarity index 99% rename from moquette-0.16/pom.xml rename to moquette-0.17/pom.xml index 2c203c79..307301c0 100644 --- a/moquette-0.16/pom.xml +++ b/moquette-0.17/pom.xml @@ -16,7 +16,7 @@ moquette-parent pom - 0.16-gg + 0.17-gg Moquette MQTT Moquette lightweight MQTT Broker 2011 diff --git a/moquette-0.16/tools_scripts/benchmark/benchmarkReportGenerator.groovy b/moquette-0.17/tools_scripts/benchmark/benchmarkReportGenerator.groovy similarity index 100% rename from moquette-0.16/tools_scripts/benchmark/benchmarkReportGenerator.groovy rename to moquette-0.17/tools_scripts/benchmark/benchmarkReportGenerator.groovy diff --git a/moquette-0.16/tools_scripts/benchmark/benchmarker.groovy b/moquette-0.17/tools_scripts/benchmark/benchmarker.groovy similarity index 100% rename from moquette-0.16/tools_scripts/benchmark/benchmarker.groovy rename to moquette-0.17/tools_scripts/benchmark/benchmarker.groovy diff --git a/moquette-0.16/tools_scripts/benchmark/conSubPubUnsub_issue204.groovy b/moquette-0.17/tools_scripts/benchmark/conSubPubUnsub_issue204.groovy similarity index 100% rename from moquette-0.16/tools_scripts/benchmark/conSubPubUnsub_issue204.groovy rename to moquette-0.17/tools_scripts/benchmark/conSubPubUnsub_issue204.groovy diff --git a/moquette-0.16/tools_scripts/benchmark/connectionTimings.groovy b/moquette-0.17/tools_scripts/benchmark/connectionTimings.groovy similarity index 100% rename from moquette-0.16/tools_scripts/benchmark/connectionTimings.groovy rename to moquette-0.17/tools_scripts/benchmark/connectionTimings.groovy diff --git a/moquette-0.16/tools_scripts/benchmark/fuse_asynch_publisher.groovy b/moquette-0.17/tools_scripts/benchmark/fuse_asynch_publisher.groovy similarity index 100% rename from moquette-0.16/tools_scripts/benchmark/fuse_asynch_publisher.groovy rename to moquette-0.17/tools_scripts/benchmark/fuse_asynch_publisher.groovy diff --git a/moquette-0.16/tools_scripts/benchmark/fuse_asynch_subscriber.groovy b/moquette-0.17/tools_scripts/benchmark/fuse_asynch_subscriber.groovy similarity index 100% rename from moquette-0.16/tools_scripts/benchmark/fuse_asynch_subscriber.groovy rename to moquette-0.17/tools_scripts/benchmark/fuse_asynch_subscriber.groovy diff --git a/moquette-0.16/tools_scripts/benchmark/fuse_benchmarker/BenchmarkPublisher.groovy b/moquette-0.17/tools_scripts/benchmark/fuse_benchmarker/BenchmarkPublisher.groovy similarity index 100% rename from moquette-0.16/tools_scripts/benchmark/fuse_benchmarker/BenchmarkPublisher.groovy rename to moquette-0.17/tools_scripts/benchmark/fuse_benchmarker/BenchmarkPublisher.groovy diff --git a/moquette-0.16/tools_scripts/benchmark/fuse_benchmarker/BenchmarkSubscriber.groovy b/moquette-0.17/tools_scripts/benchmark/fuse_benchmarker/BenchmarkSubscriber.groovy similarity index 100% rename from moquette-0.16/tools_scripts/benchmark/fuse_benchmarker/BenchmarkSubscriber.groovy rename to moquette-0.17/tools_scripts/benchmark/fuse_benchmarker/BenchmarkSubscriber.groovy diff --git a/moquette-0.16/tools_scripts/benchmark/fuse_benchmarker/fuse_asynch_benchmarker.groovy b/moquette-0.17/tools_scripts/benchmark/fuse_benchmarker/fuse_asynch_benchmarker.groovy similarity index 100% rename from moquette-0.16/tools_scripts/benchmark/fuse_benchmarker/fuse_asynch_benchmarker.groovy rename to moquette-0.17/tools_scripts/benchmark/fuse_benchmarker/fuse_asynch_benchmarker.groovy diff --git a/moquette-0.16/tools_scripts/benchmark/fuse_publisher.groovy b/moquette-0.17/tools_scripts/benchmark/fuse_publisher.groovy similarity index 100% rename from moquette-0.16/tools_scripts/benchmark/fuse_publisher.groovy rename to moquette-0.17/tools_scripts/benchmark/fuse_publisher.groovy diff --git a/moquette-0.16/tools_scripts/benchmark/fuse_subscriber.groovy b/moquette-0.17/tools_scripts/benchmark/fuse_subscriber.groovy similarity index 100% rename from moquette-0.16/tools_scripts/benchmark/fuse_subscriber.groovy rename to moquette-0.17/tools_scripts/benchmark/fuse_subscriber.groovy diff --git a/moquette-0.16/tools_scripts/benchmark/paho_benchmaker/BenchmarkPublisher.groovy b/moquette-0.17/tools_scripts/benchmark/paho_benchmaker/BenchmarkPublisher.groovy similarity index 100% rename from moquette-0.16/tools_scripts/benchmark/paho_benchmaker/BenchmarkPublisher.groovy rename to moquette-0.17/tools_scripts/benchmark/paho_benchmaker/BenchmarkPublisher.groovy diff --git a/moquette-0.16/tools_scripts/benchmark/paho_benchmaker/BenchmarkSubscriber.groovy b/moquette-0.17/tools_scripts/benchmark/paho_benchmaker/BenchmarkSubscriber.groovy similarity index 100% rename from moquette-0.16/tools_scripts/benchmark/paho_benchmaker/BenchmarkSubscriber.groovy rename to moquette-0.17/tools_scripts/benchmark/paho_benchmaker/BenchmarkSubscriber.groovy diff --git a/moquette-0.16/tools_scripts/benchmark/paho_benchmaker/benchmarker.groovy b/moquette-0.17/tools_scripts/benchmark/paho_benchmaker/benchmarker.groovy similarity index 100% rename from moquette-0.16/tools_scripts/benchmark/paho_benchmaker/benchmarker.groovy rename to moquette-0.17/tools_scripts/benchmark/paho_benchmaker/benchmarker.groovy diff --git a/moquette-0.16/tools_scripts/benchmark/publisher_client.groovy b/moquette-0.17/tools_scripts/benchmark/publisher_client.groovy similarity index 100% rename from moquette-0.16/tools_scripts/benchmark/publisher_client.groovy rename to moquette-0.17/tools_scripts/benchmark/publisher_client.groovy diff --git a/moquette-0.16/tools_scripts/benchmark/subscriber_client.groovy b/moquette-0.17/tools_scripts/benchmark/subscriber_client.groovy similarity index 100% rename from moquette-0.16/tools_scripts/benchmark/subscriber_client.groovy rename to moquette-0.17/tools_scripts/benchmark/subscriber_client.groovy diff --git a/moquette-0.16/tools_scripts/integration/allAroundWalk.groovy b/moquette-0.17/tools_scripts/integration/allAroundWalk.groovy similarity index 100% rename from moquette-0.16/tools_scripts/integration/allAroundWalk.groovy rename to moquette-0.17/tools_scripts/integration/allAroundWalk.groovy diff --git a/moquette-0.16/tools_scripts/integration/brokerEmbeddedTest.groovy b/moquette-0.17/tools_scripts/integration/brokerEmbeddedTest.groovy similarity index 100% rename from moquette-0.16/tools_scripts/integration/brokerEmbeddedTest.groovy rename to moquette-0.17/tools_scripts/integration/brokerEmbeddedTest.groovy diff --git a/moquette-0.16/tools_scripts/integration/bunch_publish_issue12.groovy b/moquette-0.17/tools_scripts/integration/bunch_publish_issue12.groovy similarity index 100% rename from moquette-0.16/tools_scripts/integration/bunch_publish_issue12.groovy rename to moquette-0.17/tools_scripts/integration/bunch_publish_issue12.groovy diff --git a/moquette-0.16/tools_scripts/integration/index.html b/moquette-0.17/tools_scripts/integration/index.html similarity index 100% rename from moquette-0.16/tools_scripts/integration/index.html rename to moquette-0.17/tools_scripts/integration/index.html diff --git a/moquette-0.16/tools_scripts/integration/issue10_acl_file_test.groovy b/moquette-0.17/tools_scripts/integration/issue10_acl_file_test.groovy similarity index 100% rename from moquette-0.16/tools_scripts/integration/issue10_acl_file_test.groovy rename to moquette-0.17/tools_scripts/integration/issue10_acl_file_test.groovy diff --git a/moquette-0.16/tools_scripts/integration/issue15_receiver.groovy b/moquette-0.17/tools_scripts/integration/issue15_receiver.groovy similarity index 100% rename from moquette-0.16/tools_scripts/integration/issue15_receiver.groovy rename to moquette-0.17/tools_scripts/integration/issue15_receiver.groovy diff --git a/moquette-0.16/tools_scripts/integration/issue15_sender.groovy b/moquette-0.17/tools_scripts/integration/issue15_sender.groovy similarity index 100% rename from moquette-0.16/tools_scripts/integration/issue15_sender.groovy rename to moquette-0.17/tools_scripts/integration/issue15_sender.groovy diff --git a/moquette-0.16/tools_scripts/integration/issue16_will_publish.groovy b/moquette-0.17/tools_scripts/integration/issue16_will_publish.groovy similarity index 100% rename from moquette-0.16/tools_scripts/integration/issue16_will_publish.groovy rename to moquette-0.17/tools_scripts/integration/issue16_will_publish.groovy diff --git a/moquette-0.16/tools_scripts/integration/issue38_pubsubQos2.groovy b/moquette-0.17/tools_scripts/integration/issue38_pubsubQos2.groovy similarity index 100% rename from moquette-0.16/tools_scripts/integration/issue38_pubsubQos2.groovy rename to moquette-0.17/tools_scripts/integration/issue38_pubsubQos2.groovy diff --git a/moquette-0.16/tools_scripts/integration/issue39_loopingConnecter.groovy b/moquette-0.17/tools_scripts/integration/issue39_loopingConnecter.groovy similarity index 100% rename from moquette-0.16/tools_scripts/integration/issue39_loopingConnecter.groovy rename to moquette-0.17/tools_scripts/integration/issue39_loopingConnecter.groovy diff --git a/moquette-0.16/tools_scripts/integration/issue410UnsubscribeAckFormat.groovy b/moquette-0.17/tools_scripts/integration/issue410UnsubscribeAckFormat.groovy similarity index 100% rename from moquette-0.16/tools_scripts/integration/issue410UnsubscribeAckFormat.groovy rename to moquette-0.17/tools_scripts/integration/issue410UnsubscribeAckFormat.groovy diff --git a/moquette-0.16/tools_scripts/integration/issue42_publishBigMessage.groovy b/moquette-0.17/tools_scripts/integration/issue42_publishBigMessage.groovy similarity index 100% rename from moquette-0.16/tools_scripts/integration/issue42_publishBigMessage.groovy rename to moquette-0.17/tools_scripts/integration/issue42_publishBigMessage.groovy diff --git a/moquette-0.16/tools_scripts/integration/issue43_badFormattedTopicSub.groovy b/moquette-0.17/tools_scripts/integration/issue43_badFormattedTopicSub.groovy similarity index 100% rename from moquette-0.16/tools_scripts/integration/issue43_badFormattedTopicSub.groovy rename to moquette-0.17/tools_scripts/integration/issue43_badFormattedTopicSub.groovy diff --git a/moquette-0.16/tools_scripts/integration/issue4_publish2Subscribers.groovy b/moquette-0.17/tools_scripts/integration/issue4_publish2Subscribers.groovy similarity index 100% rename from moquette-0.16/tools_scripts/integration/issue4_publish2Subscribers.groovy rename to moquette-0.17/tools_scripts/integration/issue4_publish2Subscribers.groovy diff --git a/moquette-0.16/tools_scripts/integration/issue58_overlappingSubscription.groovy b/moquette-0.17/tools_scripts/integration/issue58_overlappingSubscription.groovy similarity index 100% rename from moquette-0.16/tools_scripts/integration/issue58_overlappingSubscription.groovy rename to moquette-0.17/tools_scripts/integration/issue58_overlappingSubscription.groovy diff --git a/moquette-0.16/tools_scripts/integration/issue79_stress_multipub.groovy b/moquette-0.17/tools_scripts/integration/issue79_stress_multipub.groovy similarity index 100% rename from moquette-0.16/tools_scripts/integration/issue79_stress_multipub.groovy rename to moquette-0.17/tools_scripts/integration/issue79_stress_multipub.groovy diff --git a/moquette-0.16/tools_scripts/integration/issue79_stress_multisub.groovy b/moquette-0.17/tools_scripts/integration/issue79_stress_multisub.groovy similarity index 100% rename from moquette-0.16/tools_scripts/integration/issue79_stress_multisub.groovy rename to moquette-0.17/tools_scripts/integration/issue79_stress_multisub.groovy diff --git a/moquette-0.16/tools_scripts/integration/issue88_reconnect_after_bad_credentials.groovy b/moquette-0.17/tools_scripts/integration/issue88_reconnect_after_bad_credentials.groovy similarity index 100% rename from moquette-0.16/tools_scripts/integration/issue88_reconnect_after_bad_credentials.groovy rename to moquette-0.17/tools_scripts/integration/issue88_reconnect_after_bad_credentials.groovy diff --git a/moquette-0.16/tools_scripts/integration/issue_12_multiple_republish_qos2.groovy b/moquette-0.17/tools_scripts/integration/issue_12_multiple_republish_qos2.groovy similarity index 100% rename from moquette-0.16/tools_scripts/integration/issue_12_multiple_republish_qos2.groovy rename to moquette-0.17/tools_scripts/integration/issue_12_multiple_republish_qos2.groovy diff --git a/moquette-0.16/tools_scripts/integration/list-broker-topics.rb b/moquette-0.17/tools_scripts/integration/list-broker-topics.rb similarity index 100% rename from moquette-0.16/tools_scripts/integration/list-broker-topics.rb rename to moquette-0.17/tools_scripts/integration/list-broker-topics.rb diff --git a/moquette-0.16/tools_scripts/integration/pahoConnect.groovy b/moquette-0.17/tools_scripts/integration/pahoConnect.groovy similarity index 100% rename from moquette-0.16/tools_scripts/integration/pahoConnect.groovy rename to moquette-0.17/tools_scripts/integration/pahoConnect.groovy diff --git a/moquette-0.16/tools_scripts/integration/paho_test_issue8.groovy b/moquette-0.17/tools_scripts/integration/paho_test_issue8.groovy similarity index 100% rename from moquette-0.16/tools_scripts/integration/paho_test_issue8.groovy rename to moquette-0.17/tools_scripts/integration/paho_test_issue8.groovy diff --git a/moquette-0.16/tools_scripts/integration/sslSimplePublisher.groovy b/moquette-0.17/tools_scripts/integration/sslSimplePublisher.groovy similarity index 100% rename from moquette-0.16/tools_scripts/integration/sslSimplePublisher.groovy rename to moquette-0.17/tools_scripts/integration/sslSimplePublisher.groovy diff --git a/moquette-0.16/tools_scripts/integration/twoClientsWithSameID.groovy b/moquette-0.17/tools_scripts/integration/twoClientsWithSameID.groovy similarity index 100% rename from moquette-0.16/tools_scripts/integration/twoClientsWithSameID.groovy rename to moquette-0.17/tools_scripts/integration/twoClientsWithSameID.groovy diff --git a/moquette-0.16/tools_scripts/integration/webSocketClient.groovy b/moquette-0.17/tools_scripts/integration/webSocketClient.groovy similarity index 100% rename from moquette-0.16/tools_scripts/integration/webSocketClient.groovy rename to moquette-0.17/tools_scripts/integration/webSocketClient.groovy diff --git a/moquette-0.16/tools_scripts/integration/web_socket_test.html b/moquette-0.17/tools_scripts/integration/web_socket_test.html similarity index 100% rename from moquette-0.16/tools_scripts/integration/web_socket_test.html rename to moquette-0.17/tools_scripts/integration/web_socket_test.html diff --git a/moquette-0.16/tools_scripts/lab/h2MVStoreTest.groovy b/moquette-0.17/tools_scripts/lab/h2MVStoreTest.groovy similarity index 100% rename from moquette-0.16/tools_scripts/lab/h2MVStoreTest.groovy rename to moquette-0.17/tools_scripts/lab/h2MVStoreTest.groovy diff --git a/moquette-0.16/tools_scripts/lab/mapDbTest.groovy b/moquette-0.17/tools_scripts/lab/mapDbTest.groovy similarity index 100% rename from moquette-0.16/tools_scripts/lab/mapDbTest.groovy rename to moquette-0.17/tools_scripts/lab/mapDbTest.groovy diff --git a/moquette-0.16/tools_scripts/lab/squareTapeTest.groovy b/moquette-0.17/tools_scripts/lab/squareTapeTest.groovy similarity index 100% rename from moquette-0.16/tools_scripts/lab/squareTapeTest.groovy rename to moquette-0.17/tools_scripts/lab/squareTapeTest.groovy diff --git a/moquette-0.16/tools_scripts/launghLocalJetty.groovy b/moquette-0.17/tools_scripts/launghLocalJetty.groovy similarity index 100% rename from moquette-0.16/tools_scripts/launghLocalJetty.groovy rename to moquette-0.17/tools_scripts/launghLocalJetty.groovy diff --git a/moquette-0.16/tools_scripts/mapdbWalTest.groovy b/moquette-0.17/tools_scripts/mapdbWalTest.groovy similarity index 100% rename from moquette-0.16/tools_scripts/mapdbWalTest.groovy rename to moquette-0.17/tools_scripts/mapdbWalTest.groovy diff --git a/moquette-0.16/tools_scripts/perf_tmp.csv b/moquette-0.17/tools_scripts/perf_tmp.csv similarity index 100% rename from moquette-0.16/tools_scripts/perf_tmp.csv rename to moquette-0.17/tools_scripts/perf_tmp.csv diff --git a/moquette-0.16/tools_scripts/publisher_QoS2.groovy b/moquette-0.17/tools_scripts/publisher_QoS2.groovy similarity index 100% rename from moquette-0.16/tools_scripts/publisher_QoS2.groovy rename to moquette-0.17/tools_scripts/publisher_QoS2.groovy diff --git a/moquette-0.16/tools_scripts/subscriber_generic.groovy b/moquette-0.17/tools_scripts/subscriber_generic.groovy similarity index 100% rename from moquette-0.16/tools_scripts/subscriber_generic.groovy rename to moquette-0.17/tools_scripts/subscriber_generic.groovy diff --git a/moquette-0.16/tools_scripts/trivialPublish.groovy b/moquette-0.17/tools_scripts/trivialPublish.groovy similarity index 100% rename from moquette-0.16/tools_scripts/trivialPublish.groovy rename to moquette-0.17/tools_scripts/trivialPublish.groovy diff --git a/pom.xml b/pom.xml index c978c154..b9545035 100644 --- a/pom.xml +++ b/pom.xml @@ -15,7 +15,7 @@ - moquette-0.16 + moquette-0.17 integration