diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index 41e649b64cfd1..29b966d88c9eb 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -29,6 +29,10 @@ workflows: run_linux_tests: << pipeline.parameters.run_linux_tests >> requires: - linux-build-and-unit-test + - linux-presto-native-tests: + run_linux_tests: << pipeline.parameters.run_linux_tests >> + requires: + - linux-build-and-unit-test - linux-spark-e2e-tests: run_linux_tests: << pipeline.parameters.run_linux_tests >> requires: @@ -198,6 +202,54 @@ jobs: - store_artifacts: path: '/tmp/PrestoNativeQueryRunnerUtils' + linux-presto-native-tests: + executor: build + parameters: + run_linux_tests: + type: boolean + default: false + parallelism: 5 + steps: + - run: echo "Run Linux tests is << parameters.run_linux_tests >>" + - when: + condition: << parameters.run_linux_tests >> + steps: + - checkout + - attach_workspace: + at: presto-native-execution + - maven_install: + maven_install_opts: ${MAVEN_INSTALL_OPTS} + maven_fast_install: ${MAVEN_FAST_INSTALL} + - run: + name: 'Run Presto native tests' + command: | + export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib:/usr/local/lib64" + export PRESTO_SERVER_PATH="${HOME}/project/presto-native-execution/_build/debug/presto_cpp/main/presto_server" + export TEMP_PATH="/tmp" + TESTFILES=$(circleci tests glob "presto-native-tests/src/test/**/Test*.java" | circleci tests split --split-by=timings) + # Convert file paths to comma separated class names + export TESTCLASSES= + for test_file in $TESTFILES + do + tmp=${test_file##*/} + test_class=${tmp%%\.*} + export TESTCLASSES="${TESTCLASSES},$test_class" + done + export TESTCLASSES=${TESTCLASSES#,} + + if [ ! -z $TESTCLASSES ]; then + mvn test \ + ${MAVEN_TEST} \ + -pl 'presto-native-tests' \ + -Dtest="${TESTCLASSES}" \ + -DPRESTO_SERVER=${PRESTO_SERVER_PATH} \ + -DDATA_DIR=${TEMP_PATH} \ + -Duser.timezone=America/Bahia_Banderas \ + -T1C + fi + - store_artifacts: + path: '/tmp/PrestoNativeQueryRunnerUtils' + linux-presto-native-sidecar-tests: executor: build parameters: diff --git a/.github/workflows/prestocpp-linux-build-and-unit-test.yml b/.github/workflows/prestocpp-linux-build-and-unit-test.yml index a3770fb1a8df5..73cb2124ff8e1 100644 --- a/.github/workflows/prestocpp-linux-build-and-unit-test.yml +++ b/.github/workflows/prestocpp-linux-build-and-unit-test.yml @@ -169,6 +169,88 @@ jobs: -Duser.timezone=America/Bahia_Banderas \ -T1C + + prestocpp-linux-presto-native-tests: + needs: prestocpp-linux-build-for-test + runs-on: ubuntu-22.04 + container: + image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090 + env: + MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError" + MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true" + MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end" + steps: + - uses: actions/checkout@v4 + + - name: Fix git permissions + # Usually actions/checkout does this but as we run in a container + # it doesn't work + run: git config --global --add safe.directory ${GITHUB_WORKSPACE} + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: presto-native-build + path: presto-native-execution/_build/release + + # Permissions are lost when uploading. Details here: https://github.com/actions/upload-artifact/issues/38 + - name: Restore execute permissions and library path + run: | + chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server + chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main + # Ensure transitive dependency libboost-iostreams is found. + ldconfig /usr/local/lib + + - name: Install OpenJDK8 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '8' + + - name: Cache local Maven repository + id: cache-maven + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven-2- + + - name: Populate maven cache + if: steps.cache-maven.outputs.cache-hit != 'true' + run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies + + - name: Maven install + env: + # Use different Maven options to install. + MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" + run: | + for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-tests' -am && s=0 && break || s=$? && sleep 10; done; (exit $s) + + - name: Run presto-native tests + run: | + export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server" + export TESTFILES=`find ./presto-native-tests/src/test -type f -name 'Test*.java'` + # Convert file paths to comma separated class names + export TESTCLASSES= + for test_file in $TESTFILES + do + tmp=${test_file##*/} + test_class=${tmp%%\.*} + export TESTCLASSES="${TESTCLASSES},$test_class" + done + export TESTCLASSES=${TESTCLASSES#,} + echo "TESTCLASSES = $TESTCLASSES" + + mvn test \ + ${MAVEN_TEST} \ + -pl 'presto-native-tests' \ + -Dtest="${TESTCLASSES}" \ + -DPRESTO_SERVER=${PRESTO_SERVER_PATH} \ + -DDATA_DIR=${RUNNER_TEMP} \ + -Duser.timezone=America/Bahia_Banderas \ + -T1C + prestocpp-linux-spark-e2e-tests: needs: prestocpp-linux-build-for-test runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-other-modules.yml b/.github/workflows/test-other-modules.yml index 3c3f84817a0d1..b2fec9efd4cb0 100644 --- a/.github/workflows/test-other-modules.yml +++ b/.github/workflows/test-other-modules.yml @@ -64,6 +64,7 @@ jobs: run: | ./mvnw test -T 1 ${MAVEN_TEST} -pl ' !presto-tests, + !presto-native-tests, !presto-accumulo, !presto-cassandra, !presto-hive, diff --git a/pom.xml b/pom.xml index ee4920c084624..43e64c51259e9 100644 --- a/pom.xml +++ b/pom.xml @@ -197,6 +197,7 @@ presto-test-coverage presto-hudi presto-native-execution + presto-native-tests presto-router presto-open-telemetry redis-hbo-provider diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java index 58431a64f7ca7..1cd8a73102a22 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java @@ -43,7 +43,9 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.OptionalInt; import java.util.UUID; @@ -121,7 +123,7 @@ public static QueryRunner createQueryRunner( defaultQueryRunner.close(); - return createNativeQueryRunner(dataDirectory.get().toString(), prestoServerPath.get(), workerCount, cacheMaxSize, true, Optional.empty(), storageFormat, addStorageFormatToPath, false, isCoordinatorSidecarEnabled, false); + return createNativeQueryRunner(dataDirectory.get().toString(), prestoServerPath.get(), workerCount, cacheMaxSize, true, Optional.empty(), storageFormat, addStorageFormatToPath, false, isCoordinatorSidecarEnabled, false, Collections.emptyMap(), Collections.emptyMap()); } public static QueryRunner createJavaQueryRunner() @@ -326,7 +328,9 @@ public static QueryRunner createNativeQueryRunner( boolean addStorageFormatToPath, Boolean failOnNestedLoopJoin, boolean isCoordinatorSidecarEnabled, - boolean singleNodeExecutionEnabled) + boolean singleNodeExecutionEnabled, + Map extraProperties, + Map extraCoordinatorProperties) throws Exception { // The property "hive.allow-drop-table" needs to be set to true because security is always "legacy" in NativeQueryRunner. @@ -350,8 +354,9 @@ public static QueryRunner createNativeQueryRunner( .put("experimental.internal-communication.thrift-transport-enabled", String.valueOf(useThrift)) .putAll(getNativeWorkerSystemProperties()) .putAll(isCoordinatorSidecarEnabled ? getNativeSidecarProperties() : ImmutableMap.of()) + .putAll(extraProperties) .build(), - coordinatorProperties.build(), + coordinatorProperties.putAll(extraCoordinatorProperties).build(), "legacy", hiveProperties, workerCount, @@ -408,6 +413,28 @@ public static QueryRunner createNativeQueryRunner(String remoteFunctionServerUds return createNativeQueryRunner(false, DEFAULT_STORAGE_FORMAT, Optional.ofNullable(remoteFunctionServerUds), false, false, false); } + public static QueryRunner createNativeQueryRunner(Map extraProperties, Map extraCoordinatorProperties, + String storageFormat) + throws Exception + { + int cacheMaxSize = 0; + NativeQueryRunnerParameters nativeQueryRunnerParameters = getNativeQueryRunnerParameters(); + return createNativeQueryRunner( + nativeQueryRunnerParameters.dataDirectory.toString(), + nativeQueryRunnerParameters.serverBinary.toString(), + nativeQueryRunnerParameters.workerCount, + cacheMaxSize, + true, + Optional.empty(), + storageFormat, + true, + false, + false, + false, + extraProperties, + extraCoordinatorProperties); + } + public static QueryRunner createNativeQueryRunner(boolean useThrift) throws Exception { @@ -448,7 +475,9 @@ public static QueryRunner createNativeQueryRunner( true, failOnNestedLoopJoin, isCoordinatorSidecarEnabled, - singleNodeExecutionEnabled); + singleNodeExecutionEnabled, + Collections.emptyMap(), + Collections.emptyMap()); } // Start the remote function server. Return the UDS path used to communicate with it. diff --git a/presto-native-tests/README.md b/presto-native-tests/README.md new file mode 100644 index 0000000000000..a882b2a33b463 --- /dev/null +++ b/presto-native-tests/README.md @@ -0,0 +1,31 @@ +# Presto Native Tests + +This module contains end-to-end tests that run queries from test classes in +the `presto-tests` module with Presto C++ workers. Please build the module +`presto-native-execution` first. + +The following command can be used to run all tests in this module: +``` +mvn test + -pl 'presto-native-tests' + -Dtest="com.facebook.presto.nativetests.Test*" + -Duser.timezone=America/Bahia_Banderas + -DPRESTO_SERVER=${PRESTO_HOME}/presto-native-execution/cmake-build-debug/presto_cpp/main/presto_server + -DWORKER_COUNT=${WORKER_COUNT} -T1C +``` +Please update JVM argument `PRESTO_SERVER` to point to the Presto C++ worker +binary `presto_server`. + +## Adding new tests + +Presto C++ currently does not have the same behavior as Presto for certain +queries. This could be because of missing types, missing function signatures, +among other reasons. Tests with these unsupported queries are therefore +expected to fail and the test asserts the error message is as expected. + +Issues should also be created for the failing queries, so they are documented +and fixed. Please add the tag `presto-native-tests` for these issues. +Once all the failures in a testcase are fixed, the overriden test in this +module should be removed and the testcase in the corresponding base class in +`presto-tests` would be the single source of truth for Presto SQL coverage +tests. diff --git a/presto-native-tests/pom.xml b/presto-native-tests/pom.xml new file mode 100644 index 0000000000000..73120202d9575 --- /dev/null +++ b/presto-native-tests/pom.xml @@ -0,0 +1,122 @@ + + + 4.0.0 + + + com.facebook.presto + presto-root + 0.291-SNAPSHOT + + + presto-native-tests + presto-native-tests + Presto Native Tests + + + ${project.parent.basedir} + + + + + org.testng + testng + + + + com.facebook.presto + presto-native-execution + ${project.version} + test-jar + test + + + + com.facebook.presto + presto-common + + + + + com.facebook.presto + presto-spi + + + + com.facebook.presto + presto-main + + + + com.facebook.presto + presto-tests + ${project.version} + + + + com.google.guava + guava + + + + com.facebook.presto + presto-tpcds + test + + + + org.jetbrains + annotations + test + + + + + + + + pl.project13.maven + git-commit-id-plugin + + true + + + + org.basepom.maven + duplicate-finder-maven-plugin + + + parquet.thrift + about.html + mozilla/public-suffix-list.txt + iceberg-build.properties + org.apache.avro.data/Json.avsc + + + com.esotericsoftware.kryo.* + com.esotericsoftware.minlog.Log + com.esotericsoftware.reflectasm.* + module-info + META-INF.versions.9.module-info + org.apache.avro.* + com.github.benmanes.caffeine.* + org.roaringbitmap.* + + + + + org.apache.maven.plugins + maven-surefire-plugin + + -Xms4g -Xmx4g + 1 + false + remote-function,textfile_reader + + /root/project/build/debug/presto_cpp/main/presto_server + + + + + + diff --git a/presto-native-tests/src/test/java/com.facebook.presto.nativetests/AbstractTestAggregationsNative.java b/presto-native-tests/src/test/java/com.facebook.presto.nativetests/AbstractTestAggregationsNative.java new file mode 100644 index 0000000000000..1935940ecdd43 --- /dev/null +++ b/presto-native-tests/src/test/java/com.facebook.presto.nativetests/AbstractTestAggregationsNative.java @@ -0,0 +1,225 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.nativetests; + +import com.facebook.presto.tests.AbstractTestAggregations; +import org.testng.annotations.Test; + +import static java.lang.String.format; + +public abstract class AbstractTestAggregationsNative + extends AbstractTestAggregations +{ + private static final String approxDistributionTypesUnsupportedError = ".*Failed to parse type.*digest.*"; + private static final String aggregateFunctionNotRegisteredError = ".*Aggregate function not registered: .*digest_agg.*"; + + @Override + @Test + public void testApproximateCountDistinct() + { + String timeTypeUnsupportedError = "Failed to parse type.*time"; + String charTypeUnsupportedError = "Failed to parse type.*char"; + String signatureUnsupportedError = ".*Aggregate function signature is not supported.*"; + + // test NULL + assertQuery("SELECT approx_distinct(NULL)", "SELECT 0"); + assertQuery("SELECT approx_distinct(NULL, 0.023)", "SELECT 0"); + + // test date + assertQuery("SELECT approx_distinct(orderdate) FROM orders", "SELECT 2372"); + assertQuery("SELECT approx_distinct(orderdate, 0.023) FROM orders", "SELECT 2372"); + + // test timestamp + assertQuery("SELECT approx_distinct(CAST(orderdate AS TIMESTAMP)) FROM orders", "SELECT 2347"); + assertQuery("SELECT approx_distinct(CAST(orderdate AS TIMESTAMP), 0.023) FROM orders", "SELECT 2347"); + + // test timestamp with time zone + assertQueryFails("SELECT approx_distinct(CAST(orderdate AS TIMESTAMP WITH TIME ZONE)) FROM orders", + signatureUnsupportedError, true); + assertQueryFails("SELECT approx_distinct(CAST(orderdate AS TIMESTAMP WITH TIME ZONE), 0.023) FROM orders", + signatureUnsupportedError, true); + + // test time + assertQueryFails("SELECT approx_distinct(CAST(from_unixtime(custkey) AS TIME)) FROM orders", timeTypeUnsupportedError, true); + assertQueryFails("SELECT approx_distinct(CAST(from_unixtime(custkey) AS TIME), 0.023) FROM orders", timeTypeUnsupportedError, true); + + // test time with time zone + assertQueryFails("SELECT approx_distinct(CAST(from_unixtime(custkey) AS TIME WITH TIME ZONE)) FROM orders", timeTypeUnsupportedError, true); + assertQueryFails("SELECT approx_distinct(CAST(from_unixtime(custkey) AS TIME WITH TIME ZONE), 0.023) FROM orders", timeTypeUnsupportedError, true); + + // test short decimal + assertQuery("SELECT approx_distinct(CAST(custkey AS DECIMAL(18, 0))) FROM orders", "SELECT 990"); + assertQuery("SELECT approx_distinct(CAST(custkey AS DECIMAL(18, 0)), 0.023) FROM orders", "SELECT 990"); + + // test long decimal + assertQuery("SELECT approx_distinct(CAST(custkey AS DECIMAL(25, 20))) FROM orders", "SELECT 1013"); + assertQuery("SELECT approx_distinct(CAST(custkey AS DECIMAL(25, 20)), 0.023) FROM orders", "SELECT 1013"); + + // test real + assertQuery("SELECT approx_distinct(CAST(custkey AS REAL)) FROM orders", "SELECT 982"); + assertQuery("SELECT approx_distinct(CAST(custkey AS REAL), 0.023) FROM orders", "SELECT 982"); + + // test bigint + assertQuery("SELECT approx_distinct(custkey) FROM orders", "SELECT 990"); + assertQuery("SELECT approx_distinct(custkey, 0.023) FROM orders", "SELECT 990"); + + // test integer + assertQuery("SELECT approx_distinct(CAST(custkey AS INTEGER)) FROM orders", "SELECT 1028"); + assertQuery("SELECT approx_distinct(CAST(custkey AS INTEGER), 0.023) FROM orders", "SELECT 1028"); + + // test smallint + assertQuery("SELECT approx_distinct(CAST(custkey AS SMALLINT)) FROM orders", "SELECT 1023"); + assertQuery("SELECT approx_distinct(CAST(custkey AS SMALLINT), 0.023) FROM orders", "SELECT 1023"); + + // test tinyint + assertQuery("SELECT approx_distinct(CAST((custkey % 128) AS TINYINT)) FROM orders", "SELECT 128"); + assertQuery("SELECT approx_distinct(CAST((custkey % 128) AS TINYINT), 0.023) FROM orders", "SELECT 128"); + + // test double + assertQuery("SELECT approx_distinct(CAST(custkey AS DOUBLE)) FROM orders", "SELECT 1014"); + assertQuery("SELECT approx_distinct(CAST(custkey AS DOUBLE), 0.023) FROM orders", "SELECT 1014"); + + // test varchar + assertQuery("SELECT approx_distinct(CAST(custkey AS VARCHAR)) FROM orders", "SELECT 1036"); + assertQuery("SELECT approx_distinct(CAST(custkey AS VARCHAR), 0.023) FROM orders", "SELECT 1036"); + + // test char + assertQueryFails("SELECT approx_distinct(CAST(CAST(custkey AS VARCHAR) AS CHAR(20))) FROM orders", charTypeUnsupportedError, true); + assertQueryFails("SELECT approx_distinct(CAST(CAST(custkey AS VARCHAR) AS CHAR(20)), 0.023) FROM orders", charTypeUnsupportedError, true); + + // test varbinary + assertQuery("SELECT approx_distinct(to_utf8(CAST(custkey AS VARCHAR))) FROM orders", "SELECT 1036"); + assertQuery("SELECT approx_distinct(to_utf8(CAST(custkey AS VARCHAR)), 0.023) FROM orders", "SELECT 1036"); + } + + @Override + @Test(dataProvider = "getType") + public void testStatisticalDigest(String type) + { + assertQueryFails(format("SELECT value_at_quantile(%s_agg(CAST(orderkey AS DOUBLE)), 0.5E0) > 0 FROM lineitem", type), aggregateFunctionNotRegisteredError, true); + assertQueryFails(format("SELECT value_at_quantile(%s_agg(CAST(quantity AS DOUBLE)), 0.5E0) > 0 FROM lineitem", type), aggregateFunctionNotRegisteredError, true); + assertQueryFails(format("SELECT value_at_quantile(%s_agg(CAST(quantity AS DOUBLE)), 0.5E0) > 0 FROM lineitem", type), aggregateFunctionNotRegisteredError, true); + assertQueryFails(format("SELECT value_at_quantile(%s_agg(CAST(orderkey AS DOUBLE), 2), 0.5E0) > 0 FROM lineitem", type), aggregateFunctionNotRegisteredError, true); + assertQueryFails(format("SELECT value_at_quantile(%s_agg(CAST(quantity AS DOUBLE), 3), 0.5E0) > 0 FROM lineitem", type), aggregateFunctionNotRegisteredError, true); + assertQueryFails(format("SELECT value_at_quantile(%s_agg(CAST(quantity AS DOUBLE), 4), 0.5E0) > 0 FROM lineitem", type), aggregateFunctionNotRegisteredError, true); + assertQueryFails(format("SELECT value_at_quantile(%s_agg(CAST(orderkey AS DOUBLE), 2, 0.0001E0), 0.5E0) > 0 FROM lineitem", type), aggregateFunctionNotRegisteredError, true); + assertQueryFails(format("SELECT value_at_quantile(%s_agg(CAST(quantity AS DOUBLE), 3, 0.0001E0), 0.5E0) > 0 FROM lineitem", type), aggregateFunctionNotRegisteredError, true); + assertQueryFails(format("SELECT value_at_quantile(%s_agg(CAST(quantity AS DOUBLE), 4, 0.0001E0), 0.5E0) > 0 FROM lineitem", type), aggregateFunctionNotRegisteredError, true); + } + + /** + * Comprehensive correctness testing is done in the TestQuantileDigestAggregationFunction and TestTDigestAggregationFunction + */ + @Override + @Test(dataProvider = "getType") + public void testStatisticalDigestGroupBy(String type) + { + assertQueryFails(format("SELECT partkey, value_at_quantile(%s_agg(CAST(orderkey AS DOUBLE)), 0.5E0) > 0 FROM lineitem GROUP BY partkey", type), + approxDistributionTypesUnsupportedError, true); + assertQueryFails(format("SELECT partkey, value_at_quantile(%s_agg(CAST(quantity AS DOUBLE)), 0.5E0) > 0 FROM lineitem GROUP BY partkey", type), + approxDistributionTypesUnsupportedError, true); + assertQueryFails(format("SELECT partkey, value_at_quantile(%s_agg(CAST(quantity AS DOUBLE)), 0.5E0) > 0 FROM lineitem GROUP BY partkey", type), + approxDistributionTypesUnsupportedError, true); + assertQueryFails(format("SELECT partkey, value_at_quantile(%s_agg(CAST(orderkey AS DOUBLE), 2), 0.5E0) > 0 FROM lineitem GROUP BY partkey", type), + approxDistributionTypesUnsupportedError, true); + assertQueryFails(format("SELECT partkey, value_at_quantile(%s_agg(CAST(quantity AS DOUBLE), 3), 0.5E0) > 0 FROM lineitem GROUP BY partkey", type), + approxDistributionTypesUnsupportedError, true); + assertQueryFails(format("SELECT partkey, value_at_quantile(%s_agg(CAST(quantity AS DOUBLE), 4), 0.5E0) > 0 FROM lineitem GROUP BY partkey", type), + approxDistributionTypesUnsupportedError, true); + assertQueryFails(format("SELECT partkey, value_at_quantile(%s_agg(CAST(orderkey AS DOUBLE), 2, 0.0001E0), 0.5E0) > 0 FROM lineitem GROUP BY partkey", type), + approxDistributionTypesUnsupportedError, true); + assertQueryFails(format("SELECT partkey, value_at_quantile(%s_agg(CAST(quantity AS DOUBLE), 3, 0.0001E0), 0.5E0) > 0 FROM lineitem GROUP BY partkey", type), + approxDistributionTypesUnsupportedError, true); + assertQueryFails(format("SELECT partkey, value_at_quantile(%s_agg(CAST(quantity AS DOUBLE), 4, 0.0001E0), 0.5E0) > 0 FROM lineitem GROUP BY partkey", type), + approxDistributionTypesUnsupportedError, true); + } + + /** + * Comprehensive correctness testing is done in the TestMergeQuantileDigestFunction and TestMergeTDigestFunction + */ + @Override + @Test(dataProvider = "getType") + public void testStatisticalDigestMerge(String type) + { + assertQueryFails(format("SELECT value_at_quantile(merge(%s), 0.5E0) > 0 FROM (SELECT partkey, %s_agg(CAST(orderkey AS DOUBLE)) as %s FROM lineitem GROUP BY partkey)", + type, + type, + type), + aggregateFunctionNotRegisteredError, true); + } + + /** + * Comprehensive correctness testing is done in the TestMergeQuantileDigestFunction and TestMergeTDigestFunction + */ + @Override + @Test(dataProvider = "getType") + public void testStatisticalDigestMergeGroupBy(String type) + { + assertQueryFails(format("SELECT partkey, value_at_quantile(merge(%s), 0.5E0) > 0 " + + "FROM (SELECT partkey, suppkey, %s_agg(CAST(orderkey AS DOUBLE)) as %s FROM lineitem GROUP BY partkey, suppkey)" + + "GROUP BY partkey", + type, + type, + type), + aggregateFunctionNotRegisteredError, true); + } + + /// See issue for details: https://github.com/prestodb/presto/issues/20909 + @Override + @Test + public void testSumDataSizeForStats() + { + // varchar + assertQuery("SELECT \"sum_data_size_for_stats\"(comment) FROM orders", "SELECT 787364"); + + // char + // Presto removes trailing whitespaces when casting to CHAR. + // Hard code the expected data size since there is no easy to way to compute it in H2. + assertQueryFails("SELECT \"sum_data_size_for_stats\"(CAST(comment AS CHAR(1000))) FROM orders", + "Failed to parse type \\[char\\(1000\\)]", true); + + // varbinary + assertQuery("SELECT \"sum_data_size_for_stats\"(CAST(comment AS VARBINARY)) FROM orders", "SELECT 787364"); + + // array + assertQuery("SELECT \"sum_data_size_for_stats\"(ARRAY[comment]) FROM orders", "SELECT 847364"); + assertQuery("SELECT \"sum_data_size_for_stats\"(ARRAY[comment, comment]) FROM orders", "SELECT 1634728"); + + // map + assertQuery("SELECT \"sum_data_size_for_stats\"(map(ARRAY[1], ARRAY[comment])) FROM orders", "SELECT 907364"); + assertQuery("SELECT \"sum_data_size_for_stats\"(map(ARRAY[1, 2], ARRAY[comment, comment])) FROM orders", "SELECT 1754728"); + + // row + assertQuery("SELECT \"sum_data_size_for_stats\"(ROW(comment)) FROM orders", "SELECT 847364"); + assertQuery("SELECT \"sum_data_size_for_stats\"(ROW(comment, comment)) FROM orders", "SELECT 1634728"); + } + + /// See issue for details: https://github.com/prestodb/presto/issues/20909 + @Override + @Test + public void testMaxDataSizeForStats() + { + // varchar + assertQuery("SELECT \"max_data_size_for_stats\"(comment) FROM orders", "select 82"); + + // char + assertQueryFails("SELECT \"max_data_size_for_stats\"(CAST(comment AS CHAR(1000))) FROM orders", + "Failed to parse type \\[char\\(1000\\)]", true); + + // varbinary + assertQuery("SELECT \"max_data_size_for_stats\"(CAST(comment AS VARBINARY)) FROM orders", "select 82"); + + // max_data_size_for_stats is not needed for array, map and row + } +} diff --git a/presto-native-tests/src/test/java/com.facebook.presto.nativetests/AbstractTestQueriesNative.java b/presto-native-tests/src/test/java/com.facebook.presto.nativetests/AbstractTestQueriesNative.java new file mode 100644 index 0000000000000..370b2188bb1b1 --- /dev/null +++ b/presto-native-tests/src/test/java/com.facebook.presto.nativetests/AbstractTestQueriesNative.java @@ -0,0 +1,1208 @@ +package com.facebook.presto.nativetests; + +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.facebook.presto.Session; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.testing.MaterializedResult; +import com.facebook.presto.testing.MaterializedRow; +import com.facebook.presto.tests.AbstractTestQueries; +import org.intellij.lang.annotations.Language; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.util.List; +import java.util.Set; + +import static com.facebook.presto.SystemSessionProperties.FIELD_NAMES_IN_JSON_CAST_ENABLED; +import static com.facebook.presto.SystemSessionProperties.JOIN_PREFILTER_BUILD_SIDE; +import static com.facebook.presto.SystemSessionProperties.KEY_BASED_SAMPLING_ENABLED; +import static com.facebook.presto.SystemSessionProperties.KEY_BASED_SAMPLING_PERCENTAGE; +import static com.facebook.presto.SystemSessionProperties.MERGE_AGGREGATIONS_WITH_AND_WITHOUT_FILTER; +import static com.facebook.presto.SystemSessionProperties.PREFILTER_FOR_GROUPBY_LIMIT; +import static com.facebook.presto.SystemSessionProperties.PREFILTER_FOR_GROUPBY_LIMIT_TIMEOUT_MS; +import static com.facebook.presto.SystemSessionProperties.REMOVE_MAP_CAST; +import static com.facebook.presto.SystemSessionProperties.REMOVE_REDUNDANT_CAST_TO_VARCHAR_IN_JOIN; +import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.testing.MaterializedResult.resultBuilder; +import static com.facebook.presto.testing.assertions.Assert.assertEquals; +import static com.facebook.presto.tests.QueryAssertions.assertEqualsIgnoreOrder; +import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static org.testng.Assert.assertNotEquals; + +public abstract class AbstractTestQueriesNative + extends AbstractTestQueries +{ + private static final String UNSUPPORTED_CORRELATED_SUBQUERY_ERROR_MSG = "line .*: Given correlated subquery is not supported"; + private static final String P4_HLL_TIMESTAMP_TZ_TYPE_UNSUPPORTED_ERROR = ".*Failed to parse type \\[P4HyperLogLog]. Type not registered.*"; + private static final String KHLL_TIMESTAMP_TZ_TYPE_UNSUPPORTED_ERROR = ".*inferredType Failed to parse type \\[KHyperLogLog]. Type not registered.*"; + private static final String CHAR_TYPE_UNSUPPORTED_ERROR = ".*Failed to parse type \\[char\\(.*\\)]. syntax error, unexpected LPAREN, expecting WORD.*"; + private static final String HASH_GENERATION_UNSUPPORTED_ERROR = ".*Scalar function name not registered: presto.default.\\$operator\\$.*hash.*"; + private static final String ARRAY_COMPARISON_UNSUPPORTED_ERROR = ".*ARRAY comparison not supported for values that contain nulls.*"; + private static final String CREATE_HLL_FUNCTION_NOT_REGISTERED = ".*Scalar function name not registered: presto.default.create_hll, called with arguments.*"; + + @DataProvider(name = "use_default_literal_coalesce") + public static Object[][] useDefaultLiteralCoalesce() + { + return new Object[][] {{true}}; + } + + // These tests are disabled because they use the following functions that will not be supported in Presto C++: + // custom_add, custom_sum, custom_rank, apply. + @Override + @Test(enabled = false) + public void testCustomAdd() {} + + @Override + @Test(enabled = false) + public void testCustomSum() {} + + @Override + @Test(enabled = false) + public void testCustomRank() {} + + @Override + @Test(enabled = false) + public void testApplyLambdaRepeated() {} + + @Override + @Test(enabled = false) + public void testLambdaCapture() {} + + @Override + @Test(enabled = false) + public void testLambdaInAggregationContext() {} + + @Override + @Test(enabled = false) + public void testLambdaInSubqueryContext() {} + + @Override + @Test(enabled = false) + public void testNonDeterministicInLambda() {} + + @Override + @Test(enabled = false) + public void testRowSubscriptInLambda() {} + + @Override + @Test(enabled = false) + public void testTryWithLambda() {} + + // The following tests are disabled because Velox does not support optimized hash generation. + @Override + @Test(enabled = false) + public void testDoubleDistinctPositiveAndNegativeZero(String optimizeHashGeneration) {} + + @Override + @Test(enabled = false) + public void testRealDistinctPositiveAndNegativeZero(String optimizeHashGeneration) {} + + @Override + @Test(enabled = false) + public void testRealJoinPositiveAndNegativeZero(String optimizeHashGeneration) {} + + @Override + @Test(enabled = false) + public void testDoubleJoinPositiveAndNegativeZero(String optimizeHashGeneration) {} + + // reduce_agg returns different results in Presto C++. See presto-docs/src/main/sphinx/presto_cpp/limitations.rst. + @Override + @Test + public void testReduceAgg() + { + assertQuery( + "SELECT x, reduce_agg(y, 1, (a, b) -> a * b, (a, b) -> a * b) " + + "FROM (VALUES (1, 5), (1, 6), (1, 7), (2, 8), (2, 9), (3, 10)) AS t(x, y) " + + "GROUP BY x", + "VALUES (1, 5 * 6 * 7), (2, 8 * 9), (3, 10)"); + assertQuery( + "SELECT x, reduce_agg(y, 0, (a, b) -> a + b, (a, b) -> a + b) " + + "FROM (VALUES (1, 5), (1, 6), (1, 7), (2, 8), (2, 9), (3, 10)) AS t(x, y) " + + "GROUP BY x", + "VALUES (1, 5 + 6 + 7), (2, 8 + 9), (3, 10)"); + + assertQuery( + "SELECT x, reduce_agg(y, 1, (a, b) -> a * b, (a, b) -> a * b) " + + "FROM (VALUES (1, CAST(5 AS DOUBLE)), (1, 6), (1, 7), (2, 8), (2, 9), (3, 10)) AS t(x, y) " + + "GROUP BY x", + "VALUES (1, CAST(5 AS DOUBLE) * 6 * 7), (2, 8 * 9), (3, 10)"); + assertQuery( + "SELECT x, reduce_agg(y, 0, (a, b) -> a + b, (a, b) -> a + b) " + + "FROM (VALUES (1, CAST(5 AS DOUBLE)), (1, 6), (1, 7), (2, 8), (2, 9), (3, 10)) AS t(x, y) " + + "GROUP BY x", + "VALUES (1, CAST(5 AS DOUBLE) + 6 + 7), (2, 8 + 9), (3, 10)"); + + assertQuery( + "SELECT " + + "x, " + + "array_join(" + + " array_sort(" + + " split(reduce_agg(y, '', (a, b) -> a || b, (a, b) -> a || b), '')" + + " ), " + + " ''" + + ") " + + "FROM (VALUES (1, 'a'), (1, 'b'), (1, 'c'), (2, 'd'), (2, 'e'), (3, 'f')) AS t(x, y) " + + "GROUP BY x", + "VALUES (1, 'abc'), (2, 'de'), (3, 'f')"); + + assertQuery( + "SELECT " + + "x, " + + "array_join(" + + " array_sort(" + + " reduce_agg(y, ARRAY['x'], (a, b) -> a || b, (a, b) -> a || b)" + + " ), " + + " ''" + + ") " + + "FROM (VALUES (1, ARRAY['a']), (1, ARRAY['b']), (1, ARRAY['c']), (2, ARRAY['d']), (2, ARRAY['e']), (3, ARRAY['f'])) AS t(x, y) " + + "GROUP BY x", + "VALUES (1, 'abcxxx'), (2, 'dexx'), (3, 'fx')"); + + assertQuery("SELECT REDUCE_AGG((x,y), (0,0), (x, y)->(x[1],y[1]), (x,y)->(x[1],y[1]))[1] from (select 1 x, 2 y)", "select 0"); + } + + @Override + @Test + public void testReduceAggWithNulls() + { + assertQueryFails("select reduce_agg(x, null, (x,y)->try(x+y), (x,y)->try(x+y)) from (select 1 union all select 10) T(x)", ".*REDUCE_AGG only supports non-NULL literal as the initial value.*"); + assertQueryFails("select reduce_agg(x, cast(null as bigint), (x,y)->coalesce(x, 0)+coalesce(y, 0), (x,y)->coalesce(x, 0)+coalesce(y, 0)) from (values cast(10 as bigint),10)T(x)", ".*REDUCE_AGG only supports non-NULL literal as the initial value.*"); + + // here some reduce_aggs coalesce overflow/zero-divide errors to null in the input/combine functions + assertQueryFails("select reduce_agg(x, 0, (x,y)->try(1/x+1/y), (x,y)->try(1/x+1/y)) from ((select 0) union all select 10.) T(x)", "!states->isNullAt\\(i\\) Lambda expressions in reduce_agg should not return null for non-null inputs", true); + assertQueryFails("select reduce_agg(x, 0, (x, y)->try(x+y), (x, y)->try(x+y)) from (values 2817, 9223372036854775807) AS T(x)", "!states->isNullAt\\(i\\) Lambda expressions in reduce_agg should not return null for non-null inputs", true); + assertQuery("select reduce_agg(x, array[], (x, y)->array[element_at(x, 2)], (x, y)->array[element_at(x, 2)]) from (select array[array[1]]) T(x)", "select array[null]"); + } + + // The following approx_set function tests are overriden since approx_set returns different results in Presto C++. + @Override + @Test + public void testApproxSetBigint() + { + MaterializedResult actual = computeActual("SELECT cardinality(approx_set(custkey)) FROM orders"); + + MaterializedResult expected = resultBuilder(getSession(), BIGINT) + .row(1005L) + .build(); + + assertEquals(actual.getMaterializedRows(), expected.getMaterializedRows()); + } + + @Override + @Test + public void testApproxSetBigintGroupBy() + { + MaterializedResult actual = computeActual("" + + "SELECT orderstatus, cardinality(approx_set(custkey)) " + + "FROM orders " + + "GROUP BY orderstatus"); + + MaterializedResult expected = resultBuilder(getSession(), actual.getTypes()) + .row("O", 1003L) + .row("F", 1001L) + .row("P", 304L) + .build(); + + assertEqualsIgnoreOrder(actual.getMaterializedRows(), expected.getMaterializedRows()); + } + + @Override + @Test + public void testApproxSetDouble() + { + MaterializedResult actual = computeActual("SELECT cardinality(approx_set(CAST(custkey AS DOUBLE))) FROM orders"); + + MaterializedResult expected = resultBuilder(getSession(), BIGINT) + .row(1002L) + .build(); + + assertEquals(actual.getMaterializedRows(), expected.getMaterializedRows()); + } + + @Override + @Test + public void testApproxSetDoubleGroupBy() + { + MaterializedResult actual = computeActual("" + + "SELECT orderstatus, cardinality(approx_set(CAST(custkey AS DOUBLE))) " + + "FROM orders " + + "GROUP BY orderstatus"); + + MaterializedResult expected = resultBuilder(getSession(), actual.getTypes()) + .row("O", 1002L) + .row("F", 998L) + .row("P", 304L) + .build(); + + assertEqualsIgnoreOrder(actual.getMaterializedRows(), expected.getMaterializedRows()); + } + + @Override + @Test + public void testApproxSetGroupByWithNulls() + { + MaterializedResult actual = computeActual("" + + "SELECT orderstatus, cardinality(approx_set(IF(custkey % 2 <> 0, custkey))) " + + "FROM orders " + + "GROUP BY orderstatus"); + + MaterializedResult expected = resultBuilder(getSession(), actual.getTypes()) + .row("O", 499L) + .row("F", 496L) + .row("P", 153L) + .build(); + + assertEqualsIgnoreOrder(actual.getMaterializedRows(), expected.getMaterializedRows()); + } + + @Override + @Test + public void testApproxSetVarchar() + { + MaterializedResult actual = computeActual("SELECT cardinality(approx_set(CAST(custkey AS VARCHAR))) FROM orders"); + + MaterializedResult expected = resultBuilder(getSession(), BIGINT) + .row(1015L) + .build(); + + assertEquals(actual.getMaterializedRows(), expected.getMaterializedRows()); + } + + @Override + @Test + public void testApproxSetVarcharGroupBy() + { + MaterializedResult actual = computeActual("" + + "SELECT orderstatus, cardinality(approx_set(CAST(custkey AS VARCHAR))) " + + "FROM orders " + + "GROUP BY orderstatus"); + + MaterializedResult expected = resultBuilder(getSession(), actual.getTypes()) + .row("O", 1012L) + .row("F", 1011L) + .row("P", 304L) + .build(); + + assertEqualsIgnoreOrder(actual.getMaterializedRows(), expected.getMaterializedRows()); + } + + @Override + @Test + public void testApproxSetWithNulls() + { + MaterializedResult actual = computeActual("SELECT cardinality(approx_set(IF(orderstatus = 'O', custkey))) FROM orders"); + + MaterializedResult expected = resultBuilder(getSession(), actual.getTypes()) + .row(1003L) + .build(); + + assertEquals(actual.getMaterializedRows(), expected.getMaterializedRows()); + } + + @Override + @Test + public void testApproxSetGroupByWithOnlyNullsInOneGroup() + { + MaterializedResult actual = computeActual("" + + "SELECT orderstatus, cardinality(approx_set(IF(orderstatus != 'O', custkey))) " + + "FROM orders " + + "GROUP BY orderstatus"); + + MaterializedResult expected = resultBuilder(getSession(), actual.getTypes()) + .row("O", null) + .row("F", 1001L) + .row("P", 304L) + .build(); + + assertEqualsIgnoreOrder(actual.getMaterializedRows(), expected.getMaterializedRows()); + } + + @Override + @Test + public void testArrayCumSumDecimals() + { + String functionNotRegisteredError = ".*Scalar function presto.default.array_cum_sum not registered with arguments.*"; + String sql = "select array_cum_sum(k) from (values (array[cast(5.1 as decimal(38, 1)), 6, 0]), (ARRAY[]), (CAST(NULL AS array(decimal)))) t(k)"; + assertQueryFails(sql, functionNotRegisteredError, true); + + sql = "select array_cum_sum(k) from (values (array[cast(5.1 as decimal(38, 1)), 6, null, 3]), (array[cast(null as decimal(38, 1)), 6, null, 3])) t(k)"; + assertQueryFails(sql, functionNotRegisteredError, true); + } + + @Override + @Test + public void testArrayCumSumVarchar() + { + String sql = "select array_cum_sum(k) from (values (array[cast('5.1' as varchar), '6', '0']), (ARRAY[]), (CAST(NULL AS array(varchar)))) t(k)"; + assertQueryFails(sql, ".*Scalar function presto.default.array_cum_sum not registered with arguments.*", true); + + sql = "select array_cum_sum(k) from (values (array[cast(null as varchar), '6', '0'])) t(k)"; + assertQueryFails(sql, ".*Scalar function presto.default.array_cum_sum not registered with arguments: \\(ARRAY.*", true); + } + + @Override + @Test + public void testGroupByLimit() + { + Session prefilter = Session.builder(getSession()) + .setSystemProperty(PREFILTER_FOR_GROUPBY_LIMIT, "true") + .build(); + MaterializedResult result1 = computeActual(prefilter, "select count(shipdate), orderkey from lineitem group by orderkey limit 100000"); + MaterializedResult result2 = computeActual("select count(shipdate), orderkey from lineitem group by orderkey limit 100000"); + assertEqualsIgnoreOrder(result1, result2, "Prefilter and without prefilter don't give matching results"); + + assertQueryFails(prefilter, "select count(custkey), orderkey from orders where orderstatus='F' and orderkey < 50 group by orderkey limit 100", HASH_GENERATION_UNSUPPORTED_ERROR, true); + assertQuery(prefilter, "select count(1) from (select count(custkey), orderkey from orders where orderstatus='F' and orderkey < 50 group by orderkey limit 4)", "select 4"); + assertQuery(prefilter, "select count(comment), orderstatus from (select upper(comment) comment, upper(orderstatus) orderstatus from orders where orderkey < 50) group by orderstatus limit 100", "values (5, 'F'), (10, 'O')"); + + assertQuery(prefilter, "select count(comment), orderstatus from (select upper(comment) comment, upper(orderstatus) orderstatus from orders where orderkey < 50) group by orderstatus having count(1) > 1 limit 100", "values (5, 'F'), (10, 'O')"); + + prefilter = Session.builder(getSession()) + .setSystemProperty(PREFILTER_FOR_GROUPBY_LIMIT, "true") + .setSystemProperty(PREFILTER_FOR_GROUPBY_LIMIT_TIMEOUT_MS, "1") + .build(); + + result1 = computeActual(prefilter, "select count(shipdate), orderkey from lineitem group by orderkey limit 100000"); + result2 = computeActual("select count(shipdate), orderkey from lineitem group by orderkey limit 100000"); + assertEqualsIgnoreOrder(result1, result2, "Prefilter and without prefilter don't give matching results"); + + assertQuery(prefilter, "select count(1) from (select count(custkey), orderkey from orders group by orderkey limit 100000)", "values 15000"); + assertQuery(prefilter, "select count(1) from (select count(custkey), orderkey from orders group by orderkey limit 4)", "select 4"); + assertQuery(prefilter, "select count(1) from (select count(comment), orderstatus from (select upper(comment) comment, upper(orderstatus) orderstatus from orders) group by orderstatus limit 100000)", "values 3"); + } + + @Override + @Test + public void testJoinPrefilter() + { + { + // Orig + String testQuery = "SELECT 1 from region join nation using(regionkey)"; + MaterializedResult result = computeActual("explain(type distributed) " + testQuery); + assertEquals(((String) result.getMaterializedRows().get(0).getField(0)).indexOf("SemiJoin"), -1); + result = computeActual(testQuery); + assertEquals(result.getRowCount(), 25); + + // With feature + Session session = Session.builder(getSession()) + .setSystemProperty(JOIN_PREFILTER_BUILD_SIDE, String.valueOf(true)) + .build(); + result = computeActual(session, "explain(type distributed) " + testQuery); + assertNotEquals(((String) result.getMaterializedRows().get(0).getField(0)).indexOf("SemiJoin"), -1); + result = computeActual(session, testQuery); + assertEquals(result.getRowCount(), 25); + } + + { + // Orig + @Language("SQL") String testQuery = "SELECT 1 from region r join nation n on cast(r.regionkey as varchar) = cast(n.regionkey as varchar)"; + MaterializedResult result = computeActual("explain(type distributed) " + testQuery); + assertEquals(((String) result.getMaterializedRows().get(0).getField(0)).indexOf("SemiJoin"), -1); + result = computeActual(testQuery); + assertEquals(result.getRowCount(), 25); + + // With feature + Session session = Session.builder(getSession()) + .setSystemProperty(JOIN_PREFILTER_BUILD_SIDE, String.valueOf(true)) + .setSystemProperty(REMOVE_REDUNDANT_CAST_TO_VARCHAR_IN_JOIN, String.valueOf(false)) + .build(); + result = computeActual(session, "explain(type distributed) " + testQuery); + assertNotEquals(((String) result.getMaterializedRows().get(0).getField(0)).indexOf("SemiJoin"), -1); + assertNotEquals(((String) result.getMaterializedRows().get(0).getField(0)).indexOf("XX_HASH_64"), -1); + assertQueryFails(session, testQuery, HASH_GENERATION_UNSUPPORTED_ERROR, true); + } + + { + // Orig + String testQuery = "SELECT 1 from lineitem l join orders o on l.orderkey = o.orderkey and l.suppkey = o.custkey"; + MaterializedResult result = computeActual("explain(type distributed) " + testQuery); + assertEquals(((String) result.getMaterializedRows().get(0).getField(0)).indexOf("SemiJoin"), -1); + result = computeActual(testQuery); + assertEquals(result.getRowCount(), 37); + + // With feature + Session session = Session.builder(getSession()) + .setSystemProperty(JOIN_PREFILTER_BUILD_SIDE, String.valueOf(true)) + .build(); + result = computeActual(session, "explain(type distributed) " + testQuery); + assertNotEquals(((String) result.getMaterializedRows().get(0).getField(0)).indexOf("SemiJoin"), -1); + assertNotEquals(((String) result.getMaterializedRows().get(0).getField(0)).indexOf("XX_HASH_64"), -1); + assertQueryFails(session, testQuery, HASH_GENERATION_UNSUPPORTED_ERROR, true); + } + } + + @Override + @Test(expectedExceptions = {RuntimeException.class, PrestoException.class}, expectedExceptionsMessageRegExp = CREATE_HLL_FUNCTION_NOT_REGISTERED) + public void testMergeEmptyNonEmptyApproxSetWithDifferentMaxError() + { + computeActual("SELECT cardinality(merge(c)) FROM (SELECT create_hll(custkey, 0.1) c FROM orders UNION ALL SELECT empty_approx_set(0.2))"); + } + + @Override + @Test + public void testMergeHyperLogLog() + { + assertQueryFails("SELECT cardinality(merge(create_hll(custkey))) FROM orders", CREATE_HLL_FUNCTION_NOT_REGISTERED, true); + } + + @Override + @Test + public void testMergeHyperLogLogGroupBy() + { + assertQueryFails( + "SELECT orderstatus, cardinality(merge(create_hll(custkey))) " + + "FROM orders " + + "GROUP BY orderstatus", CREATE_HLL_FUNCTION_NOT_REGISTERED, true); + } + + @Override + @Test + public void testMergeHyperLogLogWithNulls() + { + assertQueryFails("SELECT cardinality(merge(create_hll(IF(orderstatus = 'O', custkey)))) FROM orders", + CREATE_HLL_FUNCTION_NOT_REGISTERED, true); + } + + @Override + @Test + public void testMergeHyperLogLogGroupByWithNulls() + { + assertQueryFails( + "SELECT orderstatus, cardinality(merge(create_hll(IF(orderstatus != 'O', custkey)))) " + + "FROM orders " + + "GROUP BY orderstatus", + CREATE_HLL_FUNCTION_NOT_REGISTERED, true); + } + + @Override + @Test + public void testKeyBasedSampling() + { + String[] queries = new String[]{ + "select count(1) from orders join lineitem using(orderkey)", + "select count(1) from (select custkey, max(orderkey) from orders group by custkey)", + "select count_if(m >= 1) from (select max(orderkey) over(partition by custkey) m from orders)", + "select cast(m as bigint) from (select sum(totalprice) over(partition by custkey order by comment) m from orders order by 1 desc limit 1)", + "select count(1) from lineitem where orderkey in (select orderkey from orders where length(comment) > 7)", + "select count(1) from lineitem where orderkey not in (select orderkey from orders where length(comment) > 27)", + "select count(1) from (select distinct orderkey, custkey from orders)", + }; + int[] unsampledResults = new int[]{60175, 1000, 15000, 5408941, 60175, 9256, 15000}; + for (int i = 0; i < queries.length; i++) { + assertQuery(queries[i], "select " + unsampledResults[i]); + } + + Session sessionWithKeyBasedSampling = Session.builder(getSession()) + .setSystemProperty(KEY_BASED_SAMPLING_ENABLED, "true") + .setSystemProperty(KEY_BASED_SAMPLING_PERCENTAGE, "0.2") + .build(); + for (int i = 0; i < queries.length; i++) { + assertQueryFails(sessionWithKeyBasedSampling, queries[i], "Scalar function name not registered: presto.default.key_sampling_percent, called with arguments", true); + } + + sessionWithKeyBasedSampling = Session.builder(getSession()) + .setSystemProperty(KEY_BASED_SAMPLING_ENABLED, "true") + .setSystemProperty(KEY_BASED_SAMPLING_PERCENTAGE, "0.1") + .build(); + for (int i = 0; i < queries.length; i++) { + assertQueryFails(sessionWithKeyBasedSampling, queries[i], "Scalar function name not registered: presto.default.key_sampling_percent, called with arguments", true); + } + } + + @Override + @Test + public void testP4ApproxSetBigint() + { + assertQueryFails("SELECT cardinality(cast(approx_set(custkey) AS P4HYPERLOGLOG)) FROM orders", + P4_HLL_TIMESTAMP_TZ_TYPE_UNSUPPORTED_ERROR, true); + } + + @Override + @Test + public void testP4ApproxSetVarchar() + { + assertQueryFails("SELECT cardinality(cast(approx_set(CAST(custkey AS VARCHAR)) AS P4HYPERLOGLOG)) FROM orders", + P4_HLL_TIMESTAMP_TZ_TYPE_UNSUPPORTED_ERROR, true); + } + + @Override + @Test + public void testP4ApproxSetBigintGroupBy() + { + assertQueryFails( + "SELECT orderstatus, cardinality(cast(approx_set(custkey) AS P4HYPERLOGLOG)) " + + "FROM orders " + + "GROUP BY orderstatus", P4_HLL_TIMESTAMP_TZ_TYPE_UNSUPPORTED_ERROR, true); + } + + @Override + @Test + public void testP4ApproxSetDouble() + { + assertQueryFails("SELECT cardinality(cast(approx_set(CAST(custkey AS DOUBLE)) AS P4HYPERLOGLOG)) FROM orders", + P4_HLL_TIMESTAMP_TZ_TYPE_UNSUPPORTED_ERROR, true); + } + + @Override + @Test + public void testP4ApproxSetDoubleGroupBy() + { + assertQueryFails( + "SELECT orderstatus, cardinality(cast(approx_set(CAST(custkey AS DOUBLE)) AS P4HYPERLOGLOG)) " + + "FROM orders " + + "GROUP BY orderstatus", P4_HLL_TIMESTAMP_TZ_TYPE_UNSUPPORTED_ERROR, true); + } + + @Override + @Test + public void testP4ApproxSetGroupByWithNulls() + { + assertQueryFails( + "SELECT orderstatus, cardinality(cast(approx_set(IF(custkey % 2 <> 0, custkey)) AS P4HYPERLOGLOG)) " + + "FROM orders " + + "GROUP BY orderstatus", P4_HLL_TIMESTAMP_TZ_TYPE_UNSUPPORTED_ERROR, true); + } + + @Override + @Test + public void testP4ApproxSetGroupByWithOnlyNullsInOneGroup() + { + assertQueryFails( + "SELECT orderstatus, cardinality(cast(approx_set(IF(orderstatus != 'O', custkey)) AS P4HYPERLOGLOG)) " + + "FROM orders " + + "GROUP BY orderstatus", P4_HLL_TIMESTAMP_TZ_TYPE_UNSUPPORTED_ERROR, true); + } + + @Override + @Test + public void testP4ApproxSetOnlyNulls() + { + assertQueryFails("SELECT cardinality(cast(approx_set(null) AS P4HYPERLOGLOG)) FROM orders", + P4_HLL_TIMESTAMP_TZ_TYPE_UNSUPPORTED_ERROR, true); + } + + @Override + @Test + public void testP4ApproxSetVarcharGroupBy() + { + assertQueryFails( + "SELECT orderstatus, cardinality(cast(approx_set(CAST(custkey AS VARCHAR)) AS P4HYPERLOGLOG)) " + + "FROM orders " + + "GROUP BY orderstatus", P4_HLL_TIMESTAMP_TZ_TYPE_UNSUPPORTED_ERROR, true); + } + + @Override + @Test + public void testP4ApproxSetWithNulls() + { + assertQueryFails("SELECT cardinality(cast(approx_set(IF(orderstatus = 'O', custkey)) AS P4HYPERLOGLOG)) FROM orders", + P4_HLL_TIMESTAMP_TZ_TYPE_UNSUPPORTED_ERROR, true); + } + + @Override + @Test + public void testLargeBytecode() + { + StringBuilder stringBuilder = new StringBuilder("SELECT x FROM (SELECT orderkey x, custkey y from orders limit 10) WHERE CASE true "); + // Generate 100 cases. + for (int i = 0; i < 100; i++) { + stringBuilder.append(" when x in ("); + for (int j = 0; j < 20; j++) { + stringBuilder.append("random(" + (i * 100 + j) + "), "); + } + + stringBuilder.append("random(" + i + ")) then x = random()"); + } + + stringBuilder.append("else x = random() end"); + assertQueryFails(stringBuilder.toString(), + "input > 0 \\(0 vs. 0\\) bound must be positive presto.default.random\\(0:INTEGER\\)", true); + } + + @Override + @Test + public void testArraySplitIntoChunks() + { + String functionNotRegistered = "Scalar function name not registered: presto.default.array_split_into_chunks"; + + String sql = "select array_split_into_chunks(array[1, 2, 3, 4, 5, 6], 2)"; + assertQueryFails(sql, functionNotRegistered, true); + + sql = "select array_split_into_chunks(array[1, 2, 3, 4, 5], 3)"; + assertQueryFails(sql, functionNotRegistered, true); + + sql = "select array_split_into_chunks(array[1, 2, 3], 5)"; + assertQueryFails(sql, functionNotRegistered, true); + + sql = "select array_split_into_chunks(null, 2)"; + assertQuery(sql, "values null"); + + sql = "select array_split_into_chunks(array[1, 2, 3], 0)"; + assertQueryFails(sql, functionNotRegistered, true); + + sql = "select array_split_into_chunks(array[1, 2, 3], -1)"; + assertQueryFails(sql, functionNotRegistered, true); + + sql = "select array_split_into_chunks(array[1, null, 3, null, 5], 2)"; + assertQueryFails(sql, functionNotRegistered, true); + + sql = "select array_split_into_chunks(array['a', 'b', 'c', 'd'], 2)"; + assertQueryFails(sql, functionNotRegistered, true); + + sql = "select array_split_into_chunks(array[1.1, 2.2, 3.3, 4.4, 5.5], 2)"; + assertQueryFails(sql, functionNotRegistered, true); + + sql = "select array_split_into_chunks(array[null, null, null], 0)"; + assertQueryFails(sql, functionNotRegistered, true); + + sql = "select array_split_into_chunks(array[null, null, null], 2)"; + assertQueryFails(sql, functionNotRegistered, true); + + sql = "select array_split_into_chunks(array[null, 1, 2], 5)"; + assertQueryFails(sql, functionNotRegistered, true); + + sql = "select array_split_into_chunks(array[], 0)"; + assertQueryFails(sql, functionNotRegistered, true); + } + + @Override + @Test + public void testMapUnionSumOverflow() + { + assertQueryFails( + "select y, map_union_sum(x) from (select 1 y, map(array['x', 'z', 'y'], cast(array[null,30,100] as array)) x " + + "union all select 1 y, map(array['x', 'y'], cast(array[1,100] as array))x) group by y", "Value 200 exceeds 127", true); + assertQueryFails( + "select y, map_union_sum(x) from (select 1 y, map(array['x', 'z', 'y'], cast(array[null,30, 32760] as array)) x " + + "union all select 1 y, map(array['x', 'y'], cast(array[1,100] as array))x) group by y", "Value 32860 exceeds 32767", true); + } + + @Override + @Test + public void testRows() + { + // Using JSON_FORMAT(CAST(_ AS JSON)) because H2 does not support ROW type + Session session = Session.builder(getSession()).setSystemProperty(FIELD_NAMES_IN_JSON_CAST_ENABLED, "true").build(); + assertQuery(session, "SELECT JSON_FORMAT(CAST(ROW(1 + 2, CONCAT('a', 'b')) AS JSON))", "SELECT '{\"\":3,\"\":\"ab\"}'"); + assertQuery(session, "SELECT JSON_FORMAT(CAST(ROW(a + b) AS JSON)) FROM (VALUES (1, 2)) AS t(a, b)", "SELECT '[3]'"); + assertQuery(session, "SELECT JSON_FORMAT(CAST(ROW(1, ROW(9, a, ARRAY[], NULL), ROW(1, 2)) AS JSON)) FROM (VALUES ('a')) t(a)", + "SELECT '[1,[9,\"a\",[],null],[1,2]]'"); + assertQuery(session, "SELECT JSON_FORMAT(CAST(ROW(ROW(ROW(ROW(ROW(a, b), c), d), e), f) AS JSON)) FROM (VALUES (ROW(0, 1), 2, '3', NULL, ARRAY[5], ARRAY[])) t(a, b, c, d, e, f)", + "SELECT '[[[[[[0,1],2],\"3\"],null],[5]],[]]'"); + assertQuery(session, "SELECT JSON_FORMAT(CAST(ARRAY_AGG(ROW(a, b)) AS JSON)) FROM (VALUES (1, 2), (3, 4), (5, 6)) t(a, b)", + "SELECT '[[1,2],[3,4],[5,6]]'"); + assertQuery(session, "SELECT CONTAINS(ARRAY_AGG(ROW(a, b)), ROW(1, 2)) FROM (VALUES (1, 2), (3, 4), (5, 6)) t(a, b)", "SELECT TRUE"); + assertQuery(session, "SELECT JSON_FORMAT(CAST(ARRAY_AGG(ROW(c, d)) AS JSON)) FROM (VALUES (ARRAY[1, 3, 5], ARRAY[2, 4, 6])) AS t(a, b) CROSS JOIN UNNEST(a, b) AS u(c, d)", + "SELECT '[[1,2],[3,4],[5,6]]'"); + assertQuery(session, "SELECT JSON_FORMAT(CAST(ROW(x, y, z) AS JSON)) FROM (VALUES ROW(1, NULL, '3')) t(x,y,z)", "SELECT '[1,null,\"3\"]'"); + assertQuery(session, "SELECT JSON_FORMAT(CAST(ROW(x, y, z) AS JSON)) FROM (VALUES ROW(1, CAST(NULL AS INTEGER), '3')) t(x,y,z)", "SELECT '[1,null,\"3\"]'"); + } + + @Override + @Test + public void testDuplicateUnnestRows() + { + assertQueryFails("SELECT * from (select * FROM (values 1) as t(k)) CROSS JOIN unnest(ARRAY[row(2, 3), row(3, 5)], ARRAY[row(2, 3), row(3, 5)]) AS r(r1, r2, r3, r4)", + "Field not found: field_.*. Available fields are: field, field_.*", true); + assertQueryFails("SELECT * from (select * FROM (values 1) as t(k)) CROSS JOIN unnest(ARRAY[row(2, 3), row(3, 5)], ARRAY[row(2, 3), row(3, 5)], ARRAY[row(10, 13, 15), row(23, 25, 20)]) AS r(r1, r2, r3, r4, r5, r6, r7)", + "Field not found: field_.*. Available fields are: field, field_.*", true); + assertQueryFails("SELECT * from (select * FROM (values 1) as t(k)) CROSS JOIN unnest(ARRAY[row(2, 3), row(3, 5)], ARRAY[row(2, 3), row(3, 5)]) WITH ORDINALITY AS r(r1, r2, r3, r4, ord)", + "Field not found: field_.*. Available fields are: field, field_.*", true); + assertQueryFails("SELECT * from (select * FROM (values 1) as t(k)) CROSS JOIN unnest(ARRAY[row(2, 3), row(3, 5)], ARRAY[row(2, 3), row(3, 5)], ARRAY[row(10, 13, 15), row(23, 25, 20)]) WITH ORDINALITY AS r(r1, r2, r3, r4, r5, r6, r7, ord)", + "Field not found: field_.*. Available fields are: field, field_.*", true); + + assertQueryFails("SELECT * from unnest(ARRAY[row(2, 3), row(3, 5)], ARRAY[row(2, 3), row(3, 5)]) AS r(r1, r2, r3, r4)", + "Field not found: field_.*. Available fields are: field, field_.*", true); + assertQueryFails("SELECT * from unnest(ARRAY[row(2, 3), row(3, 5)], ARRAY[row(2, 3), row(3, 5)]) WITH ORDINALITY AS r(r1, r2, r3, r4, ord)", + "Field not found: field_.*. Available fields are: field, field_.*", true); + } + + @Override + @Test + public void testMergeEmptyNonEmptyApproxSet() + { + assertQueryFails("SELECT cardinality(merge(c)) FROM (SELECT create_hll(custkey) c FROM orders UNION ALL SELECT empty_approx_set())", + CREATE_HLL_FUNCTION_NOT_REGISTERED, true); + } + + @Override + @Test + public void testSetUnionWithNulls() + { + // all nulls should return empty array to match behavior of array_distinct(flatten(array_agg(x))) + assertQueryFails( + "select set_union(x) from (values null, null, null) as t(x)", + "Unexpected type UNKNOWN", true); + // nulls inside arrays should be captured while pure nulls should be ignored + assertQueryFails( + "select set_union(x) from (values null, array[null], null) as t(x)", + "Unexpected type UNKNOWN", true); + // return null for empty rows + assertQueryFails( + "select set_union(x) from (values null, array[null], null) as t(x) where x != null", + "Unexpected type UNKNOWN", true); + } + + @Override + @Test + public void testSamplingJoinChain() + { + Session sessionWithKeyBasedSampling = Session.builder(getSession()) + .setSystemProperty(KEY_BASED_SAMPLING_ENABLED, "true") + .build(); + String query = "select count(1) FROM lineitem l left JOIN orders o ON l.orderkey = o.orderkey JOIN customer c ON o.custkey = c.custkey"; + + assertQuery(query, "select 60175"); + assertQueryFails(sessionWithKeyBasedSampling, query, "Scalar function name not registered: presto.default.key_sampling_percent, called with arguments", true); + } + + @Override + @Test + public void testMergeEmptyNonEmptyApproxSetWithSameMaxError() + { + assertQueryFails("SELECT cardinality(merge(c)) FROM (SELECT create_hll(custkey, 0.1) c FROM orders UNION ALL SELECT empty_approx_set(0.1))", + CREATE_HLL_FUNCTION_NOT_REGISTERED, true); + } + + @Override + @Test + public void testCorrelatedNonAggregationScalarSubqueries() + { + String subqueryReturnedTooManyRows = ".*Scalar sub-query has returned multiple rows.*"; + + assertQuery("SELECT (SELECT 1 WHERE a = 2) FROM (VALUES 1) t(a)", "SELECT null"); + assertQuery("SELECT (SELECT 2 WHERE a = 1) FROM (VALUES 1) t(a)", "SELECT 2"); + assertQueryFails( + "SELECT (SELECT 2 FROM (VALUES 3, 4) WHERE a = 1) FROM (VALUES 1) t(a)", + subqueryReturnedTooManyRows); + + // multiple subquery output projections + // TODO: Check why native query runner doesn't throw an error for below queries. + computeActual("SELECT name FROM nation n WHERE 'AFRICA' = (SELECT 'bleh' FROM region WHERE regionkey > n.regionkey)"); + computeActual("SELECT name FROM nation n WHERE 'AFRICA' = (SELECT name FROM region WHERE regionkey > n.regionkey)"); + assertQueryFails( + "SELECT name FROM nation n WHERE 1 = (SELECT 1 FROM region WHERE regionkey > n.regionkey)", + subqueryReturnedTooManyRows); + + // correlation used in subquery output + assertQueryFails( + "SELECT name FROM nation n WHERE 'AFRICA' = (SELECT n.name FROM region WHERE regionkey > n.regionkey)", + UNSUPPORTED_CORRELATED_SUBQUERY_ERROR_MSG); + + assertQuery( + "SELECT (SELECT 2 WHERE o.orderkey = 1) FROM orders o ORDER BY orderkey LIMIT 5", + "VALUES 2, null, null, null, null"); + // outputs plain correlated orderkey symbol which causes ambiguity with outer query orderkey symbol + assertQueryFails( + "SELECT (SELECT o.orderkey WHERE o.orderkey = 1) FROM orders o ORDER BY orderkey LIMIT 5", + UNSUPPORTED_CORRELATED_SUBQUERY_ERROR_MSG); + assertQueryFails( + "SELECT (SELECT o.orderkey * 2 WHERE o.orderkey = 1) FROM orders o ORDER BY orderkey LIMIT 5", + UNSUPPORTED_CORRELATED_SUBQUERY_ERROR_MSG); + // correlation used outside the subquery + assertQueryFails( + "SELECT o.orderkey, (SELECT o.orderkey * 2 WHERE o.orderkey = 1) FROM orders o ORDER BY orderkey LIMIT 5", + UNSUPPORTED_CORRELATED_SUBQUERY_ERROR_MSG); + + // aggregation with having +// TODO: uncomment below test once #8456 is fixed +// assertQuery("SELECT (SELECT avg(totalprice) FROM orders GROUP BY custkey, orderdate HAVING avg(totalprice) < a) FROM (VALUES 900) t(a)"); + + // correlation in predicate + assertQuery("SELECT name FROM nation n WHERE 'AFRICA' = (SELECT name FROM region WHERE regionkey = n.regionkey)"); + + // same correlation in predicate and projection + assertQueryFails( + "SELECT nationkey FROM nation n WHERE " + + "(SELECT n.regionkey * 2 FROM region r WHERE n.regionkey = r.regionkey) > 6", + UNSUPPORTED_CORRELATED_SUBQUERY_ERROR_MSG); + + // different correlation in predicate and projection + assertQueryFails( + "SELECT nationkey FROM nation n WHERE " + + "(SELECT n.nationkey * 2 FROM region r WHERE n.regionkey = r.regionkey) > 6", + UNSUPPORTED_CORRELATED_SUBQUERY_ERROR_MSG); + + // correlation used in subrelation + assertQuery( + "SELECT nationkey FROM nation n WHERE " + + "(SELECT regionkey * 2 FROM (SELECT regionkey FROM region r WHERE n.regionkey = r.regionkey)) > 6 " + + "ORDER BY 1 LIMIT 3", + "VALUES 4, 10, 11"); // h2 didn't make it + + // with duplicated rows + assertQuery( + "SELECT (SELECT name FROM nation WHERE nationkey = a) FROM (VALUES 1, 1, 2, 3) t(a)", + "VALUES 'ARGENTINA', 'ARGENTINA', 'BRAZIL', 'CANADA'"); // h2 didn't make it + + // returning null when nothing matched + assertQuery( + "SELECT (SELECT name FROM nation WHERE nationkey = a) FROM (VALUES 31) t(a)", + "VALUES null"); + + assertQuery( + "SELECT (SELECT r.name FROM nation n, region r WHERE r.regionkey = n.regionkey AND n.nationkey = a) FROM (VALUES 1) t(a)", + "VALUES 'AMERICA'"); + } + + @Override + @Test + public void testScalarSubquery() + { + // nested + assertQuery("SELECT (SELECT (SELECT (SELECT 1)))"); + + // TODO: Investigate error seen with disabled queries: 'expected types count (22) does not + // match actual column count (16)' + // aggregation + computeActual("SELECT * FROM lineitem WHERE orderkey = \n" + + "(SELECT max(orderkey) FROM orders)"); + + // no output + assertQuery("SELECT * FROM lineitem WHERE orderkey = \n" + + "(SELECT orderkey FROM orders WHERE 0=1)"); + + // no output matching with null test + computeActual("SELECT * FROM lineitem WHERE \n" + + "(SELECT orderkey FROM orders WHERE 0=1) " + + "is null"); + assertQuery("SELECT * FROM lineitem WHERE \n" + + "(SELECT orderkey FROM orders WHERE 0=1) " + + "is not null"); + + // subquery results and in in-predicate + assertQuery("SELECT (SELECT 1) IN (1, 2, 3)"); + assertQuery("SELECT (SELECT 1) IN ( 2, 3)"); + + // multiple subqueries + assertQuery("SELECT (SELECT 1) = (SELECT 3)"); + assertQuery("SELECT (SELECT 1) < (SELECT 3)"); + assertQuery("SELECT COUNT(*) FROM lineitem WHERE " + + "(SELECT min(orderkey) FROM orders)" + + "<" + + "(SELECT max(orderkey) FROM orders)"); + assertQuery("SELECT (SELECT 1), (SELECT 2), (SELECT 3)"); + + // distinct + assertQuery("SELECT DISTINCT orderkey FROM lineitem " + + "WHERE orderkey BETWEEN" + + " (SELECT avg(orderkey) FROM orders) - 10 " + + " AND" + + " (SELECT avg(orderkey) FROM orders) + 10"); + + // subqueries with joins + assertQuery("SELECT o1.orderkey, COUNT(*) " + + "FROM orders o1 " + + "INNER JOIN (SELECT * FROM orders ORDER BY orderkey LIMIT 10) o2 " + + "ON o1.orderkey " + + "BETWEEN (SELECT avg(orderkey) FROM orders) - 10 AND (SELECT avg(orderkey) FROM orders) + 10 " + + "GROUP BY o1.orderkey"); + assertQuery("SELECT o1.orderkey, COUNT(*) " + + "FROM (SELECT * FROM orders ORDER BY orderkey LIMIT 5) o1 " + + "LEFT JOIN (SELECT * FROM orders ORDER BY orderkey LIMIT 10) o2 " + + "ON o1.orderkey " + + "BETWEEN (SELECT avg(orderkey) FROM orders) - 10 AND (SELECT avg(orderkey) FROM orders) + 10 " + + "GROUP BY o1.orderkey"); + assertQuery("SELECT o1.orderkey, COUNT(*) " + + "FROM orders o1 RIGHT JOIN (SELECT * FROM orders ORDER BY orderkey LIMIT 10) o2 " + + "ON o1.orderkey " + + "BETWEEN (SELECT avg(orderkey) FROM orders) - 10 AND (SELECT avg(orderkey) FROM orders) + 10 " + + "GROUP BY o1.orderkey"); + assertQuery("SELECT DISTINCT COUNT(*) " + + "FROM (SELECT * FROM orders ORDER BY orderkey LIMIT 5) o1 " + + "FULL JOIN (SELECT * FROM orders ORDER BY orderkey LIMIT 10) o2 " + + "ON o1.orderkey " + + "BETWEEN (SELECT avg(orderkey) FROM orders) - 10 AND (SELECT avg(orderkey) FROM orders) + 10 " + + "GROUP BY o1.orderkey", + "VALUES 1, 10"); + + // subqueries with ORDER BY + assertQuery("SELECT orderkey, totalprice FROM orders ORDER BY (SELECT 2)"); + + // subquery returns multiple rows + String multipleRowsErrorMsg = ".*Expected single row of input.*"; + assertQueryFails("SELECT * FROM lineitem WHERE orderkey = (\n" + + "SELECT orderkey FROM orders ORDER BY totalprice)", + multipleRowsErrorMsg); + assertQueryFails("SELECT orderkey, totalprice FROM orders ORDER BY (VALUES 1, 2)", + multipleRowsErrorMsg); + + // exposes a bug in optimize hash generation because EnforceSingleNode does not + // support more than one column from the underlying query + assertQuery("SELECT custkey, (SELECT DISTINCT custkey FROM orders ORDER BY custkey LIMIT 1) FROM orders"); + + // cast scalar sub-query + assertQuery("SELECT 1.0/(SELECT 1), CAST(1.0 AS REAL)/(SELECT 1), 1/(SELECT 1)"); + assertQuery("SELECT 1.0 = (SELECT 1) AND 1 = (SELECT 1), 2.0 = (SELECT 1) WHERE 1.0 = (SELECT 1) AND 1 = (SELECT 1)"); + assertQuery("SELECT 1.0 = (SELECT 1), 2.0 = (SELECT 1), CAST(2.0 AS REAL) = (SELECT 1) WHERE 1.0 = (SELECT 1)"); + + // coerce correlated symbols + assertQuery("SELECT * FROM (VALUES 1) t(a) WHERE 1=(SELECT count(*) WHERE 1.0 = a)", "SELECT 1"); + assertQuery("SELECT * FROM (VALUES 1.0) t(a) WHERE 1=(SELECT count(*) WHERE 1 = a)", "SELECT 1.0"); + } + + @Override + @Test + public void testCorrelatedScalarSubqueries() + { + assertQuery("SELECT (SELECT n.nationkey + n.NATIONKEY) FROM nation n"); + assertQuery("SELECT (SELECT 2 * n.nationkey) FROM nation n"); + assertQuery("SELECT nationkey FROM nation n WHERE 2 = (SELECT 2 * n.nationkey)"); + assertQuery("SELECT nationkey FROM nation n ORDER BY (SELECT 2 * n.nationkey)"); + + // group by + assertQuery("SELECT max(n.regionkey), 2 * n.nationkey, (SELECT n.nationkey) FROM nation n GROUP BY n.nationkey"); + assertQuery( + "SELECT max(l.quantity), 2 * l.orderkey FROM lineitem l GROUP BY l.orderkey HAVING max(l.quantity) < (SELECT l.orderkey)"); + assertQuery("SELECT max(l.quantity), 2 * l.orderkey FROM lineitem l GROUP BY l.orderkey, (SELECT l.orderkey)"); + + // join + assertQuery("SELECT * FROM nation n1 JOIN nation n2 ON n1.nationkey = (SELECT n2.nationkey)"); + assertQueryFails( + "SELECT (SELECT l3.* FROM lineitem l2 CROSS JOIN (SELECT l1.orderkey) l3 LIMIT 1) FROM lineitem l1", + UNSUPPORTED_CORRELATED_SUBQUERY_ERROR_MSG); + + // subrelation + assertQuery( + "SELECT 1 FROM nation n WHERE 2 * nationkey - 1 = (SELECT * FROM (SELECT n.nationkey))", + "SELECT 1"); // h2 fails to parse this query + + // two level of nesting + assertQuery("SELECT * FROM nation n WHERE 2 = (SELECT (SELECT 2 * n.nationkey))"); + + // explicit LIMIT in subquery + assertQueryFails( + "SELECT (SELECT count(*) FROM (VALUES (7,1)) t(orderkey, value) WHERE orderkey = corr_key LIMIT 1) FROM (values 7) t(corr_key)", + UNSUPPORTED_CORRELATED_SUBQUERY_ERROR_MSG); + + assertQuery( + "SELECT (SELECT count(*) FROM (VALUES (7,1)) t(orderkey, value) WHERE orderkey = corr_key GROUP BY value LIMIT 2) FROM (values 7) t(corr_key)"); + + // Limit(1) and non-constant output symbol of the subquery (count) + assertQueryFails("SELECT (SELECT count(*) FROM (VALUES (7,1), (7,2)) t(orderkey, value) WHERE orderkey = corr_key GROUP BY value LIMIT 1) FROM (values 7) t(corr_key)", + UNSUPPORTED_CORRELATED_SUBQUERY_ERROR_MSG); + } + + @Override + @Test + public void testLikePrefixAndSuffixWithChars() + { + assertQueryFails("select x like 'abc%' from (values CAST ('abc' AS CHAR(3)), CAST ('def' AS CHAR(3)), CAST ('bcd' AS CHAR(3))) T(x)", CHAR_TYPE_UNSUPPORTED_ERROR, true); + assertQueryFails("select x like '%abc%' from (values CAST ('xabcy' AS CHAR(5)), CAST ('abxabcdef' AS CHAR(9)), CAST ('bcd' AS CHAR(3)), CAST ('xabcyabcz' AS CHAR(9))) T(x)", CHAR_TYPE_UNSUPPORTED_ERROR, true); + assertQueryFails( + "select x like '%abc' from (values CAST('xa bc' AS CHAR(5)), CAST ('xabcy' AS CHAR(5)), CAST ('abcd' AS CHAR(4)), CAST ('xabc' AS CHAR(4)), CAST (' xabc' AS CHAR(5))) T(x)", CHAR_TYPE_UNSUPPORTED_ERROR, true); + assertQueryFails("select x like '%ab_c' from (values CAST('xa bc' AS CHAR(5)), CAST ('xabcy' AS CHAR(5)), CAST ('abcd' AS CHAR(4))) T(x)", CHAR_TYPE_UNSUPPORTED_ERROR, true); + assertQueryFails("select x like '%' from (values CAST('xa bc' AS CHAR(5)), CAST ('xabcy' AS CHAR(5)), CAST ('abcd' AS CHAR(4))) T(x)", CHAR_TYPE_UNSUPPORTED_ERROR, true); + assertQueryFails("select x like '%_%' from (values CAST('xa bc' AS CHAR(5)), CAST ('xabcy' AS CHAR(5)), CAST ('abcd' AS CHAR(4))) T(x)", CHAR_TYPE_UNSUPPORTED_ERROR, true); + assertQueryFails("select x like '%a%' from (values CAST('xa bc' AS CHAR(5)), CAST ('xabcy' AS CHAR(5)), CAST ('abcd' AS CHAR(4))) T(x)", CHAR_TYPE_UNSUPPORTED_ERROR, true); + assertQueryFails("select x like '%acd%xy%' from (values CAST('xa bc' AS CHAR(5)), CAST ('xabcy' AS CHAR(5)), CAST ('abcd' AS CHAR(4))) T(x)", CHAR_TYPE_UNSUPPORTED_ERROR, true); + } + + @Override + public void testMapBlockBug() + { + assertQueryFails(" VALUES(MAP_AGG(12345,123))", "Scalar function name not registered: presto.default.map_agg", true); + } + + @Override + public void testMergeKHyperLogLog() + { + assertQueryFails("select k1, cardinality(merge(khll)), uniqueness_distribution(merge(khll)) from (select k1, k2, khyperloglog_agg(v1, v2) khll from (values (1, 1, 2, 3), (1, 1, 4, 0), (1, 2, 90, 20), (1, 2, 87, 1), " + + "(2, 1, 11, 30), (2, 1, 11, 11), (2, 2, 9, 1), (2, 2, 87, 2)) t(k1, k2, v1, v2) group by k1, k2) group by k1", KHLL_TIMESTAMP_TZ_TYPE_UNSUPPORTED_ERROR, true); + + assertQueryFails("select cardinality(merge(khll)), uniqueness_distribution(merge(khll)) from (select k1, k2, khyperloglog_agg(v1, v2) khll from (values (1, 1, 2, 3), (1, 1, 4, 0), (1, 2, 90, 20), (1, 2, 87, 1), " + + "(2, 1, 11, 30), (2, 1, 11, 11), (2, 2, 9, 1), (2, 2, 87, 2)) t(k1, k2, v1, v2) group by k1, k2)", KHLL_TIMESTAMP_TZ_TYPE_UNSUPPORTED_ERROR, true); + } + + @Override + @Test + public void testRemoveMapCastFailure() + { + Session enableOptimization = Session.builder(getSession()) + .setSystemProperty(REMOVE_MAP_CAST, "true") + .build(); + assertQueryFails(enableOptimization, "select feature[key] from (values (map(array[cast(1 as integer), 2, 3, 4], array[0.3, 0.5, 0.9, 0.1]), cast(2 as bigint)), (map(array[cast(1 as integer), 2, 3, 4], array[0.3, 0.5, 0.9, 0.1]), 400000000000)) t(feature, key)", + "Cannot cast BIGINT.*to INTEGER. Overflow during arithmetic conversion", true); + } + + @Override + @Test + public void testRemoveRedundantCastToVarcharInJoinClause() + { + Session session = Session.builder(getSession()) + .setSystemProperty(REMOVE_REDUNDANT_CAST_TO_VARCHAR_IN_JOIN, "true") + .build(); + // TODO: Check why results do not match for disabled queries. + // Trigger optimization + computeActual("select * from orders o join customer c on cast(o.custkey as varchar) = cast(c.custkey as varchar)"); + assertQuery(session, "select o.orderkey, c.name from orders o join customer c on cast(o.custkey as varchar) = cast(c.custkey as varchar)"); + computeActual(session, "select *, cast(o.custkey as varchar), cast(c.custkey as varchar) from orders o join customer c on cast(o.custkey as varchar) = cast(c.custkey as varchar)"); + assertQuery(session, "select r.custkey, r.orderkey, r.name, n.nationkey from (select o.custkey, o.orderkey, c.name from orders o join customer c on cast(o.custkey as varchar) = cast(c.custkey as varchar)) r, nation n"); + // Do not trigger optimization + assertQuery(session, "select * from customer c join orders o on cast(acctbal as varchar) = cast(totalprice as varchar)"); + } + + @Override + @Test + public void testSameAggregationWithAndWithoutFilter() + { + Session enableOptimization = Session.builder(getSession()) + .setSystemProperty(MERGE_AGGREGATIONS_WITH_AND_WITHOUT_FILTER, "true") + .build(); + Session disableOptimization = Session.builder(getSession()) + .setSystemProperty(MERGE_AGGREGATIONS_WITH_AND_WITHOUT_FILTER, "false") + .build(); + + // TODO: Check why results do not match with MERGE_AGGREGATIONS_WITH_AND_WITHOUT_FILTER enabled. + // Disabled tests to be enabled after fixing this issue. + computeActual(getQueryRunner(), enableOptimization, "select regionkey, count(name) filter (where name like '%N%') n_nations, count(name) all_nations from nation group by regionkey"); + assertQuery(enableOptimization, "select count(name) filter (where name like '%N%') n_nations, count(name) all_nations from nation", "values (15,25)"); + assertQuery(enableOptimization, "select count(1), count(1) filter (where k > 5) from (values 1, null, 3, 5, null, 8, 10) t(k)", "values (7, 2)"); + + String sql = "select regionkey, count(name) filter (where name like '%N%') n_nations, count(name) all_nations from nation group by regionkey"; + // MaterializedResult resultWithOptimization = computeActual(enableOptimization, sql); + // MaterializedResult resultWithoutOptimization = computeActual(disableOptimization, sql); + // assertEqualsIgnoreOrder(resultWithOptimization, resultWithoutOptimization); + + sql = "select count(name) filter (where name like '%N%') n_nations, count(name) all_nations from nation"; + // resultWithOptimization = computeActual(enableOptimization, sql); + // resultWithoutOptimization = computeActual(disableOptimization, sql); + // assertEqualsIgnoreOrder(resultWithOptimization, resultWithoutOptimization); + + sql = "select partkey, sum(quantity), sum(quantity) filter (where discount > 0.1) from lineitem group by grouping sets((), (partkey))"; + MaterializedResult resultWithOptimization = computeActual(enableOptimization, sql); + MaterializedResult resultWithoutOptimization = computeActual(disableOptimization, sql); + assertEqualsIgnoreOrder(resultWithOptimization, resultWithoutOptimization); + + // multiple aggregations in query + sql = "select partkey, sum(quantity), sum(quantity) filter (where discount < 0.05), sum(linenumber), sum(linenumber) filter (where discount < 0.05) from lineitem group by partkey"; + resultWithOptimization = computeActual(enableOptimization, sql); + resultWithoutOptimization = computeActual(disableOptimization, sql); + assertEqualsIgnoreOrder(resultWithOptimization, resultWithoutOptimization); + // aggregations in multiple levels + sql = "select partkey, avg(sum), avg(sum) filter (where tax < 0.05), avg(filtersum) from (select partkey, suppkey, sum(quantity) sum, sum(quantity) filter (where discount > 0.05) filtersum, max(tax) tax from lineitem where partkey=1598 group by partkey, suppkey) t group by partkey"; + resultWithOptimization = computeActual(enableOptimization, sql); + resultWithoutOptimization = computeActual(disableOptimization, sql); + assertEqualsIgnoreOrder(resultWithOptimization, resultWithoutOptimization); + // global aggregation + sql = "select sum(quantity), sum(quantity) filter (where discount < 0.05) from lineitem"; + resultWithOptimization = computeActual(enableOptimization, sql); + resultWithoutOptimization = computeActual(disableOptimization, sql); + assertEqualsIgnoreOrder(resultWithOptimization, resultWithoutOptimization); + // order by + sql = "select partkey, array_agg(suppkey order by suppkey), array_agg(suppkey order by suppkey) filter (where discount > 0.05) from lineitem group by partkey"; + resultWithOptimization = computeActual(enableOptimization, sql); + resultWithoutOptimization = computeActual(disableOptimization, sql); + assertEqualsIgnoreOrder(resultWithOptimization, resultWithoutOptimization); + // grouping sets + sql = "SELECT partkey, suppkey, sum(quantity), sum(quantity) filter (where discount > 0.05) from lineitem group by grouping sets((), (partkey), (partkey, suppkey))"; + resultWithOptimization = computeActual(enableOptimization, sql); + resultWithoutOptimization = computeActual(disableOptimization, sql); + assertEqualsIgnoreOrder(resultWithOptimization, resultWithoutOptimization); + // aggregation over union + sql = "SELECT partkey, sum(quantity), sum(quantity) filter (where orderkey > 0) from (select quantity, orderkey, partkey from lineitem union all select totalprice as quantity, orderkey, custkey as partkey from orders) group by partkey"; + resultWithOptimization = computeActual(enableOptimization, sql); + resultWithoutOptimization = computeActual(disableOptimization, sql); + assertEqualsIgnoreOrder(resultWithOptimization, resultWithoutOptimization); + // aggregation over join + sql = "select custkey, sum(quantity), sum(quantity) filter (where tax < 0.05) from lineitem l join orders o on l.orderkey=o.orderkey group by custkey"; + resultWithOptimization = computeActual(enableOptimization, sql); + resultWithoutOptimization = computeActual(disableOptimization, sql); + assertEqualsIgnoreOrder(resultWithOptimization, resultWithoutOptimization); + } + + @Override + @Test + public void testSetAggIndeterminateArrays() + { + // union all is to force usage of the serialized state + assertQueryFails("SELECT unnested from (SELECT set_agg(x) as agg_result from (" + + "SELECT ARRAY[ARRAY[null, 2]] x " + + "UNION ALL " + + "SELECT ARRAY[null, ARRAY[1, null]] " + + "UNION ALL " + + "SELECT ARRAY[ARRAY[null, 2]])) " + + "CROSS JOIN unnest(agg_result) as r(unnested)", + ARRAY_COMPARISON_UNSUPPORTED_ERROR, true); + } + + @Override + @Test + public void testSetAggIndeterminateRows() + { + // union all is to force usage of the serialized state + assertQueryFails("SELECT unnested from (SELECT set_agg(x) as agg_result from (" + + "SELECT ARRAY[CAST(row(null, 2) AS ROW(INTEGER, INTEGER))] x " + + "UNION ALL " + + "SELECT ARRAY[null, CAST(row(1, null) AS ROW(INTEGER, INTEGER))] " + + "UNION ALL " + + "SELECT ARRAY[CAST(row(null, 2) AS ROW(INTEGER, INTEGER))])) " + + "CROSS JOIN unnest(agg_result) as r(unnested)", + ARRAY_COMPARISON_UNSUPPORTED_ERROR, true); + } + + @Override + @Test + public void testSetUnionIndeterminateRows() + { + // union all is to force usage of the serialized state + assertQueryFails("SELECT c1, c2 from (SELECT set_union(x) as agg_result from (" + + "SELECT ARRAY[CAST(row(null, 2) AS ROW(INTEGER, INTEGER))] x " + + "UNION ALL " + + "SELECT ARRAY[null, CAST(row(1, null) AS ROW(INTEGER, INTEGER))] " + + "UNION ALL " + + "SELECT ARRAY[CAST(row(null, 2) AS ROW(INTEGER, INTEGER))])) " + + "CROSS JOIN unnest(agg_result) as r(c1, c2)", + ".*Field not found.*", true); + } + + @Override + @Test + public void testLambdaInAggregation() + { + assertQuery("SELECT id, reduce_agg(value, 0, (a, b) -> a + b+0, (a, b) -> a + b) FROM ( VALUES (1, 2), (1, 3), (1, 4), (2, 20), (2, 30), (2, 40) ) AS t(id, value) GROUP BY id", "values (1, 9), (2, 90)"); + assertQuery("SELECT id, 's' || reduce_agg(value, '', (a, b) -> concat(a, b, 's'), (a, b) -> concat(a, b, 's')) FROM ( VALUES (1, '2'), (1, '3'), (1, '4'), (2, '20'), (2, '30'), (2, '40') ) AS t(id, value) GROUP BY id", + "values (1, 's2s3ss4ss'), (2, 's20s30ss40ss')"); + assertQueryFails("SELECT id, reduce_agg(value, array[id, value], (a, b) -> a || b, (a, b) -> a || b) FROM ( VALUES (1, 2), (1, 3), (1, 4), (2, 20), (2, 30), (2, 40) ) AS t(id, value) GROUP BY id", + ".*REDUCE_AGG only supports non-NULL literal as the initial value.*"); + } + + @Override + @Test + public void testPreserveAssignmentsInJoin() + { + // The following two timestamps represent the same point in time but with different time zones + String timestampLosAngeles = "2001-08-22 03:04:05.321 America/Los_Angeles"; + String timestampNewYork = "2001-08-22 06:04:05.321 America/New_York"; + Set> rows = computeActual("WITH source AS (" + + "SELECT * FROM (" + + " VALUES" + + " (TIMESTAMP '" + timestampLosAngeles + "')," + + " (TIMESTAMP '" + timestampNewYork + "')" + + ") AS tbl (tstz)" + + ")" + + "SELECT * FROM source a JOIN source b ON a.tstz = b.tstz").getMaterializedRows().stream() + .map(MaterializedRow::getFields) + .collect(toImmutableSet()); + // TODO: Enable assertion after fixing result mismatch + /* Assert.assertEquals(rows, + ImmutableSet.of( + ImmutableList.of(zonedDateTime(timestampLosAngeles), zonedDateTime(timestampLosAngeles)), + ImmutableList.of(zonedDateTime(timestampLosAngeles), zonedDateTime(timestampNewYork)), + ImmutableList.of(zonedDateTime(timestampNewYork), zonedDateTime(timestampLosAngeles)), + ImmutableList.of(zonedDateTime(timestampNewYork), zonedDateTime(timestampNewYork)))); */ + } +} diff --git a/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestAggregations.java b/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestAggregations.java new file mode 100644 index 0000000000000..53ee6e496354b --- /dev/null +++ b/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestAggregations.java @@ -0,0 +1,43 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.nativetests; + +import com.facebook.presto.nativeworker.NativeQueryRunnerUtils; +import com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils; +import com.facebook.presto.testing.QueryRunner; + +public class TestAggregations + extends AbstractTestAggregationsNative +{ + private static final String storageFormat = "PARQUET"; + + @Override + protected QueryRunner createQueryRunner() throws Exception + { + return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(true, storageFormat); + } + + @Override + protected void createTables() + { + try { + QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.createJavaQueryRunner(storageFormat); + NativeQueryRunnerUtils.createAllTables(javaQueryRunner, false); + javaQueryRunner.close(); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestDistributedEngineOnlyQueries.java b/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestDistributedEngineOnlyQueries.java new file mode 100644 index 0000000000000..fccfeaad5c39e --- /dev/null +++ b/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestDistributedEngineOnlyQueries.java @@ -0,0 +1,122 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.nativetests; + +import com.facebook.presto.Session; +import com.facebook.presto.common.type.TimeZoneKey; +import com.facebook.presto.nativeworker.NativeQueryRunnerUtils; +import com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils; +import com.facebook.presto.testing.QueryRunner; +import com.facebook.presto.tests.AbstractTestEngineOnlyQueries; +import org.intellij.lang.annotations.Language; +import org.testng.annotations.Test; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Objects; + +import static com.google.common.base.Preconditions.checkState; +import static org.testng.Assert.assertEquals; + +public class TestDistributedEngineOnlyQueries + extends AbstractTestEngineOnlyQueries +{ + private static final String timeTypeUnsupportedError = ".*Failed to parse type \\[time.*"; + + private static final String storageFormat = "PARQUET"; + + @Override + protected QueryRunner createQueryRunner() throws Exception + { + return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(true, storageFormat); + } + + @Override + protected void createTables() + { + try { + QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.createJavaQueryRunner(storageFormat); + NativeQueryRunnerUtils.createAllTables(javaQueryRunner, false); + javaQueryRunner.close(); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + @Test + public void testTimeLiterals() + { + Session chicago = Session.builder(getSession()).setTimeZoneKey(TimeZoneKey.getTimeZoneKey("America/Chicago")).build(); + Session kathmandu = Session.builder(getSession()).setTimeZoneKey(TimeZoneKey.getTimeZoneKey("Asia/Kathmandu")).build(); + + assertEquals(computeScalar("SELECT DATE '2013-03-22'"), LocalDate.of(2013, 3, 22)); + assertQuery("SELECT DATE '2013-03-22'"); + assertQuery(chicago, "SELECT DATE '2013-03-22'"); + assertQuery(kathmandu, "SELECT DATE '2013-03-22'"); + + assertQueryFails("SELECT TIME '3:04:05'", timeTypeUnsupportedError, true); + assertQueryFails("SELECT TIME '3:04:05.123'", timeTypeUnsupportedError, true); + assertQueryFails("SELECT TIME '3:04:05'", timeTypeUnsupportedError, true); + assertQueryFails("SELECT TIME '0:04:05'", timeTypeUnsupportedError, true); + assertQueryFails(chicago, "SELECT TIME '3:04:05'", timeTypeUnsupportedError, true); + assertQueryFails(kathmandu, "SELECT TIME '3:04:05'", timeTypeUnsupportedError, true); + + assertQueryFails("SELECT TIME '01:02:03.400 Z'", timeTypeUnsupportedError, true); + assertQueryFails("SELECT TIME '01:02:03.400 UTC'", timeTypeUnsupportedError, true); + assertQueryFails("SELECT TIME '3:04:05 +06:00'", timeTypeUnsupportedError, true); + assertQueryFails("SELECT TIME '3:04:05 +0507'", timeTypeUnsupportedError, true); + assertQueryFails("SELECT TIME '3:04:05 +03'", timeTypeUnsupportedError, true); + + assertEquals(computeScalar("SELECT TIMESTAMP '1960-01-22 3:04:05'"), LocalDateTime.of(1960, 1, 22, 3, 4, 5)); + assertEquals(computeScalar("SELECT TIMESTAMP '1960-01-22 3:04:05.123'"), LocalDateTime.of(1960, 1, 22, 3, 4, 5, 123_000_000)); + assertQuery("SELECT TIMESTAMP '1960-01-22 3:04:05'"); + assertQuery("SELECT TIMESTAMP '1960-01-22 3:04:05.123'"); + // TODO #7122 assertQuery(chicago, "SELECT TIMESTAMP '1960-01-22 3:04:05.123'"); + // TODO #7122 assertQuery(kathmandu, "SELECT TIMESTAMP '1960-01-22 3:04:05.123'"); + + assertEquals(computeScalar("SELECT TIMESTAMP '1960-01-22 3:04:05 +06:00'"), ZonedDateTime.of(1960, 1, 22, 3, 4, 5, 0, ZoneOffset.ofHoursMinutes(6, 0))); + } + + @Override + @Test + public void testLocallyUnrepresentableTimeLiterals() + { + LocalDateTime localTimeThatDidNotExist = LocalDateTime.of(2017, 4, 2, 2, 10); + checkState(ZoneId.systemDefault().getRules().getValidOffsets(localTimeThatDidNotExist).isEmpty(), "This test assumes certain JVM time zone"); + // This tests that both Presto runner and H2 can return TIMESTAMP value that never happened in JVM's zone (e.g. is not representable using java.sql.Timestamp) + @Language("SQL") String sql = DateTimeFormatter.ofPattern("'SELECT TIMESTAMP '''uuuu-MM-dd HH:mm:ss''").format(localTimeThatDidNotExist); + assertEquals(computeScalar(sql), localTimeThatDidNotExist); // this tests Presto and the QueryRunner + assertQuery(sql); // this tests H2QueryRunner + + LocalDate localDateThatDidNotHaveMidnight = LocalDate.of(1970, 1, 1); + checkState(ZoneId.systemDefault().getRules().getValidOffsets(localDateThatDidNotHaveMidnight.atStartOfDay()).isEmpty(), "This test assumes certain JVM time zone"); + // This tests that both Presto runner and H2 can return DATE value for a day which midnight never happened in JVM's zone (e.g. is not exactly representable using java.sql.Date) + sql = DateTimeFormatter.ofPattern("'SELECT DATE '''uuuu-MM-dd''").format(localDateThatDidNotHaveMidnight); + assertEquals(computeScalar(sql), localDateThatDidNotHaveMidnight); // this tests Presto and the QueryRunner + assertQuery(sql); // this tests H2QueryRunner + + LocalTime localTimeThatDidNotOccurOn19700101 = LocalTime.of(0, 10); + checkState(ZoneId.systemDefault().getRules().getValidOffsets(localTimeThatDidNotOccurOn19700101.atDate(LocalDate.ofEpochDay(0))).isEmpty(), "This test assumes certain JVM time zone"); + checkState(!Objects.equals(java.sql.Time.valueOf(localTimeThatDidNotOccurOn19700101).toLocalTime(), localTimeThatDidNotOccurOn19700101), "This test assumes certain JVM time zone"); + sql = DateTimeFormatter.ofPattern("'SELECT TIME '''HH:mm:ss''").format(localTimeThatDidNotOccurOn19700101); + assertQueryFails(sql, timeTypeUnsupportedError, true); + } +} diff --git a/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestDistributedQueriesNoHashGeneration.java b/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestDistributedQueriesNoHashGeneration.java new file mode 100644 index 0000000000000..ee57b560ed60b --- /dev/null +++ b/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestDistributedQueriesNoHashGeneration.java @@ -0,0 +1,47 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.nativetests; + +import com.facebook.presto.nativeworker.NativeQueryRunnerUtils; +import com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils; +import com.facebook.presto.testing.QueryRunner; +import com.google.common.collect.ImmutableMap; + +public class TestDistributedQueriesNoHashGeneration + extends AbstractTestQueriesNative +{ + private static final String storageFormat = "PARQUET"; + + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + return PrestoNativeQueryRunnerUtils.createNativeQueryRunner( + ImmutableMap.of(), + ImmutableMap.of("optimizer.optimize-hash-generation", "false"), storageFormat); + } + + @Override + protected void createTables() + { + try { + QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.createJavaQueryRunner(storageFormat); + NativeQueryRunnerUtils.createAllTables(javaQueryRunner, false); + javaQueryRunner.close(); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestNonIterativeDistributedQueries.java b/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestNonIterativeDistributedQueries.java new file mode 100644 index 0000000000000..214914e2628e7 --- /dev/null +++ b/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestNonIterativeDistributedQueries.java @@ -0,0 +1,49 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.nativetests; + +import com.facebook.presto.nativeworker.NativeQueryRunnerUtils; +import com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils; +import com.facebook.presto.testing.QueryRunner; +import com.google.common.collect.ImmutableMap; + +/** + * Test that Presto works with {@link com.facebook.presto.sql.planner.iterative.IterativeOptimizer} disabled. + */ +public class TestNonIterativeDistributedQueries + extends AbstractTestQueriesNative +{ + private static final String storageFormat = "PARQUET"; + + @Override + protected QueryRunner createQueryRunner() throws Exception + { + return PrestoNativeQueryRunnerUtils.createNativeQueryRunner( + ImmutableMap.of("experimental.iterative-optimizer-enabled", "false"), + ImmutableMap.of(), storageFormat); + } + + @Override + protected void createTables() + { + try { + QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.createJavaQueryRunner(storageFormat); + NativeQueryRunnerUtils.createAllTables(javaQueryRunner, false); + javaQueryRunner.close(); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestOptimizeMixedDistinctAggregations.java b/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestOptimizeMixedDistinctAggregations.java new file mode 100644 index 0000000000000..8ee9f35f9f601 --- /dev/null +++ b/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestOptimizeMixedDistinctAggregations.java @@ -0,0 +1,55 @@ + +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.nativetests; + +import com.facebook.presto.nativeworker.NativeQueryRunnerUtils; +import com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils; +import com.facebook.presto.testing.QueryRunner; +import com.google.common.collect.ImmutableMap; + +public class TestOptimizeMixedDistinctAggregations + extends AbstractTestAggregationsNative +{ + private static final String storageFormat = "PARQUET"; + + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + return PrestoNativeQueryRunnerUtils.createNativeQueryRunner( + ImmutableMap.of(), + ImmutableMap.of("optimizer.optimize-mixed-distinct-aggregations", "true"), storageFormat); + } + + @Override + protected void createTables() + { + try { + QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.createJavaQueryRunner(storageFormat); + NativeQueryRunnerUtils.createAllTables(javaQueryRunner, false); + javaQueryRunner.close(); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void testCountDistinct() + { + assertQuery("SELECT COUNT(DISTINCT custkey + 1) FROM orders", "SELECT COUNT(*) FROM (SELECT DISTINCT custkey + 1 FROM orders) t"); + assertQuery("SELECT COUNT(DISTINCT linenumber), COUNT(*) from lineitem where linenumber < 0"); + } +} diff --git a/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestOrderByQueries.java b/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestOrderByQueries.java new file mode 100644 index 0000000000000..836f0816498d9 --- /dev/null +++ b/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestOrderByQueries.java @@ -0,0 +1,54 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.nativetests; + +import com.facebook.presto.nativeworker.NativeQueryRunnerUtils; +import com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils; +import com.facebook.presto.testing.QueryRunner; +import com.facebook.presto.tests.AbstractTestOrderByQueries; +import org.testng.annotations.Test; + +public class TestOrderByQueries + extends AbstractTestOrderByQueries +{ + private static final String storageFormat = "PARQUET"; + + @Override + protected QueryRunner createQueryRunner() throws Exception + { + return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(true, storageFormat); + } + + @Override + protected void createTables() + { + try { + QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.createJavaQueryRunner(storageFormat); + NativeQueryRunnerUtils.createAllTables(javaQueryRunner, false); + javaQueryRunner.close(); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + @Test + public void testOrderByWithOutputColumnReferenceInLambdas() + { + assertQueryFails("SELECT x AS y FROM (values (1,2), (2,3)) t(x, y) GROUP BY x ORDER BY apply(x, x -> -x) + 2*x", ".*Scalar function name not registered: presto.default.apply.*"); + assertQueryFails("SELECT -y AS x FROM (values (1,2), (2,3)) t(x, y) GROUP BY y ORDER BY apply(x, x -> -x)", ".*Scalar function name not registered: presto.default.apply.*"); + assertQueryFails("SELECT -y AS x FROM (values (1,2), (2,3)) t(x, y) GROUP BY y ORDER BY sum(apply(-y, x -> x * 1.0))", ".*Scalar function name not registered: presto.default.apply.*"); + } +} diff --git a/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestRepartitionQueries.java b/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestRepartitionQueries.java new file mode 100644 index 0000000000000..fa5d2738d3cca --- /dev/null +++ b/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestRepartitionQueries.java @@ -0,0 +1,82 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.nativetests; + +import com.facebook.presto.nativeworker.NativeQueryRunnerUtils; +import com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils; +import com.facebook.presto.testing.QueryRunner; +import com.facebook.presto.tests.AbstractTestRepartitionQueries; +import org.testng.annotations.Test; + +public class TestRepartitionQueries + extends AbstractTestRepartitionQueries +{ + private static final String storageFormat = "PARQUET"; + + @Override + protected QueryRunner createQueryRunner() throws Exception + { + return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(true, storageFormat); + } + + @Override + protected void createTables() + { + try { + QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.createJavaQueryRunner(storageFormat); + NativeQueryRunnerUtils.createAllTables(javaQueryRunner, false); + javaQueryRunner.close(); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + @Test + public void testIpAddress() + { + // TODO: Enable this query after result mismatch is fixed + computeActual("WITH lineitem_ex AS \n" + + "(\n" + + "SELECT\n" + + " partkey,\n" + + " CAST(\n" + + " CONCAT(\n" + + " CONCAT(\n" + + " CONCAT(\n" + + " CONCAT(\n" + + " CONCAT(\n" + + " CONCAT(CAST((orderkey % 255) AS VARCHAR), '.'),\n" + + " CAST((partkey % 255) AS VARCHAR)\n" + + " ),\n" + + " '.'\n" + + " ),\n" + + " CAST(suppkey AS VARCHAR)\n" + + " ),\n" + + " '.'\n" + + " ),\n" + + " CAST(linenumber AS VARCHAR)\n" + + " ) AS ipaddress\n" + + " ) AS ip\n" + + " FROM lineitem\n" + + " )\n" + + "SELECT\n" + + " CHECKSUM(l.ip) \n" + + "FROM lineitem_ex l,\n" + + " partsupp p\n" + + "WHERE\n" + + " l.partkey = p.partkey"); + } +} diff --git a/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestRepartitionQueriesWithSmallPages.java b/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestRepartitionQueriesWithSmallPages.java new file mode 100644 index 0000000000000..a7ff3540655d1 --- /dev/null +++ b/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestRepartitionQueriesWithSmallPages.java @@ -0,0 +1,86 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.nativetests; + +import com.facebook.presto.nativeworker.NativeQueryRunnerUtils; +import com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils; +import com.facebook.presto.testing.QueryRunner; +import com.facebook.presto.tests.AbstractTestRepartitionQueries; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +public class TestRepartitionQueriesWithSmallPages + extends AbstractTestRepartitionQueries +{ + private static final String storageFormat = "PARQUET"; + + @Override + protected QueryRunner createQueryRunner() throws Exception + { + return PrestoNativeQueryRunnerUtils.createNativeQueryRunner( + ImmutableMap.of("experimental.optimized-repartitioning", "true", + // Use small SerializedPages to force flushing + "driver.max-page-partitioning-buffer-size", "200B"), ImmutableMap.of(), storageFormat); + } + + @Override + protected void createTables() + { + try { + QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.createJavaQueryRunner(storageFormat); + NativeQueryRunnerUtils.createAllTables(javaQueryRunner, false); + javaQueryRunner.close(); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + @Test + public void testIpAddress() + { + // TODO: Enable this query after result mismatch is fixed + computeActual("WITH lineitem_ex AS \n" + + "(\n" + + "SELECT\n" + + " partkey,\n" + + " CAST(\n" + + " CONCAT(\n" + + " CONCAT(\n" + + " CONCAT(\n" + + " CONCAT(\n" + + " CONCAT(\n" + + " CONCAT(CAST((orderkey % 255) AS VARCHAR), '.'),\n" + + " CAST((partkey % 255) AS VARCHAR)\n" + + " ),\n" + + " '.'\n" + + " ),\n" + + " CAST(suppkey AS VARCHAR)\n" + + " ),\n" + + " '.'\n" + + " ),\n" + + " CAST(linenumber AS VARCHAR)\n" + + " ) AS ipaddress\n" + + " ) AS ip\n" + + " FROM lineitem\n" + + " )\n" + + "SELECT\n" + + " CHECKSUM(l.ip) \n" + + "FROM lineitem_ex l,\n" + + " partsupp p\n" + + "WHERE\n" + + " l.partkey = p.partkey"); + } +} diff --git a/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestTpchDistributedQueries.java b/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestTpchDistributedQueries.java new file mode 100644 index 0000000000000..d6afb3f46c50a --- /dev/null +++ b/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestTpchDistributedQueries.java @@ -0,0 +1,98 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.nativetests; + +import com.facebook.presto.nativeworker.NativeQueryRunnerUtils; +import com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils; +import com.facebook.presto.testing.QueryRunner; +import com.google.common.base.Strings; +import org.intellij.lang.annotations.Language; +import org.testng.annotations.Test; + +import static org.testng.Assert.assertTrue; + +public class TestTpchDistributedQueries + extends AbstractTestQueriesNative +{ + private static final String storageFormat = "PARQUET"; + + @Override + protected QueryRunner createQueryRunner() throws Exception + { + return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(true, storageFormat); + } + + @Override + protected void createTables() + { + try { + QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.createJavaQueryRunner(storageFormat); + NativeQueryRunnerUtils.createAllTables(javaQueryRunner, false); + javaQueryRunner.close(); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Test + public void testTooLongQuery() + { + // Generate a super-long query: SELECT x,x,x,x,x,... FROM (VALUES 1,2,3,4,5) t(x) + @Language("SQL") String longQuery = "SELECT x" + Strings.repeat(",x", 500_000) + " FROM (VALUES 1,2,3,4,5) t(x)"; + assertQueryFails(longQuery, "Query text length \\(1000037\\) exceeds the maximum length \\(1000000\\)"); + } + + @Test + public void testAnalyzePropertiesSystemTable() + { + assertQuery("SELECT COUNT(*) FROM system.metadata.analyze_properties WHERE catalog_name = 'tpch'", "SELECT 0"); + } + + @Test + public void testAnalyze() + { + assertUpdate("ANALYZE orders", 15000); + assertQueryFails("ANALYZE orders WITH (foo = 'bar')", ".* does not support analyze property 'foo'.*"); + } + + @Test + public void testTooManyStages() + { + @Language("SQL") String query = "WITH\n" + + " t1 AS (SELECT nationkey AS x FROM nation where name='UNITED STATES'),\n" + + " t2 AS (SELECT a.x+b.x+c.x+d.x AS x FROM t1 a, t1 b, t1 c, t1 d),\n" + + " t3 AS (SELECT a.x+b.x+c.x+d.x AS x FROM t2 a, t2 b, t2 c, t2 d),\n" + + " t4 AS (SELECT a.x+b.x+c.x+d.x AS x FROM t3 a, t3 b, t3 c, t3 d),\n" + + " t5 AS (SELECT a.x+b.x+c.x+d.x AS x FROM t4 a, t4 b, t4 c, t4 d)\n" + + "SELECT x FROM t5\n"; + assertQueryFails(query, "Number of stages in the query \\([0-9]+\\) exceeds the allowed maximum \\([0-9]+\\).*"); + } + + @Test + public void testTableSampleSystem() + { + int total = computeActual("SELECT orderkey FROM orders").getMaterializedRows().size(); + + boolean sampleSizeFound = false; + for (int i = 0; i < 100; i++) { + int sampleSize = computeActual("SELECT orderkey FROM ORDERS TABLESAMPLE SYSTEM (50)").getMaterializedRows().size(); + if (sampleSize > 0 && sampleSize < total) { + sampleSizeFound = true; + break; + } + } + assertTrue(sampleSizeFound, "Table sample returned unexpected number of rows"); + } +} diff --git a/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestWindowQueries.java b/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestWindowQueries.java new file mode 100644 index 0000000000000..0599b7cd29291 --- /dev/null +++ b/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestWindowQueries.java @@ -0,0 +1,654 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.nativetests; + +import com.facebook.presto.nativeworker.NativeQueryRunnerUtils; +import com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils; +import com.facebook.presto.testing.QueryRunner; +import com.facebook.presto.tests.AbstractTestWindowQueries; +import org.testng.annotations.Test; + +public class TestWindowQueries + extends AbstractTestWindowQueries +{ + private String frameTypeDiffersError = ".*Window frame of type RANGE does not match types of the ORDER BY and frame column.*"; + private String unsupportedWindowTypeError = ".*Unsupported window type: 2.*"; + private String invalidOffsetError = ".*Window frame offset value must not be negative or null.*"; + private String functionNotRegisteredError = ".*Scalar function presto.default.plus not registered with arguments.*"; + + private String storageFormat = "PARQUET"; + + @Override + protected QueryRunner createQueryRunner() throws Exception + { + return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(true, storageFormat); + } + + @Override + protected void createTables() + { + try { + QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.createJavaQueryRunner("PARQUET"); + NativeQueryRunnerUtils.createAllTables(javaQueryRunner, false); + javaQueryRunner.close(); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + @Test + public void testAllPartitionSameValuesGroup() + { + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a GROUPS BETWEEN 1 FOLLOWING AND 2 FOLLOWING) " + + "FROM (VALUES 'a', 'a', 'a') T(a)", + unsupportedWindowTypeError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a GROUPS BETWEEN 2 PRECEDING AND 1 PRECEDING) " + + "FROM (VALUES 'a', 'a', 'a') T(a)", + unsupportedWindowTypeError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING) " + + "FROM (VALUES 'a', 'a', 'a') T(a)", + unsupportedWindowTypeError); + + // test frame bounds at partition bounds + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a GROUPS BETWEEN 10 PRECEDING AND 10 FOLLOWING) " + + "FROM (VALUES 'a', 'a', 'a') T(a)", + unsupportedWindowTypeError); + } + + @Override + @Test + public void testConstantOffset() + { + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a ASC NULLS FIRST GROUPS BETWEEN 1 PRECEDING AND 2 FOLLOWING) " + + "FROM (VALUES 3, 3, 3, 2, 2, 1, null, null) T(a)", + unsupportedWindowTypeError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a ASC NULLS FIRST GROUPS CURRENT ROW) " + + "FROM (VALUES 3, 3, 3, 2, 2, 1, null, null) T(a)", + unsupportedWindowTypeError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a ASC NULLS FIRST GROUPS BETWEEN 0 PRECEDING AND 0 FOLLOWING) " + + "FROM (VALUES 3, 3, 3, 2, 2, 1, null, null) T(a)", + unsupportedWindowTypeError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a ASC NULLS FIRST GROUPS BETWEEN 1 FOLLOWING AND 2 FOLLOWING) " + + "FROM (VALUES 3, 3, 3, 2, 2, 1, null, null) T(a)", + unsupportedWindowTypeError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a ASC NULLS FIRST GROUPS BETWEEN 2 PRECEDING AND 1 PRECEDING) " + + "FROM (VALUES 3, 3, 3, 2, 2, 1, null, null) T(a)", + unsupportedWindowTypeError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a ASC NULLS FIRST GROUPS BETWEEN 2 FOLLOWING AND 1 FOLLOWING) " + + "FROM (VALUES 3, 3, 3, 2, 2, 1, null, null) T(a)", + unsupportedWindowTypeError); + } + + @Override + @Test + public void testEmptyFrameRealBounds() + { + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a RANGE BETWEEN 0.5 FOLLOWING AND 1.5 FOLLOWING) " + + "FROM (VALUES 1, 2, 4) T(a)", frameTypeDiffersError, true); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a NULLS FIRST RANGE BETWEEN 2 PRECEDING AND 1.5 PRECEDING) " + + "FROM (VALUES null, 1, 2) T(a)", + frameTypeDiffersError, true); + } + + @Override + @Test + public void testEmptyFrameGroup() + { + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a ASC NULLS LAST GROUPS BETWEEN 90 PRECEDING AND 100 PRECEDING) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + unsupportedWindowTypeError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a ASC NULLS LAST GROUPS BETWEEN 100 FOLLOWING AND 90 FOLLOWING) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + unsupportedWindowTypeError); + } + + // TODO: This test is flaky so disabled for now. + @Override + @Test(enabled = false) + public void testInvalidOffset() + { + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a ASC RANGE x PRECEDING) " + + "FROM (VALUES (1, 0.1), (2, -0.2)) T(a, x)", + frameTypeDiffersError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a ASC RANGE BETWEEN 1 PRECEDING AND x FOLLOWING) " + + "FROM (VALUES (1, 0.1), (2, -0.2)) T(a, x)", + frameTypeDiffersError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a DESC RANGE x PRECEDING) " + + "FROM (VALUES (1, 0.1), (2, -0.2)) T(a, x)", + frameTypeDiffersError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a DESC RANGE BETWEEN 1 PRECEDING AND x FOLLOWING) " + + "FROM (VALUES (1, 0.1), (2, -0.2)) T(a, x)", + frameTypeDiffersError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a DESC RANGE x PRECEDING) " + + "FROM (VALUES (1, 0.1), (2, null)) T(a, x)", + frameTypeDiffersError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a DESC RANGE BETWEEN 1 PRECEDING AND x FOLLOWING) " + + "FROM (VALUES (1, 0.1), (2, null)) T(a, x)", + frameTypeDiffersError); + + // fail if offset is invalid for null sort key + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a DESC RANGE BETWEEN 1 PRECEDING AND x FOLLOWING) " + + "FROM (VALUES (1, 0.1), (null, null)) T(a, x)", + frameTypeDiffersError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a DESC RANGE BETWEEN 1 PRECEDING AND x FOLLOWING) " + + "FROM (VALUES (1, 0.1), (null, -0.1)) T(a, x)", + invalidOffsetError); + + // test invalid offset of different types + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a RANGE x PRECEDING) " + + "FROM (VALUES (1, BIGINT '-1')) T(a, x)", + frameTypeDiffersError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a RANGE x PRECEDING) " + + "FROM (VALUES (1, INTEGER '-1')) T(a, x)", + invalidOffsetError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a RANGE x PRECEDING) " + + "FROM (VALUES (SMALLINT '1', SMALLINT '-1')) T(a, x)", + frameTypeDiffersError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a RANGE x PRECEDING) " + + "FROM (VALUES (TINYINT '1', TINYINT '-1')) T(a, x)", + frameTypeDiffersError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a RANGE x PRECEDING) " + + "FROM (VALUES (1, -1.1e0)) T(a, x)", + frameTypeDiffersError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a RANGE x PRECEDING) " + + "FROM (VALUES (1, REAL '-1.1')) T(a, x)", + frameTypeDiffersError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a RANGE x PRECEDING) " + + "FROM (VALUES (1, -1.0001)) T(a, x)", + frameTypeDiffersError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a RANGE x PRECEDING) " + + "FROM (VALUES (DATE '2001-01-31', INTERVAL '-1' YEAR)) T(a, x)", + functionNotRegisteredError, true); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a RANGE x PRECEDING) " + + "FROM (VALUES (DATE '2001-01-31', INTERVAL '-1' MONTH)) T(a, x)", + functionNotRegisteredError, true); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a RANGE x PRECEDING) " + + "FROM (VALUES (DATE '2001-01-31', INTERVAL '-1' DAY)) T(a, x)", + functionNotRegisteredError, true); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a RANGE x PRECEDING) " + + "FROM (VALUES (DATE '2001-01-31', INTERVAL '-1' HOUR)) T(a, x)", + functionNotRegisteredError, true); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a RANGE x PRECEDING) " + + "FROM (VALUES (DATE '2001-01-31', INTERVAL '-1' MINUTE)) T(a, x)", + functionNotRegisteredError, true); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a RANGE x PRECEDING) " + + "FROM (VALUES (DATE '2001-01-31', INTERVAL '-1' SECOND)) T(a, x)", + functionNotRegisteredError, true); + } + + @Override + @Test + public void testInvalidOffsetGroup() + { + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a ASC GROUPS x PRECEDING) " + + "FROM (VALUES (1, 1), (2, -2)) T(a, x)", + unsupportedWindowTypeError); + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a ASC GROUPS x PRECEDING) " + + "FROM (VALUES (1, 1), (2, -2)) T(a, x)", + unsupportedWindowTypeError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a ASC GROUPS BETWEEN 1 PRECEDING AND x FOLLOWING) " + + "FROM (VALUES (1, 1), (2, -2)) T(a, x)", + unsupportedWindowTypeError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a DESC GROUPS x PRECEDING) " + + "FROM (VALUES (1, 1), (2, -2)) T(a, x)", + unsupportedWindowTypeError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a DESC GROUPS BETWEEN 1 PRECEDING AND x FOLLOWING) " + + "FROM (VALUES (1, 1), (2, -2)) T(a, x)", + unsupportedWindowTypeError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a DESC GROUPS x PRECEDING) " + + "FROM (VALUES (1, 1), (2, null)) T(a, x)", + unsupportedWindowTypeError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a DESC GROUPS BETWEEN 1 PRECEDING AND x FOLLOWING) " + + "FROM (VALUES (1, 1), (2, null)) T(a, x)", + unsupportedWindowTypeError); + + // fail if offset is invalid for null sort key + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a DESC GROUPS BETWEEN 1 PRECEDING AND x FOLLOWING) " + + "FROM (VALUES (1, 1), (null, null)) T(a, x)", + unsupportedWindowTypeError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a DESC GROUPS BETWEEN 1 PRECEDING AND x FOLLOWING) " + + "FROM (VALUES (1, 1), (null, -1)) T(a, x)", + unsupportedWindowTypeError); + + // test invalid offset of different types + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a GROUPS x PRECEDING) " + + "FROM (VALUES (1, BIGINT '-1')) T(a, x)", + unsupportedWindowTypeError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a GROUPS x PRECEDING) " + + "FROM (VALUES (1, INTEGER '-1')) T(a, x)", + unsupportedWindowTypeError); + } + + @Override + @Test + public void testMixedTypeFrameBounds() + { + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a ASC NULLS LAST GROUPS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + unsupportedWindowTypeError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a ASC NULLS LAST GROUPS BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + unsupportedWindowTypeError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a ASC NULLS LAST GROUPS BETWEEN CURRENT ROW AND 1 FOLLOWING) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + unsupportedWindowTypeError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a ASC NULLS LAST GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + unsupportedWindowTypeError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a ASC NULLS LAST GROUPS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + unsupportedWindowTypeError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a ASC NULLS LAST GROUPS BETWEEN 1 FOLLOWING AND UNBOUNDED FOLLOWING) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + unsupportedWindowTypeError); + } + + @Override + @Test + public void testMixedTypeFrameBoundsAscendingNullsFirst() + { + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND 0.5 PRECEDING) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + frameTypeDiffersError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND 1.5 FOLLOWING) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + frameTypeDiffersError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND 1.5 FOLLOWING) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + frameTypeDiffersError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a ASC NULLS FIRST RANGE BETWEEN 1.5 PRECEDING AND CURRENT ROW) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + frameTypeDiffersError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a ASC NULLS FIRST RANGE BETWEEN 0.5 PRECEDING AND UNBOUNDED FOLLOWING) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + frameTypeDiffersError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a ASC NULLS FIRST RANGE BETWEEN 0.5 FOLLOWING AND UNBOUNDED FOLLOWING) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + frameTypeDiffersError); + } + + @Override + @Test + public void testMixedTypeFrameBoundsAscendingNullsLast() + { + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a ASC NULLS LAST RANGE BETWEEN UNBOUNDED PRECEDING AND 0.5 PRECEDING) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + frameTypeDiffersError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a ASC NULLS LAST RANGE BETWEEN UNBOUNDED PRECEDING AND 1.5 FOLLOWING) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + frameTypeDiffersError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a ASC NULLS LAST RANGE BETWEEN CURRENT ROW AND 1.5 FOLLOWING) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + frameTypeDiffersError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a ASC NULLS LAST RANGE BETWEEN 1.5 PRECEDING AND CURRENT ROW) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + frameTypeDiffersError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a ASC NULLS LAST RANGE BETWEEN 0.5 PRECEDING AND UNBOUNDED FOLLOWING) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + frameTypeDiffersError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a ASC NULLS LAST RANGE BETWEEN 0.5 FOLLOWING AND UNBOUNDED FOLLOWING) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + frameTypeDiffersError); + } + + @Override + @Test + public void testMixedTypeFrameBoundsDescendingNullsFirst() + { + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a DESC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND 0.5 PRECEDING) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + frameTypeDiffersError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a DESC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND 0.5 FOLLOWING) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + frameTypeDiffersError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a DESC NULLS FIRST RANGE BETWEEN CURRENT ROW AND 1.5 FOLLOWING) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + frameTypeDiffersError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a DESC NULLS FIRST RANGE BETWEEN 1.5 PRECEDING AND CURRENT ROW) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + frameTypeDiffersError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a DESC NULLS FIRST RANGE BETWEEN 1.5 PRECEDING AND UNBOUNDED FOLLOWING) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + frameTypeDiffersError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a DESC NULLS FIRST RANGE BETWEEN 1.5 FOLLOWING AND UNBOUNDED FOLLOWING) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + frameTypeDiffersError); + } + + @Override + @Test + public void testMixedTypeFrameBoundsDescendingNullsLast() + { + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a DESC NULLS LAST RANGE BETWEEN UNBOUNDED PRECEDING AND 0.5 PRECEDING) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + frameTypeDiffersError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a DESC NULLS LAST RANGE BETWEEN UNBOUNDED PRECEDING AND 1.5 FOLLOWING) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + frameTypeDiffersError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a DESC NULLS LAST RANGE BETWEEN CURRENT ROW AND 1.5 FOLLOWING) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + frameTypeDiffersError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a DESC NULLS LAST RANGE BETWEEN 0.5 PRECEDING AND CURRENT ROW) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + frameTypeDiffersError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a DESC NULLS LAST RANGE BETWEEN 0.5 PRECEDING AND UNBOUNDED FOLLOWING) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + frameTypeDiffersError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a DESC NULLS LAST RANGE BETWEEN 1.5 FOLLOWING AND UNBOUNDED FOLLOWING) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + frameTypeDiffersError); + } + + @Override + @Test + public void testMultipleWindowFunctionsGroup() + { + // two functions with frame type GROUPS + assertQueryFails("SELECT x, array_agg(date) OVER(ORDER BY x GROUPS BETWEEN 1 PRECEDING AND 1 PRECEDING), avg(number) OVER(ORDER BY x GROUPS BETWEEN 1 FOLLOWING AND 1 FOLLOWING) " + + "FROM (VALUES " + + "(2, DATE '2222-01-01', 4.4), " + + "(1, DATE '1111-01-01', 2.2), " + + "(3, DATE '3333-01-01', 6.6)) T(x, date, number)", + unsupportedWindowTypeError); + + // three functions with different frame types + assertQueryFails("SELECT " + + "x, " + + "array_agg(a) OVER(ORDER BY x RANGE BETWEEN 2 PRECEDING AND CURRENT ROW), " + + "array_agg(a) OVER(ORDER BY x GROUPS BETWEEN 1 FOLLOWING AND 2 FOLLOWING), " + + "array_agg(a) OVER(ORDER BY x ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) " + + "FROM (VALUES " + + "(1.0, 1), " + + "(2.0, 2), " + + "(3.0, 3), " + + "(4.0, 4), " + + "(5.0, 5), " + + "(6.0, 6)) T(x, a)", + unsupportedWindowTypeError); + } + + @Override + public void testNonConstantOffset() + { + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a RANGE BETWEEN x * 10 PRECEDING AND y / 10.0 FOLLOWING) " + + "FROM (VALUES (1, 0.1, 10), (2, 0.2, 20), (4, 0.4, 40)) T(a, x, y)", + frameTypeDiffersError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a RANGE BETWEEN x * 10 PRECEDING AND y / 10.0 FOLLOWING) " + + "FROM (VALUES (1, 0.1, 10), (2, 0.2, 20), (4, 0.4, 40), (null, 0.5, 50)) T(a, x, y)", + frameTypeDiffersError); + } + + @Override + @Test + public void testNonConstantOffsetGroup() + { + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a GROUPS BETWEEN x PRECEDING AND y FOLLOWING) " + + "FROM (VALUES ('a', 1, 1), ('b', 2, 0), ('c', 0, 3)) T(a, x, y)", + unsupportedWindowTypeError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a GROUPS BETWEEN x FOLLOWING AND y FOLLOWING) " + + "FROM (VALUES ('a', 1, 1), ('b', 2, 0), ('c', 3, 3), ('d', 0, 0)) T(a, x, y)", + unsupportedWindowTypeError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a GROUPS BETWEEN x PRECEDING AND y PRECEDING) " + + "FROM (VALUES ('a', 1, 1), ('b', 0, 2), ('c', 2, 1), ('d', 0, 2)) T(a, x, y)", + unsupportedWindowTypeError); + } + + @Override + @Test + public void testNoValueFrameBoundsGroup() + { + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a ASC NULLS FIRST GROUPS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + unsupportedWindowTypeError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a ASC NULLS FIRST GROUPS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + unsupportedWindowTypeError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a ASC NULLS FIRST GROUPS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + unsupportedWindowTypeError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a ASC NULLS FIRST GROUPS BETWEEN CURRENT ROW AND CURRENT ROW) " + + "FROM (VALUES 1, null, null, 2, 1) T(a)", + unsupportedWindowTypeError); + } + + @Override + @Test + public void testOnlyNullsGroup() + { + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a GROUPS BETWEEN 1 PRECEDING AND 2 FOLLOWING) " + + "FROM (VALUES CAST(null AS integer), null, null) T(a)", + unsupportedWindowTypeError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a GROUPS BETWEEN 1 FOLLOWING AND 2 FOLLOWING) " + + "FROM (VALUES CAST(null AS integer), null, null) T(a)", + unsupportedWindowTypeError); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a GROUPS BETWEEN 2 PRECEDING AND 1 PRECEDING) " + + "FROM (VALUES CAST(null AS integer), null, null) T(a)", + unsupportedWindowTypeError); + } + + @Override + @Test + public void testWindowPartitioning() + { + assertQueryFails("SELECT a, p, array_agg(a) OVER(PARTITION BY p ORDER BY a ASC NULLS FIRST RANGE BETWEEN 0.5 PRECEDING AND 1 FOLLOWING) " + + "FROM (VALUES (1, 'x'), (2, 'x'), (null, 'x'), (null, 'y'), (2, 'y')) T(a, p)", + frameTypeDiffersError); + + assertQueryFails("SELECT a, p, array_agg(a) OVER(PARTITION BY p ORDER BY a ASC NULLS FIRST RANGE BETWEEN 0.5 PRECEDING AND 1 FOLLOWING) " + + "FROM (VALUES (1, 'x'), (2, 'x'), (null, 'x'), (null, 'y'), (2, 'y'), (null, null), (null, null), (1, null)) T(a, p)", + frameTypeDiffersError); + } + + @Override + @Test + public void testWindowPartitioningGroup() + { + assertQueryFails("SELECT a, p, array_agg(a) OVER(PARTITION BY p ORDER BY a ASC NULLS FIRST GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING) " + + "FROM (VALUES (1, 'x'), (2, 'x'), (null, 'x'), (null, 'y'), (2, 'y')) T(a, p)", + unsupportedWindowTypeError); + + assertQueryFails("SELECT a, p, array_agg(a) OVER(PARTITION BY p ORDER BY a ASC NULLS FIRST GROUPS BETWEEN 0 PRECEDING AND 1 FOLLOWING) " + + "FROM (VALUES (1, 'x'), (2, 'x'), (null, 'x'), (null, 'y'), (2, 'y'), (null, null), (null, null), (1, null)) T(a, p)", + unsupportedWindowTypeError); + } + + @Override + @Test + public void testTypes() + { + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a RANGE BETWEEN DOUBLE '0.5' PRECEDING AND TINYINT '1' FOLLOWING) " + + "FROM (VALUES 1, null, 2) T(a)", + frameTypeDiffersError); + + assertQuery("SELECT array_agg(a) OVER(ORDER BY a RANGE BETWEEN 0.5 PRECEDING AND 1.000 FOLLOWING) " + + "FROM (VALUES REAL '1', null, 2) T(a)", + "VALUES " + + "ARRAY[CAST('1' AS REAL), CAST('2' AS REAL)], " + + "ARRAY[CAST('2' AS REAL)], " + + "ARRAY[null]"); + + assertQuery("SELECT x, array_agg(x) OVER(ORDER BY x DESC RANGE BETWEEN interval '1' month PRECEDING AND interval '1' month FOLLOWING) " + + "FROM (VALUES DATE '2001-01-31', DATE '2001-08-25', DATE '2001-09-25', DATE '2001-09-26') T(x)", + "VALUES " + + "(DATE '2001-09-26', ARRAY[DATE '2001-09-26', DATE '2001-09-25']), " + + "(DATE '2001-09-25', ARRAY[DATE '2001-09-26', DATE '2001-09-25', DATE '2001-08-25']), " + + "(DATE '2001-08-25', ARRAY[DATE '2001-09-25', DATE '2001-08-25']), " + + "(DATE '2001-01-31', ARRAY[DATE '2001-01-31'])"); + + // January 31 + 1 month sets the frame bound to the last day of February. March 1 is out of range. + assertQuery("SELECT x, array_agg(x) OVER(ORDER BY x RANGE BETWEEN CURRENT ROW AND interval '1' month FOLLOWING) " + + "FROM (VALUES DATE '2001-01-31', DATE '2001-02-28', DATE '2001-03-01') T(x)", + "VALUES " + + "(DATE '2001-01-31', ARRAY[DATE '2001-01-31', DATE '2001-02-28']), " + + "(DATE '2001-02-28', ARRAY[DATE '2001-02-28', DATE '2001-03-01']), " + + "(DATE '2001-03-01', ARRAY[DATE '2001-03-01'])"); + + // H2 and Presto has some type conversion problem for Interval type, hence use the same query runner for this query + assertQueryFails("SELECT x, array_agg(x) OVER(ORDER BY x RANGE BETWEEN interval '1' year PRECEDING AND interval '1' month FOLLOWING) " + + "FROM (VALUES " + + "INTERVAL '1' month, " + + "INTERVAL '2' month, " + + "INTERVAL '5' year) T(x)", + functionNotRegisteredError, true); + } + + @Override + @Test + public void testEmptyFrameIntegralBounds() + { + assertQuery("SELECT array_agg(a) OVER(ORDER BY a DESC NULLS LAST RANGE BETWEEN 1 PRECEDING AND 10 PRECEDING) " + + "FROM (VALUES 1, 2, 3, null, null, 2, 1, null, null) T(a)", + "VALUES " + + "CAST(null AS array), " + + "null, " + + "null, " + + "null, " + + "null, " + + "ARRAY[null, null, null, null], " + + "ARRAY[null, null, null, null], " + + "ARRAY[null, null, null, null], " + + "ARRAY[null, null, null, null]"); + + assertQuery("SELECT array_agg(a) OVER(ORDER BY a DESC NULLS LAST RANGE BETWEEN 10 FOLLOWING AND 1 FOLLOWING) " + + "FROM (VALUES 1, 2, 3, null, null, 2, 1, null, null) T(a)", + "VALUES " + + "CAST(null AS array), " + + "null, " + + "null, " + + "null, " + + "null, " + + "ARRAY[null, null, null, null], " + + "ARRAY[null, null, null, null], " + + "ARRAY[null, null, null, null], " + + "ARRAY[null, null, null, null]"); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a RANGE BETWEEN 1 FOLLOWING AND 2 FOLLOWING) " + + "FROM (VALUES 1.0, 1.1) T(a)", frameTypeDiffersError, true); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a NULLS LAST RANGE BETWEEN 1 FOLLOWING AND 2 FOLLOWING) " + + "FROM (VALUES 1.0, 1.1, null) T(a)", frameTypeDiffersError, true); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a RANGE BETWEEN 2 PRECEDING AND 1 PRECEDING) " + + "FROM (VALUES 1.0, 1.1) T(a)", frameTypeDiffersError, true); + + assertQueryFails("SELECT array_agg(a) OVER(ORDER BY a NULLS FIRST RANGE BETWEEN 2 PRECEDING AND 1 PRECEDING) " + + "FROM (VALUES null, 1.0, 1.1) T(a)", frameTypeDiffersError, true); + + assertQuery("SELECT array_agg(a) OVER(ORDER BY a RANGE BETWEEN 2 PRECEDING AND 1 PRECEDING) " + + "FROM (VALUES 1, 2) T(a)", + "VALUES " + + "null, " + + "ARRAY[1]"); + + assertQuery("SELECT array_agg(a) OVER(ORDER BY a NULLS FIRST RANGE BETWEEN 2 PRECEDING AND 1 PRECEDING) " + + "FROM (VALUES null, 1, 2) T(a)", + "VALUES " + + "ARRAY[null], " + + "null, " + + "ARRAY[1]"); + } + + @Override + @Test + public void testMultipleWindowFunctions() + { + assertQuery("SELECT x, array_agg(date) OVER(ORDER BY x RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING), avg(number) OVER(ORDER BY x RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING) " + + "FROM (VALUES " + + "(2, DATE '2222-01-01', 4.4), " + + "(1, DATE '1111-01-01', 2.2), " + + "(3, DATE '3333-01-01', 6.6)) T(x, date, number)", + "VALUES " + + "(1, ARRAY[DATE '1111-01-01', DATE '2222-01-01'], 3.3), " + + "(2, ARRAY[DATE '1111-01-01', DATE '2222-01-01', DATE '3333-01-01'], 4.4), " + + "(3, ARRAY[DATE '2222-01-01', DATE '3333-01-01'], 5.5)"); + + assertQueryFails("SELECT x, array_agg(a) OVER(ORDER BY x RANGE BETWEEN 2 PRECEDING AND CURRENT ROW), array_agg(a) OVER(ORDER BY x RANGE BETWEEN CURRENT ROW AND 2 FOLLOWING) " + + "FROM (VALUES " + + "(1.0, 1), " + + "(2.0, 2), " + + "(3.0, 3), " + + "(4.0, 4), " + + "(5.0, 5), " + + "(6.0, 6)) T(x, a)", frameTypeDiffersError, true); + } +} diff --git a/presto-tests/src/test/java/com/facebook/presto/tests/AbstractTestEngineOnlyQueries.java b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestEngineOnlyQueries.java similarity index 100% rename from presto-tests/src/test/java/com/facebook/presto/tests/AbstractTestEngineOnlyQueries.java rename to presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestEngineOnlyQueries.java diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestOrderByQueries.java b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestOrderByQueries.java index 9e633ff52e9ee..62a086c951bfe 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestOrderByQueries.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestOrderByQueries.java @@ -86,11 +86,6 @@ public void testOrderByWithOutputColumnReference() assertQueryOrdered("SELECT -a AS a, a AS b FROM (VALUES 1, 2) t(a) GROUP BY a ORDER BY t.a+2*a", "VALUES (-2, 2), (-1, 1)"); assertQueryOrdered("SELECT -a AS a, a AS b FROM (VALUES 1, 2) t(a) GROUP BY t.a ORDER BY t.a+2*a", "VALUES (-2, 2), (-1, 1)"); - // lambdas - assertQueryOrdered("SELECT x AS y FROM (values (1,2), (2,3)) t(x, y) GROUP BY x ORDER BY apply(x, x -> -x) + 2*x", "VALUES 1, 2"); - assertQueryOrdered("SELECT -y AS x FROM (values (1,2), (2,3)) t(x, y) GROUP BY y ORDER BY apply(x, x -> -x)", "VALUES -2, -3"); - assertQueryOrdered("SELECT -y AS x FROM (values (1,2), (2,3)) t(x, y) GROUP BY y ORDER BY sum(apply(-y, x -> x * 1.0))", "VALUES -3, -2"); - // distinct assertQueryOrdered("SELECT DISTINCT -a AS b FROM (VALUES 1, 2) t(a) ORDER BY b", "VALUES -2, -1"); assertQueryOrdered("SELECT DISTINCT -a AS b FROM (VALUES 1, 2) t(a) ORDER BY 1", "VALUES -2, -1"); @@ -106,6 +101,14 @@ public void testOrderByWithOutputColumnReference() assertQueryFails("SELECT a, a* -1 AS a FROM (VALUES -1, 0, 2) t(a) ORDER BY a", ".*'a' is ambiguous"); } + @Test + public void testOrderByWithOutputColumnReferenceInLambdas() + { + assertQueryOrdered("SELECT x AS y FROM (values (1,2), (2,3)) t(x, y) GROUP BY x ORDER BY apply(x, x -> -x) + 2*x", "VALUES 1, 2"); + assertQueryOrdered("SELECT -y AS x FROM (values (1,2), (2,3)) t(x, y) GROUP BY y ORDER BY apply(x, x -> -x)", "VALUES -2, -3"); + assertQueryOrdered("SELECT -y AS x FROM (values (1,2), (2,3)) t(x, y) GROUP BY y ORDER BY sum(apply(-y, x -> x * 1.0))", "VALUES -3, -2"); + } + @Test public void testOrderByWithAggregation() { diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java index 080c530875095..a318dcccd5b75 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java @@ -290,9 +290,13 @@ public void testNonDeterministic() .distinct() .count(); assertTrue(distinctCount >= 8, "rand() must produce different rows"); + } - materializedResult = computeActual("SELECT apply(1, x -> x + rand()) FROM orders LIMIT 10"); - distinctCount = materializedResult.getMaterializedRows().stream() + @Test + public void testNonDeterministicInLambda() + { + MaterializedResult materializedResult = computeActual("SELECT apply(1, x -> x + rand()) FROM orders LIMIT 10"); + long distinctCount = materializedResult.getMaterializedRows().stream() .map(row -> row.getField(0)) .distinct() .count(); @@ -361,11 +365,16 @@ public void testLambdaInValuesAndUnnest() } @Test - public void testTryLambdaRepeated() + public void testApplyLambdaRepeated() { assertQuery("SELECT x + x FROM (SELECT apply(a, i -> i * i) x FROM (VALUES 3) t(a))", "SELECT 18"); assertQuery("SELECT apply(a, i -> i * i) + apply(a, i -> i * i) FROM (VALUES 3) t(a)", "SELECT 18"); assertQuery("SELECT apply(a, i -> i * i), apply(a, i -> i * i) FROM (VALUES 3) t(a)", "SELECT 9, 9"); + } + + @Test + public void testTryLambdaRepeated() + { assertQuery("SELECT try(10 / a) + try(10 / a) FROM (VALUES 5) t(a)", "SELECT 4"); assertQuery("SELECT try(10 / a), try(10 / a) FROM (VALUES 5) t(a)", "SELECT 2, 2"); } @@ -437,8 +446,11 @@ public void testRowSubscript() // Row subscript in join condition assertQuery("SELECT n.name, r.name FROM nation n JOIN region r ON ROW (n.name, n.regionkey)[2] = ROW (r.name, r.regionkey)[2] ORDER BY n.name LIMIT 1", "VALUES ('ALGERIA', 'AFRICA')"); + } - //Row subscript in a lambda + @Test + public void testRowSubscriptInLambda() + { assertQuery("SELECT apply(ROW (1, 2), r -> r[2])", "SELECT 2"); } @@ -2400,12 +2412,17 @@ public void testIn() assertQuery("SELECT 1 in (1, NULL, 3)", "values true"); assertQuery("SELECT 2 in (1, NULL, 3)", "values null"); assertQuery("SELECT x FROM (values DATE '1970-01-01', DATE '1970-01-03') t(x) WHERE x IN (DATE '1970-01-01')", "values DATE '1970-01-01'"); + assertQuery("SELECT COUNT(*) FROM (values 1) t(x) WHERE x IN (null, 0)", "SELECT 0"); + assertQuery("SELECT d IN (DECIMAL '2.0', DECIMAL '30.0') FROM (VALUES (2.0E0)) t(d)", "SELECT true"); // coercion with type only coercion inside IN list + } + + @Test + public void testInTimestampWithTimezone() + { assertEquals( computeActual("SELECT x FROM (values TIMESTAMP '1970-01-01 00:01:00+00:00', TIMESTAMP '1970-01-01 08:01:00+08:00', TIMESTAMP '1970-01-01 00:01:00+08:00') t(x) WHERE x IN (TIMESTAMP '1970-01-01 00:01:00+00:00')") .getOnlyColumn().collect(toList()), ImmutableList.of(zonedDateTime("1970-01-01 00:01:00.000 UTC"), zonedDateTime("1970-01-01 08:01:00.000 +08:00"))); - assertQuery("SELECT COUNT(*) FROM (values 1) t(x) WHERE x IN (null, 0)", "SELECT 0"); - assertQuery("SELECT d IN (DECIMAL '2.0', DECIMAL '30.0') FROM (VALUES (2.0E0)) t(d)", "SELECT true"); // coercion with type only coercion inside IN list } @Test @@ -3202,14 +3219,9 @@ public void testTry() assertQueryFails("SELECT TRY()", "line 1:8: The 'try' function must have exactly one argument"); // check that TRY is not pushed down - assertQueryFails("SELECT TRY(x) IS NULL FROM (SELECT 1/y AS x FROM (VALUES 1, 2, 3, 0, 4) t(y))", "/ by zero"); + assertQueryFails("SELECT TRY(x) IS NULL FROM (SELECT 1/y AS x FROM (VALUES 1, 2, 3, 0, 4) t(y))", ".*(/|division) by zero.*"); assertQuery("SELECT x IS NULL FROM (SELECT TRY(1/y) AS x FROM (VALUES 3, 0, 4) t(y))", "VALUES false, true, false"); - // test try with lambda function - assertQuery("SELECT TRY(apply(5, x -> x + 1) / 0)", "SELECT NULL"); - assertQuery("SELECT TRY(apply(5 + RANDOM(1), x -> x + 1) / 0)", "SELECT NULL"); - assertQuery("SELECT apply(5 + RANDOM(1), x -> x + TRY(1 / 0))", "SELECT NULL"); - // test try with invalid JSON assertQuery("SELECT JSON_FORMAT(TRY(JSON 'INVALID'))", "SELECT NULL"); assertQuery("SELECT JSON_FORMAT(TRY (JSON_PARSE('INVALID')))", "SELECT NULL"); @@ -3236,6 +3248,14 @@ public void testTry() assertQuery("SELECT TRY(1 / x) FROM (SELECT NULL as x)", "SELECT NULL"); } + @Test + public void testTryWithLambda() + { + assertQuery("SELECT TRY(apply(5, x -> x + 1) / 0)", "SELECT NULL"); + assertQuery("SELECT TRY(apply(5 + RANDOM(1), x -> x + 1) / 0)", "SELECT NULL"); + assertQuery("SELECT apply(5 + RANDOM(1), x -> x + TRY(1 / 0))", "SELECT NULL"); + } + @Test public void testTryNoMergeProjections() { @@ -6020,6 +6040,11 @@ public void testSetUnion() assertQuery( "select group_id, set_union(numbers) from (values (1, array[1, 2]), (2, array[2, 3]), (3, array[4, 5]), (4, array[5, 6])) as t(group_id, numbers) group by group_id", "select group_id, numbers from (values (1, array[1, 2]), (2, array[2, 3]), (3, array[4, 5]), (4, array[5, 6])) as t(group_id, numbers)"); + } + + @Test + public void testSetUnionWithNulls() + { // all nulls should return empty array to match behavior of array_distinct(flatten(array_agg(x))) assertQuery( "select set_union(x) from (values null, null, null) as t(x)", @@ -6135,7 +6160,8 @@ public void testApproxMostFrequentWithLong() MaterializedResult actual2 = computeActual("SELECT approx_most_frequent(2, cast(x as bigint), 15) FROM (values 1, 2, 1, 3, 1, 2, 3, 4, 5) t(x)"); assertEquals(actual2.getRowCount(), 1); - assertEquals(actual2.getMaterializedRows().get(0).getFields().get(0), ImmutableMap.of(1L, 3L, 2L, 2L)); + Object actual2Value = actual2.getMaterializedRows().get(0).getFields().get(0); + assertTrue(actual2Value.equals(ImmutableMap.of(1L, 3L, 2L, 2L)) || actual2Value.equals(ImmutableMap.of(1L, 3L, 3L, 2L))); } @Test @@ -6147,7 +6173,8 @@ public void testApproxMostFrequentWithVarchar() MaterializedResult actual2 = computeActual("SELECT approx_most_frequent(2, x, 15) FROM (values 'A', 'B', 'A', 'C', 'A', 'B', 'C', 'D', 'E') t(x)"); assertEquals(actual2.getRowCount(), 1); - assertEquals(actual2.getMaterializedRows().get(0).getFields().get(0), ImmutableMap.of("A", 3L, "B", 2L)); + Object actual2Value = actual2.getMaterializedRows().get(0).getFields().get(0); + assertTrue(actual2Value.equals(ImmutableMap.of("A", 3L, "B", 2L)) || actual2Value.equals(ImmutableMap.of("A", 3L, "C", 2L))); } @Test @@ -6580,15 +6607,6 @@ public void testDuplicateUnnestItem() "VALUES (1, 2, 'a', 2, 'a', 1, 'a', 1), (1, 3, 'b', 3, 'b', 2, 'b', 2), (1, NULL, NULL, NULL, NULL, 3, 'c', 3)"); assertQuery("SELECT * from ( SELECT ARRAY[1] AS kv FROM (select 1)) CROSS JOIN UNNEST( kv, kv ) WITH ORDINALITY AS t(r1, r2, ord)", "VALUES (ARRAY[1], 1, 1, 1)"); - assertQuery("SELECT * from (select * FROM (values 1) as t(k)) CROSS JOIN unnest(ARRAY[row(2, 3), row(3, 5)], ARRAY[row(2, 3), row(3, 5)]) AS r(r1, r2, r3, r4)", - "VALUES (1, 2, 3, 2, 3), (1, 3, 5, 3, 5)"); - assertQuery("SELECT * from (select * FROM (values 1) as t(k)) CROSS JOIN unnest(ARRAY[row(2, 3), row(3, 5)], ARRAY[row(2, 3), row(3, 5)], ARRAY[row(10, 13, 15), row(23, 25, 20)]) AS r(r1, r2, r3, r4, r5, r6, r7)", - "VALUES (1, 2, 3, 2, 3, 10, 13, 15), (1, 3, 5, 3, 5, 23, 25, 20)"); - assertQuery("SELECT * from (select * FROM (values 1) as t(k)) CROSS JOIN unnest(ARRAY[row(2, 3), row(3, 5)], ARRAY[row(2, 3), row(3, 5)]) WITH ORDINALITY AS r(r1, r2, r3, r4, ord)", - "VALUES (1, 2, 3, 2, 3, 1), (1, 3, 5, 3, 5, 2)"); - assertQuery("SELECT * from (select * FROM (values 1) as t(k)) CROSS JOIN unnest(ARRAY[row(2, 3), row(3, 5)], ARRAY[row(2, 3), row(3, 5)], ARRAY[row(10, 13, 15), row(23, 25, 20)]) WITH ORDINALITY AS r(r1, r2, r3, r4, r5, r6, r7, ord)", - "VALUES (1, 2, 3, 2, 3, 10, 13, 15, 1), (1, 3, 5, 3, 5, 23, 25, 20, 2)"); - Session useLegacyUnnest = Session.builder(getSession()) .setSystemProperty(LEGACY_UNNEST, "true") .build(); @@ -6624,11 +6642,6 @@ public void testDuplicateUnnestItem() assertQuery("SELECT * from unnest(ARRAY[2, 3], ARRAY[10,11,12], ARRAY[2, 3]) WITH ORDINALITY AS r(r1, r2, r3, ord)", "VALUES (2, 10, 2, 1), (3, 11, 3, 2), (NULL, 12, NULL, 3)"); assertQuery("SELECT * from unnest(ARRAY[2, 3], ARRAY[2, 3], ARRAY[10,11,12]) WITH ORDINALITY AS r(r1, r2, r3, ord)", "VALUES (2, 2, 10, 1), (3, 3, 11, 2), (NULL, NULL, 12, 3)"); - assertQuery("SELECT * from unnest(ARRAY[row(2, 3), row(3, 5)], ARRAY[row(2, 3), row(3, 5)]) AS r(r1, r2, r3, r4)", - "VALUES (2, 3, 2, 3), (3, 5, 3, 5)"); - assertQuery("SELECT * from unnest(ARRAY[row(2, 3), row(3, 5)], ARRAY[row(2, 3), row(3, 5)]) WITH ORDINALITY AS r(r1, r2, r3, r4, ord)", - "VALUES (2, 3, 2, 3, 1), (3, 5, 3, 5, 2)"); - assertQuery(useLegacyUnnest, "SELECT cast(r1 as row(x int, y int)), cast(r2 as row(x int, y int)) from unnest(ARRAY[row(2, 3), row(3, 5)], ARRAY[row(2, 3), row(3, 5)]) AS r(r1, r2)", "VALUES (row(2, 3), row(2, 3)), (row(3, 5), row(3, 5))"); assertQuery(useLegacyUnnest, "SELECT cast(r1 as row(x int, y int)), cast(r2 as row(x int, y int)), cast(r3 as row(x int, y int, z int)) from unnest(ARRAY[row(2, 3), row(3, 5)], ARRAY[row(2, 3), row(3, 5)], ARRAY[row(10, 13, 15), row(23, 25, 20)]) AS r(r1, r2, r3)", @@ -6643,6 +6656,24 @@ public void testDuplicateUnnestItem() "VALUES (2, 2, 1, 2, 2), (2, 2, 1, 3, 3), (3, 3, 2, 2, 2), (3, 3, 2, 3, 3)"); } + @Test + public void testDuplicateUnnestRows() + { + assertQuery("SELECT * from (select * FROM (values 1) as t(k)) CROSS JOIN unnest(ARRAY[row(2, 3), row(3, 5)], ARRAY[row(2, 3), row(3, 5)]) AS r(r1, r2, r3, r4)", + "VALUES (1, 2, 3, 2, 3), (1, 3, 5, 3, 5)"); + assertQuery("SELECT * from (select * FROM (values 1) as t(k)) CROSS JOIN unnest(ARRAY[row(2, 3), row(3, 5)], ARRAY[row(2, 3), row(3, 5)], ARRAY[row(10, 13, 15), row(23, 25, 20)]) AS r(r1, r2, r3, r4, r5, r6, r7)", + "VALUES (1, 2, 3, 2, 3, 10, 13, 15), (1, 3, 5, 3, 5, 23, 25, 20)"); + assertQuery("SELECT * from (select * FROM (values 1) as t(k)) CROSS JOIN unnest(ARRAY[row(2, 3), row(3, 5)], ARRAY[row(2, 3), row(3, 5)]) WITH ORDINALITY AS r(r1, r2, r3, r4, ord)", + "VALUES (1, 2, 3, 2, 3, 1), (1, 3, 5, 3, 5, 2)"); + assertQuery("SELECT * from (select * FROM (values 1) as t(k)) CROSS JOIN unnest(ARRAY[row(2, 3), row(3, 5)], ARRAY[row(2, 3), row(3, 5)], ARRAY[row(10, 13, 15), row(23, 25, 20)]) WITH ORDINALITY AS r(r1, r2, r3, r4, r5, r6, r7, ord)", + "VALUES (1, 2, 3, 2, 3, 10, 13, 15, 1), (1, 3, 5, 3, 5, 23, 25, 20, 2)"); + + assertQuery("SELECT * from unnest(ARRAY[row(2, 3), row(3, 5)], ARRAY[row(2, 3), row(3, 5)]) AS r(r1, r2, r3, r4)", + "VALUES (2, 3, 2, 3), (3, 5, 3, 5)"); + assertQuery("SELECT * from unnest(ARRAY[row(2, 3), row(3, 5)], ARRAY[row(2, 3), row(3, 5)]) WITH ORDINALITY AS r(r1, r2, r3, r4, ord)", + "VALUES (2, 3, 2, 3, 1), (3, 5, 3, 5, 2)"); + } + @Test public void testDependentWindowFunction() { @@ -6993,14 +7024,14 @@ public void testArraySplitIntoChunks() } @Test - public void testArrayCumSum() + public void testArrayCumSumIntegers() { // int String sql = "select array_cum_sum(k) from (values (array[cast(5 as INTEGER), 6, 0]), (ARRAY[]), (CAST(NULL AS array(integer)))) t(k)"; assertQuery(sql, "values array[cast(5 as integer), cast(11 as integer), cast(11 as integer)], array[], null"); sql = "select array_cum_sum(k) from (values (array[cast(5 as INTEGER), 6, 0]), (ARRAY[]), (CAST(NULL AS array(integer))), (ARRAY [cast(2147483647 as INTEGER), 2147483647, 2147483647])) t(k)"; - assertQueryFails(sql, "integer addition overflow:.*"); + assertQueryFails(sql, "integer (addition )?overflow:.*", true); sql = "select array_cum_sum(k) from (values (array[cast(5 as INTEGER), 6, null, 2, 3])) t(k)"; assertQuery(sql, "values array[cast(5 as integer), cast(11 as integer), cast(null as integer), cast(null as integer), cast(null as integer)]"); @@ -7017,9 +7048,24 @@ public void testArrayCumSum() sql = "select array_cum_sum(k) from (values (array[cast(null as bigint), 6, null, 2, 3])) t(k)"; assertQuery(sql, "values array[cast(null as bigint), cast(null as bigint), cast(null as bigint), cast(null as bigint), cast(null as bigint)]"); + } + + @Test + public void testArrayCumSumDecimals() + { + String sql = "select array_cum_sum(k) from (values (array[cast(5.1 as decimal(38, 1)), 6, 0]), (ARRAY[]), (CAST(NULL AS array(decimal)))) t(k)"; + assertQuery(sql, "values array[cast(5.1 as decimal), cast(11.1 as decimal), cast(11.1 as decimal)], array[], null"); + sql = "select array_cum_sum(k) from (values (array[cast(5.1 as decimal(38, 1)), 6, null, 3]), (array[cast(null as decimal(38, 1)), 6, null, 3])) t(k)"; + assertQuery(sql, "values array[cast(5.1 as decimal), cast(11.1 as decimal), cast(null as decimal), cast(null as decimal)], " + + "array[cast(null as decimal), cast(null as decimal), cast(null as decimal), cast(null as decimal)]"); + } + + @Test + public void testArrayCumSumReals() + { // real - sql = "select array_cum_sum(k) from (values (array[cast(null as real), 6, null, 2, 3])) t(k)"; + String sql = "select array_cum_sum(k) from (values (array[cast(null as real), 6, null, 2, 3])) t(k)"; assertQuery(sql, "values array[cast(null as real), cast(null as real), cast(null as real), cast(null as real), cast(null as real)]"); MaterializedResult raw = computeActual("SELECT array_cum_sum(k) FROM (values (ARRAY [cast(5.1 as real), 6.1, 0.5]), (ARRAY[]), (CAST(NULL AS array(real))), " + @@ -7080,17 +7126,12 @@ public void testArrayCumSum() for (int i = 2; i < actualDouble.size(); ++i) { assertNull(actualDouble.get(i)); } + } - // decimal - sql = "select array_cum_sum(k) from (values (array[cast(5.1 as decimal(38, 1)), 6, 0]), (ARRAY[]), (CAST(NULL AS array(decimal)))) t(k)"; - assertQuery(sql, "values array[cast(5.1 as decimal), cast(11.1 as decimal), cast(11.1 as decimal)], array[], null"); - - sql = "select array_cum_sum(k) from (values (array[cast(5.1 as decimal(38, 1)), 6, null, 3]), (array[cast(null as decimal(38, 1)), 6, null, 3])) t(k)"; - assertQuery(sql, "values array[cast(5.1 as decimal), cast(11.1 as decimal), cast(null as decimal), cast(null as decimal)], " + - "array[cast(null as decimal), cast(null as decimal), cast(null as decimal), cast(null as decimal)]"); - - // varchar - sql = "select array_cum_sum(k) from (values (array[cast('5.1' as varchar), '6', '0']), (ARRAY[]), (CAST(NULL AS array(varchar)))) t(k)"; + @Test + public void testArrayCumSumVarchar() + { + String sql = "select array_cum_sum(k) from (values (array[cast('5.1' as varchar), '6', '0']), (ARRAY[]), (CAST(NULL AS array(varchar)))) t(k)"; assertQueryFails(sql, ".*cannot be applied to.*"); sql = "select array_cum_sum(k) from (values (array[cast(null as varchar), '6', '0'])) t(k)"; @@ -7788,12 +7829,20 @@ public void testRemoveMapCast() "values 0.5, 0.1"); assertQuery(enableOptimization, "select element_at(feature, key) from (values (map(array[cast(1 as integer), 2, 3, 4], array[0.3, 0.5, 0.9, 0.1]), cast(2 as bigint)), (map(array[cast(1 as integer), 2, 3, 4], array[0.3, 0.5, 0.9, 0.1]), 400000000000)) t(feature, key)", "values 0.5, null"); - assertQueryFails(enableOptimization, "select feature[key] from (values (map(array[cast(1 as integer), 2, 3, 4], array[0.3, 0.5, 0.9, 0.1]), cast(2 as bigint)), (map(array[cast(1 as integer), 2, 3, 4], array[0.3, 0.5, 0.9, 0.1]), 400000000000)) t(feature, key)", - ".*Out of range for integer.*"); assertQuery(enableOptimization, "select feature[key] from (values (map(array[cast(1 as varchar), '2', '3', '4'], array[0.3, 0.5, 0.9, 0.1]), cast('2' as varchar)), (map(array[cast(1 as varchar), '2', '3', '4'], array[0.3, 0.5, 0.9, 0.1]), '4')) t(feature, key)", "values 0.5, 0.1"); } + @Test + public void testRemoveMapCastFailure() + { + Session enableOptimization = Session.builder(getSession()) + .setSystemProperty(REMOVE_MAP_CAST, "true") + .build(); + assertQueryFails(enableOptimization, "select feature[key] from (values (map(array[cast(1 as integer), 2, 3, 4], array[0.3, 0.5, 0.9, 0.1]), cast(2 as bigint)), (map(array[cast(1 as integer), 2, 3, 4], array[0.3, 0.5, 0.9, 0.1]), 400000000000)) t(feature, key)", + ".*Out of range for integer.*"); + } + // Test to guardrail problems in constraint framework mentioned in https://github.com/prestodb/presto/pull/22171 @Test public void testGuardConstraintFramework() diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java index 303ae0dfe7801..6ab65e61bd80a 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java @@ -310,6 +310,11 @@ protected void assertQueryFails(QueryRunner queryRunner, @Language("SQL") String QueryAssertions.assertQueryFails(queryRunner, getSession(), sql, expectedMessageRegExp); } + protected void assertQueryFails(@Language("SQL") String sql, @Language("RegExp") String expectedMessageRegExp, Boolean usePatternMatcher) + { + QueryAssertions.assertQueryFails(queryRunner, getSession(), sql, expectedMessageRegExp, usePatternMatcher); + } + protected void assertQueryFails(Session session, @Language("SQL") String sql, @Language("RegExp") String expectedMessageRegExp) { QueryAssertions.assertQueryFails(queryRunner, session, sql, expectedMessageRegExp); @@ -330,6 +335,11 @@ protected void assertQueryError(@Language("SQL") String sql, @Language("RegExp") assertQueryError(queryRunner, getSession(), sql, expectedMessageRegExp); } + protected void assertQueryFails(Session session, @Language("SQL") String sql, @Language("RegExp") String expectedMessageRegExp, Boolean usePatternMatcher) + { + QueryAssertions.assertQueryFails(queryRunner, session, sql, expectedMessageRegExp, usePatternMatcher); + } + protected void assertQueryReturnsEmptyResult(@Language("SQL") String sql) { QueryAssertions.assertQueryReturnsEmptyResult(queryRunner, getSession(), sql); diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestWindowQueries.java b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestWindowQueries.java index 14fa5afa8ebf3..b218ff280bd68 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestWindowQueries.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestWindowQueries.java @@ -1016,7 +1016,7 @@ public void testEmptyInput() } @Test - public void testEmptyFrame() + public void testEmptyFrameIntegralBounds() { assertQuery("SELECT array_agg(a) OVER(ORDER BY a DESC NULLS LAST RANGE BETWEEN 1 PRECEDING AND 10 PRECEDING) " + "FROM (VALUES 1, 2, 3, null, null, 2, 1, null, null) T(a)", @@ -1044,13 +1044,6 @@ public void testEmptyFrame() "ARRAY[null, null, null, null], " + "ARRAY[null, null, null, null]"); - assertQuery("SELECT array_agg(a) OVER(ORDER BY a RANGE BETWEEN 0.5 FOLLOWING AND 1.5 FOLLOWING) " + - "FROM (VALUES 1, 2, 4) T(a)", - "VALUES " + - "ARRAY[2], " + - "null, " + - "null"); - assertQuery("SELECT array_agg(a) OVER(ORDER BY a RANGE BETWEEN 1 FOLLOWING AND 2 FOLLOWING) " + "FROM (VALUES 1.0, 1.1) T(a)", "VALUES " + @@ -1089,6 +1082,17 @@ public void testEmptyFrame() "ARRAY[null], " + "null, " + "ARRAY[1]"); + } + + @Test + public void testEmptyFrameRealBounds() + { + assertQuery("SELECT array_agg(a) OVER(ORDER BY a RANGE BETWEEN 0.5 FOLLOWING AND 1.5 FOLLOWING) " + + "FROM (VALUES 1, 2, 4) T(a)", + "VALUES " + + "ARRAY[2], " + + "null, " + + "null"); assertQuery("SELECT array_agg(a) OVER(ORDER BY a NULLS FIRST RANGE BETWEEN 2 PRECEDING AND 1.5 PRECEDING) " + "FROM (VALUES null, 1, 2) T(a)", diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/QueryAssertions.java b/presto-tests/src/main/java/com/facebook/presto/tests/QueryAssertions.java index 035051acf2866..6c39ba53366ea 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/QueryAssertions.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/QueryAssertions.java @@ -37,6 +37,7 @@ import java.util.OptionalLong; import java.util.function.Consumer; import java.util.function.Supplier; +import java.util.regex.Pattern; import static com.google.common.base.Strings.nullToEmpty; import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly; @@ -350,6 +351,30 @@ protected static void assertQueryFails(QueryRunner queryRunner, Session session, } } + private static void assertExceptionWithPatternMatch(String sql, Exception exception, @Language("RegExp") String regex) + { + Pattern p = Pattern.compile(regex, Pattern.MULTILINE); + if (!(p.matcher(exception.getMessage()).find())) { + fail(format("Expected exception message '%s' to match '%s' for query: %s", exception.getMessage(), regex, sql), exception); + } + } + + protected static void assertQueryFails(QueryRunner queryRunner, Session session, @Language("SQL") String sql, @Language("RegExp") String expectedMessageRegExp, Boolean usePatternMatcher) + { + try { + queryRunner.execute(session, sql); + fail(format("Expected query to fail: %s", sql)); + } + catch (RuntimeException ex) { + if (usePatternMatcher) { + assertExceptionWithPatternMatch(sql, ex, expectedMessageRegExp); + } + else { + assertExceptionMessage(sql, ex, expectedMessageRegExp); + } + } + } + protected static void assertQueryReturnsEmptyResult(QueryRunner queryRunner, Session session, @Language("SQL") String sql) { try {