From f714c4023c62c3516d2baa0b0ffa808c7c76d15a Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Fri, 22 Nov 2024 15:23:03 +0100 Subject: [PATCH] merge polaris-service-quarkus with polaris-service --- build.gradle.kts | 4 +- .../eclipselink-quarkus/build.gradle.kts | 52 - ...pseLinkPolarisMetaStoreManagerFactory.java | 59 - ...olarisEclipseLinkMetaStoreSessionImpl.java | 746 ------ .../eclipselink/PolarisEclipseLinkStore.java | 450 ---- .../impl/eclipselink/PolarisSequenceUtil.java | 120 - .../main/resources/META-INF/persistence.xml | 47 - ...olarisEclipseLinkMetaStoreManagerTest.java | 91 - .../persistence/eclipselink/build.gradle.kts | 9 +- ...pseLinkPolarisMetaStoreManagerFactory.java | 19 +- ...olarisEclipseLinkMetaStoreSessionImpl.java | 3 +- gradle.properties | 2 - gradle/libs.versions.toml | 5 +- gradle/projects.main.properties | 3 - polaris-core/build.gradle.kts | 8 +- .../core}/config/RuntimeCandidate.java | 3 +- .../persistence/MetaStoreManagerFactory.java | 7 +- polaris-service-quarkus/build.gradle.kts | 264 -- .../service/admin/PolarisAdminService.java | 1719 ------------ .../service/admin/PolarisServiceImpl.java | 633 ----- .../auth/BasePolarisAuthenticator.java | 119 - .../polaris/service/auth/DecodedToken.java | 29 - .../service/auth/DefaultOAuth2ApiService.java | 131 - .../auth/DefaultPolarisAuthenticator.java | 56 - .../polaris/service/auth/JWTBroker.java | 169 -- .../polaris/service/auth/JWTRSAKeyPair.java | 43 - .../service/auth/JWTRSAKeyPairFactory.java | 53 - .../service/auth/JWTSymmetricKeyBroker.java | 41 - .../service/auth/JWTSymmetricKeyFactory.java | 84 - .../polaris/service/auth/KeyProvider.java | 28 - .../service/auth/LocalRSAKeyProvider.java | 88 - .../service/auth/OAuthTokenErrorResponse.java | 71 - .../polaris/service/auth/OAuthUtils.java | 57 - .../apache/polaris/service/auth/PemUtils.java | 95 - ...InlineBearerTokenPolarisAuthenticator.java | 113 - .../service/auth/TestOAuth2ApiService.java | 119 - .../polaris/service/auth/TokenBroker.java | 66 - .../service/auth/TokenBrokerFactory.java | 28 - .../auth/TokenInfoExchangeResponse.java | 153 -- .../service/auth/TokenRequestValidator.java | 78 - .../polaris/service/auth/TokenResponse.java | 59 - .../service/catalog/AccessDelegationMode.java | 74 - .../service/catalog/BasePolarisCatalog.java | 2113 --------------- .../catalog/IcebergCatalogAdapter.java | 476 ---- .../catalog/PolarisCatalogHandlerWrapper.java | 1150 -------- .../catalog/SupportsCredentialDelegation.java | 39 - .../catalog/SupportsNotifications.java | 27 - .../catalog/io/DefaultFileIOFactory.java | 38 - .../service/catalog/io/FileIOFactory.java | 27 - .../catalog/io/WasbTranslatingFileIO.java | 94 - .../io/WasbTranslatingFileIOFactory.java | 39 - .../config/DefaultConfigurationStore.java | 98 - .../config/RealmEntityManagerFactory.java | 61 - .../polaris/service/config/Serializers.java | 246 -- .../config/TaskHandlerConfiguration.java | 53 - .../context/CallContextCatalogFactory.java | 31 - .../service/context/CallContextResolver.java | 33 - .../PolarisCallContextCatalogFactory.java | 113 - .../service/context/RealmContextResolver.java | 34 - .../exception/IcebergExceptionMapper.java | 145 - ...IcebergJerseyViolationExceptionMapper.java | 46 - .../IcebergJsonProcessingExceptionMapper.java | 89 - .../exception/PolarisExceptionMapper.java | 58 - ...nMemoryPolarisMetaStoreManagerFactory.java | 109 - .../service/ratelimiter/NoOpRateLimiter.java | 34 - .../service/ratelimiter/RateLimiter.java | 30 - .../ratelimiter/RateLimiterFilter.java | 52 - .../RealmTokenBucketRateLimiter.java | 83 - .../ratelimiter/TokenBucketRateLimiter.java | 66 - ...PolarisStorageIntegrationProviderImpl.java | 161 -- .../task/ManifestFileCleanupTaskHandler.java | 225 -- .../service/task/TableCleanupTaskHandler.java | 168 -- .../polaris/service/task/TaskExecutor.java | 29 - .../service/task/TaskExecutorImpl.java | 151 -- .../service/task/TaskFileIOSupplier.java | 71 - .../polaris/service/task/TaskHandler.java | 27 - .../polaris/service/task/TaskUtils.java | 53 - .../service/types/CommitTableRequest.java | 23 - .../service/types/CommitViewRequest.java | 23 - .../service/types/NotificationRequest.java | 93 - .../service/types/NotificationType.java | 94 - .../types/TableUpdateNotification.java | 203 -- .../polaris/service/types/TokenType.java | 66 - .../PolarisApplicationIntegrationTest.java | 723 ----- .../admin/PolarisAdminServiceAuthzTest.java | 1842 ------------- .../service/admin/PolarisAuthzTestBase.java | 548 ---- .../admin/PolarisOverlappingCatalogTest.java | 179 -- .../PolarisServiceImplIntegrationTest.java | 2340 ----------------- .../service/auth/JWTRSAKeyPairTest.java | 155 -- .../auth/JWTSymmetricKeyGeneratorTest.java | 96 - .../auth/TokenRequestValidatorTest.java | 89 - .../polaris/service/auth/TokenUtils.java | 66 - .../catalog/AccessDelegationModeTest.java | 66 - .../catalog/BasePolarisCatalogTest.java | 1530 ----------- .../catalog/BasePolarisCatalogViewTest.java | 187 -- ...PolarisCatalogHandlerWrapperAuthzTest.java | 1829 ------------- .../PolarisPassthroughResolutionView.java | 143 - .../PolarisRestCatalogIntegrationTest.java | 973 ------- ...PolarisRestCatalogViewIntegrationTest.java | 282 -- .../catalog/PolarisSparkIntegrationTest.java | 389 --- .../catalog/io/MeasuredFileIOFactory.java | 154 -- .../service/entity/CatalogEntityTest.java | 224 -- .../MockRealmTokenBucketRateLimiter.java | 33 - .../ratelimiter/RateLimitResultAsserter.java | 42 - .../RateLimiterFilterIntegrationTest.java | 89 - .../RealmTokenBucketRateLimiterTest.java | 57 - .../polaris/service/ratelimiter/TestUtil.java | 48 - .../TokenBucketRateLimiterTest.java | 96 - .../ManifestFileCleanupTaskHandlerTest.java | 226 -- .../task/TableCleanupTaskHandlerTest.java | 364 --- .../polaris/service/task/TaskTestUtils.java | 106 - .../polaris/service/task/TestSnapshot.java | 133 - .../README-quarkus.md | 10 +- polaris-service/build.gradle.kts | 166 +- .../src/main/docker/Dockerfile.jvm | 0 .../service/BootstrapRealmsCommand.java | 85 - .../polaris/service/PolarisApplication.java | 415 --- .../polaris/service/PolarisHealthCheck.java | 29 - .../polaris/service/PurgeRealmsCommand.java | 62 - .../TimedApplicationEventListener.java | 119 - .../service/admin/PolarisAdminService.java | 30 +- .../service/admin/PolarisServiceImpl.java | 11 +- .../polaris/service/auth/Authenticator.java | 0 .../auth/BasePolarisAuthenticator.java | 18 +- .../service/auth/DefaultOAuth2ApiService.java | 44 +- .../auth/DefaultPolarisAuthenticator.java | 38 +- .../auth/DiscoverableAuthenticator.java | 38 - .../service/auth/JWTRSAKeyPairFactory.java | 33 +- .../service/auth/JWTSymmetricKeyFactory.java | 63 +- .../polaris/service/auth/KeyProvider.java | 5 +- .../service/auth/LocalRSAKeyProvider.java | 6 + .../auth/OAuthCredentialAuthFilter.java | 0 ...InlineBearerTokenPolarisAuthenticator.java | 27 +- .../service/auth/TestOAuth2ApiService.java | 29 +- .../polaris/service/auth/TokenBroker.java | 10 +- .../service/auth/TokenBrokerFactory.java | 5 +- .../service/catalog/BasePolarisCatalog.java | 20 +- .../catalog/IcebergCatalogAdapter.java | 6 +- .../catalog/io/DefaultFileIOFactory.java | 8 +- .../service/catalog/io/FileIOFactory.java | 8 +- .../io/WasbTranslatingFileIOFactory.java | 13 +- .../config/ConfigurationStoreAware.java | 27 - .../service/config/CorsConfiguration.java | 95 - .../config/DefaultConfigurationStore.java | 82 +- .../config/HasMetaStoreManagerFactory.java | 25 - .../polaris/service/config/JacksonConfig.java | 0 .../service/config/OAuth2ApiService.java | 29 - .../config/PolarisApplicationConfig.java | 281 -- .../config/PolarisQuarkusInfrastructure.java | 1 + .../config/RealmEntityManagerFactory.java | 15 +- .../config/TaskHandlerConfiguration.java | 21 +- .../service/context/CallContextResolver.java | 6 +- .../context/DefaultCallContextResolver.java | 2 +- .../context/DefaultContextResolver.java | 173 -- .../context/DefaultRealmContextResolver.java | 2 +- .../PolarisCallContextCatalogFactory.java | 4 + .../service/context/RealmContextResolver.java | 8 +- .../exception/IcebergExceptionMapper.java | 4 +- ...IcebergJerseyViolationExceptionMapper.java | 11 +- .../IcebergJsonProcessingExceptionMapper.java | 43 +- .../logging/PolarisJsonLayoutFactory.java | 242 -- ...nMemoryPolarisMetaStoreManagerFactory.java | 43 +- .../service/ratelimiter/NoOpRateLimiter.java | 8 +- .../service/ratelimiter/RateLimiter.java | 6 +- .../ratelimiter/RateLimiterFilter.java | 12 +- .../RealmTokenBucketRateLimiter.java | 38 +- .../ratelimiter/TokenBucketRateLimiter.java | 2 + ...PolarisStorageIntegrationProviderImpl.java | 78 +- .../service/task/TaskExecutorImpl.java | 31 +- .../service/task/TaskFileIOSupplier.java | 4 + .../RequestThrottlingErrorResponse.java | 33 - .../StreamReadConstraintsExceptionMapper.java | 42 - .../service/tracing/HeadersMapAccessor.java | 57 - .../service/tracing/OpenTelemetryAware.java | 26 - .../service/tracing/TracingFilter.java | 100 - .../io.dropwizard.jackson.Discoverable | 27 - ...ng.common.layout.DiscoverableLayoutFactory | 20 - ...s.core.persistence.MetaStoreManagerFactory | 21 - ...he.polaris.service.auth.TokenBrokerFactory | 21 - ...e.polaris.service.catalog.io.FileIOFactory | 21 - ...he.polaris.service.config.OAuth2ApiService | 21 - ...olaris.service.context.CallContextResolver | 20 - ...laris.service.context.RealmContextResolver | 20 - ...he.polaris.service.ratelimiter.RateLimiter | 21 - .../src/main/resources/application.properties | 0 .../src/main/resources/log4j.properties | 24 - .../org/apache/polaris/service/banner.txt | 20 - .../src/main/resources/polaris-banner.txt | 0 .../PolarisApplicationIntegrationTest.java | 240 +- .../TimedApplicationEventListenerTest.java | 234 -- .../admin/PolarisAdminServiceAuthzTest.java | 2 + .../service/admin/PolarisAuthzTestBase.java | 111 +- .../admin/PolarisOverlappingCatalogTest.java | 79 +- .../admin/PolarisOverlappingTableLaxTest.java | 0 .../PolarisOverlappingTableStrictTest.java | 0 .../admin/PolarisOverlappingTableTest.java | 301 --- .../PolarisOverlappingTableTestBase.java | 0 .../PolarisServiceImplIntegrationTest.java | 361 +-- .../auth/JWTSymmetricKeyGeneratorTest.java | 7 +- .../polaris/service/auth/TokenUtils.java | 13 +- .../catalog/BasePolarisCatalogTest.java | 54 +- .../catalog/BasePolarisCatalogViewTest.java | 84 +- ...PolarisCatalogHandlerWrapperAuthzTest.java | 30 +- .../PolarisRestCatalogIntegrationTest.java | 485 ++-- ...arisRestCatalogViewAwsIntegrationTest.java | 2 + ...isRestCatalogViewAzureIntegrationTest.java | 2 + ...risRestCatalogViewFileIntegrationTest.java | 2 + ...arisRestCatalogViewGcpIntegrationTest.java | 2 + ...PolarisRestCatalogViewIntegrationTest.java | 93 +- .../catalog/PolarisSparkIntegrationTest.java | 125 +- .../polaris/service/catalog/TestUtil.java | 63 +- .../catalog/io/FileIOIntegrationTest.java | 95 +- .../service/catalog/io/TestFileIOFactory.java | 6 +- .../config/DefaultConfigurationStoreTest.java | 19 +- .../MockRealmTokenBucketRateLimiter.java | 18 +- .../ratelimiter/RateLimiterFilterTest.java | 105 +- .../RealmTokenBucketRateLimiterTest.java | 2 +- .../polaris/service/ratelimiter/TestUtil.java | 25 +- .../TokenBucketRateLimiterTest.java | 3 +- .../ManifestFileCleanupTaskHandlerTest.java | 15 +- .../task/TableCleanupTaskHandlerTest.java | 15 +- .../polaris/service/task/TaskTestUtils.java | 4 +- .../DropwizardTestEnvironmentResolver.java | 85 - .../test/PolarisConnectionExtension.java | 211 -- .../test/PolarisIntegrationTestHelper.java | 34 +- .../polaris/service/test/PolarisRealm.java | 32 - .../test/SnowmanCredentialsExtension.java | 227 -- .../polaris/service/test/TestEnvironment.java | 37 - .../test/TestEnvironmentExtension.java | 90 - .../service/test/TestEnvironmentResolver.java | 26 - .../polaris/service/test/TestMetricsUtil.java | 13 +- ...ris.service.auth.DiscoverableAuthenticator | 20 - ...e.polaris.service.catalog.io.FileIOFactory | 20 - ...he.polaris.service.ratelimiter.RateLimiter | 20 - .../polaris-server-integrationtest.yml | 161 -- 235 files changed, 1544 insertions(+), 30892 deletions(-) delete mode 100644 extension/persistence/eclipselink-quarkus/build.gradle.kts delete mode 100644 extension/persistence/eclipselink-quarkus/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java delete mode 100644 extension/persistence/eclipselink-quarkus/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java delete mode 100644 extension/persistence/eclipselink-quarkus/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java delete mode 100644 extension/persistence/eclipselink-quarkus/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisSequenceUtil.java delete mode 100644 extension/persistence/eclipselink-quarkus/src/main/resources/META-INF/persistence.xml delete mode 100644 extension/persistence/eclipselink-quarkus/src/test/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreManagerTest.java rename {polaris-service-quarkus/src/main/java/org/apache/polaris/service => polaris-core/src/main/java/org/apache/polaris/core}/config/RuntimeCandidate.java (95%) delete mode 100644 polaris-service-quarkus/build.gradle.kts delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/BasePolarisAuthenticator.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/DecodedToken.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/DefaultOAuth2ApiService.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/DefaultPolarisAuthenticator.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/JWTBroker.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/JWTRSAKeyPair.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/JWTRSAKeyPairFactory.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/JWTSymmetricKeyBroker.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/JWTSymmetricKeyFactory.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/KeyProvider.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/LocalRSAKeyProvider.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/OAuthTokenErrorResponse.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/OAuthUtils.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/PemUtils.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/TestInlineBearerTokenPolarisAuthenticator.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/TestOAuth2ApiService.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/TokenBroker.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/TokenBrokerFactory.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/TokenInfoExchangeResponse.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/TokenRequestValidator.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/TokenResponse.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/AccessDelegationMode.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/SupportsCredentialDelegation.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/SupportsNotifications.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/io/FileIOFactory.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/io/WasbTranslatingFileIO.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/io/WasbTranslatingFileIOFactory.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/config/DefaultConfigurationStore.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/config/RealmEntityManagerFactory.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/config/Serializers.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/config/TaskHandlerConfiguration.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/context/CallContextCatalogFactory.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/context/CallContextResolver.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/context/RealmContextResolver.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/exception/IcebergExceptionMapper.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/exception/IcebergJerseyViolationExceptionMapper.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/exception/IcebergJsonProcessingExceptionMapper.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/exception/PolarisExceptionMapper.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/ratelimiter/NoOpRateLimiter.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiter.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiterFilter.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/ratelimiter/RealmTokenBucketRateLimiter.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/ratelimiter/TokenBucketRateLimiter.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/task/ManifestFileCleanupTaskHandler.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/task/TableCleanupTaskHandler.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/task/TaskExecutor.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/task/TaskExecutorImpl.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/task/TaskHandler.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/task/TaskUtils.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/types/CommitTableRequest.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/types/CommitViewRequest.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/types/NotificationRequest.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/types/NotificationType.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/types/TableUpdateNotification.java delete mode 100644 polaris-service-quarkus/src/main/java/org/apache/polaris/service/types/TokenType.java delete mode 100644 polaris-service-quarkus/src/test/java/org/apache/polaris/service/PolarisApplicationIntegrationTest.java delete mode 100644 polaris-service-quarkus/src/test/java/org/apache/polaris/service/admin/PolarisAdminServiceAuthzTest.java delete mode 100644 polaris-service-quarkus/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java delete mode 100644 polaris-service-quarkus/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingCatalogTest.java delete mode 100644 polaris-service-quarkus/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplIntegrationTest.java delete mode 100644 polaris-service-quarkus/src/test/java/org/apache/polaris/service/auth/JWTRSAKeyPairTest.java delete mode 100644 polaris-service-quarkus/src/test/java/org/apache/polaris/service/auth/JWTSymmetricKeyGeneratorTest.java delete mode 100644 polaris-service-quarkus/src/test/java/org/apache/polaris/service/auth/TokenRequestValidatorTest.java delete mode 100644 polaris-service-quarkus/src/test/java/org/apache/polaris/service/auth/TokenUtils.java delete mode 100644 polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/AccessDelegationModeTest.java delete mode 100644 polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java delete mode 100644 polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogViewTest.java delete mode 100644 polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapperAuthzTest.java delete mode 100644 polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/PolarisPassthroughResolutionView.java delete mode 100644 polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogIntegrationTest.java delete mode 100644 polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogViewIntegrationTest.java delete mode 100644 polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/PolarisSparkIntegrationTest.java delete mode 100644 polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/io/MeasuredFileIOFactory.java delete mode 100644 polaris-service-quarkus/src/test/java/org/apache/polaris/service/entity/CatalogEntityTest.java delete mode 100644 polaris-service-quarkus/src/test/java/org/apache/polaris/service/ratelimiter/MockRealmTokenBucketRateLimiter.java delete mode 100644 polaris-service-quarkus/src/test/java/org/apache/polaris/service/ratelimiter/RateLimitResultAsserter.java delete mode 100644 polaris-service-quarkus/src/test/java/org/apache/polaris/service/ratelimiter/RateLimiterFilterIntegrationTest.java delete mode 100644 polaris-service-quarkus/src/test/java/org/apache/polaris/service/ratelimiter/RealmTokenBucketRateLimiterTest.java delete mode 100644 polaris-service-quarkus/src/test/java/org/apache/polaris/service/ratelimiter/TestUtil.java delete mode 100644 polaris-service-quarkus/src/test/java/org/apache/polaris/service/ratelimiter/TokenBucketRateLimiterTest.java delete mode 100644 polaris-service-quarkus/src/test/java/org/apache/polaris/service/task/ManifestFileCleanupTaskHandlerTest.java delete mode 100644 polaris-service-quarkus/src/test/java/org/apache/polaris/service/task/TableCleanupTaskHandlerTest.java delete mode 100644 polaris-service-quarkus/src/test/java/org/apache/polaris/service/task/TaskTestUtils.java delete mode 100644 polaris-service-quarkus/src/test/java/org/apache/polaris/service/task/TestSnapshot.java rename polaris-service-quarkus/README.md => polaris-service/README-quarkus.md (91%) rename {polaris-service-quarkus => polaris-service}/src/main/docker/Dockerfile.jvm (100%) delete mode 100644 polaris-service/src/main/java/org/apache/polaris/service/BootstrapRealmsCommand.java delete mode 100644 polaris-service/src/main/java/org/apache/polaris/service/PolarisApplication.java delete mode 100644 polaris-service/src/main/java/org/apache/polaris/service/PolarisHealthCheck.java delete mode 100644 polaris-service/src/main/java/org/apache/polaris/service/PurgeRealmsCommand.java delete mode 100644 polaris-service/src/main/java/org/apache/polaris/service/TimedApplicationEventListener.java rename {polaris-service-quarkus => polaris-service}/src/main/java/org/apache/polaris/service/auth/Authenticator.java (100%) delete mode 100644 polaris-service/src/main/java/org/apache/polaris/service/auth/DiscoverableAuthenticator.java rename {polaris-service-quarkus => polaris-service}/src/main/java/org/apache/polaris/service/auth/OAuthCredentialAuthFilter.java (100%) delete mode 100644 polaris-service/src/main/java/org/apache/polaris/service/config/ConfigurationStoreAware.java delete mode 100644 polaris-service/src/main/java/org/apache/polaris/service/config/CorsConfiguration.java delete mode 100644 polaris-service/src/main/java/org/apache/polaris/service/config/HasMetaStoreManagerFactory.java rename {polaris-service-quarkus => polaris-service}/src/main/java/org/apache/polaris/service/config/JacksonConfig.java (100%) delete mode 100644 polaris-service/src/main/java/org/apache/polaris/service/config/OAuth2ApiService.java delete mode 100644 polaris-service/src/main/java/org/apache/polaris/service/config/PolarisApplicationConfig.java rename {polaris-service-quarkus => polaris-service}/src/main/java/org/apache/polaris/service/config/PolarisQuarkusInfrastructure.java (99%) rename {polaris-service-quarkus => polaris-service}/src/main/java/org/apache/polaris/service/context/DefaultCallContextResolver.java (98%) delete mode 100644 polaris-service/src/main/java/org/apache/polaris/service/context/DefaultContextResolver.java rename {polaris-service-quarkus => polaris-service}/src/main/java/org/apache/polaris/service/context/DefaultRealmContextResolver.java (98%) delete mode 100644 polaris-service/src/main/java/org/apache/polaris/service/logging/PolarisJsonLayoutFactory.java delete mode 100644 polaris-service/src/main/java/org/apache/polaris/service/throttling/RequestThrottlingErrorResponse.java delete mode 100644 polaris-service/src/main/java/org/apache/polaris/service/throttling/StreamReadConstraintsExceptionMapper.java delete mode 100644 polaris-service/src/main/java/org/apache/polaris/service/tracing/HeadersMapAccessor.java delete mode 100644 polaris-service/src/main/java/org/apache/polaris/service/tracing/OpenTelemetryAware.java delete mode 100644 polaris-service/src/main/java/org/apache/polaris/service/tracing/TracingFilter.java delete mode 100644 polaris-service/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable delete mode 100644 polaris-service/src/main/resources/META-INF/services/io.dropwizard.logging.common.layout.DiscoverableLayoutFactory delete mode 100644 polaris-service/src/main/resources/META-INF/services/org.apache.polaris.core.persistence.MetaStoreManagerFactory delete mode 100644 polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.auth.TokenBrokerFactory delete mode 100644 polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.catalog.io.FileIOFactory delete mode 100644 polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.config.OAuth2ApiService delete mode 100644 polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.context.CallContextResolver delete mode 100644 polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.context.RealmContextResolver delete mode 100644 polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.ratelimiter.RateLimiter rename {polaris-service-quarkus => polaris-service}/src/main/resources/application.properties (100%) delete mode 100644 polaris-service/src/main/resources/log4j.properties delete mode 100644 polaris-service/src/main/resources/org/apache/polaris/service/banner.txt rename {polaris-service-quarkus => polaris-service}/src/main/resources/polaris-banner.txt (100%) delete mode 100644 polaris-service/src/test/java/org/apache/polaris/service/TimedApplicationEventListenerTest.java rename {polaris-service-quarkus => polaris-service}/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingTableLaxTest.java (100%) rename {polaris-service-quarkus => polaris-service}/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingTableStrictTest.java (100%) delete mode 100644 polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingTableTest.java rename {polaris-service-quarkus => polaris-service}/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingTableTestBase.java (100%) delete mode 100644 polaris-service/src/test/java/org/apache/polaris/service/test/DropwizardTestEnvironmentResolver.java delete mode 100644 polaris-service/src/test/java/org/apache/polaris/service/test/PolarisConnectionExtension.java rename {polaris-service-quarkus => polaris-service}/src/test/java/org/apache/polaris/service/test/PolarisIntegrationTestHelper.java (90%) delete mode 100644 polaris-service/src/test/java/org/apache/polaris/service/test/PolarisRealm.java delete mode 100644 polaris-service/src/test/java/org/apache/polaris/service/test/SnowmanCredentialsExtension.java delete mode 100644 polaris-service/src/test/java/org/apache/polaris/service/test/TestEnvironment.java delete mode 100644 polaris-service/src/test/java/org/apache/polaris/service/test/TestEnvironmentExtension.java delete mode 100644 polaris-service/src/test/java/org/apache/polaris/service/test/TestEnvironmentResolver.java delete mode 100644 polaris-service/src/test/resources/META-INF/services/org.apache.polaris.service.auth.DiscoverableAuthenticator delete mode 100644 polaris-service/src/test/resources/META-INF/services/org.apache.polaris.service.catalog.io.FileIOFactory delete mode 100644 polaris-service/src/test/resources/META-INF/services/org.apache.polaris.service.ratelimiter.RateLimiter delete mode 100644 polaris-service/src/test/resources/polaris-server-integrationtest.yml diff --git a/build.gradle.kts b/build.gradle.kts index 856f710cb..96069bd46 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -84,11 +84,9 @@ tasks.named("rat").configure { excludes.add("gradle/wrapper/gradle-wrapper*.jar*") excludes.add("logs/**") - excludes.add("polaris-service/src/**/banner.txt") + excludes.add("polaris-service/src/**/polaris-banner.txt") excludes.add("polaris-service/logs") - excludes.add("polaris-service-quarkus/src/**/polaris-banner.txt") - excludes.add("site/node_modules/**") excludes.add("site/layouts/robots.txt") // Ignore generated stuff, when the Hugo is run w/o Docker diff --git a/extension/persistence/eclipselink-quarkus/build.gradle.kts b/extension/persistence/eclipselink-quarkus/build.gradle.kts deleted file mode 100644 index f4211fda7..000000000 --- a/extension/persistence/eclipselink-quarkus/build.gradle.kts +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -plugins { - alias(libs.plugins.quarkus) - id("polaris-server") -} - -dependencies { - implementation(project(":polaris-core")) - // TODO @RuntimeCandidate is in polaris-service-quarkus, maybe move a polaris-spi module ? - implementation(project(":polaris-service-quarkus")) - implementation(libs.eclipselink) - implementation(platform(libs.quarkus.bom)) - - implementation(libs.slf4j.api) - implementation(libs.guava) - - // TODO used for Discoverable, it will be removed with Dropwizard - implementation(platform(libs.dropwizard.bom)) - implementation("io.dropwizard:dropwizard-core") - - compileOnly(libs.jakarta.enterprise.cdi.api) - compileOnly(libs.jakarta.inject.api) - compileOnly("io.quarkus:quarkus-arc") - compileOnly("org.eclipse.microprofile.config:microprofile-config-api") - - testImplementation(libs.h2) - testImplementation(testFixtures(project(":polaris-core"))) - - testImplementation(platform(libs.junit.bom)) - testImplementation("org.junit.jupiter:junit-jupiter") - testImplementation(libs.assertj.core) - testImplementation(libs.mockito.core) - testRuntimeOnly("org.junit.platform:junit-platform-launcher") -} diff --git a/extension/persistence/eclipselink-quarkus/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java b/extension/persistence/eclipselink-quarkus/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java deleted file mode 100644 index 355d66d66..000000000 --- a/extension/persistence/eclipselink-quarkus/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.extension.persistence.impl.eclipselink; - -import io.quarkus.arc.lookup.LookupIfProperty; -import jakarta.enterprise.context.ApplicationScoped; -import org.apache.polaris.core.PolarisDiagnostics; -import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.core.persistence.LocalPolarisMetaStoreManagerFactory; -import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.PolarisMetaStoreSession; -import org.apache.polaris.service.config.RuntimeCandidate; -import org.eclipse.microprofile.config.inject.ConfigProperty; - -/** - * The implementation of Configuration interface for configuring the {@link PolarisMetaStoreManager} - * using an EclipseLink based meta store to store and retrieve all Polaris metadata. It can be - * configured through persistence.xml to use supported RDBMS as the meta store. - */ -@ApplicationScoped -@RuntimeCandidate -@LookupIfProperty(name = "polaris.persistence.metastore-manager.type", stringValue = "eclipse-link") -public class EclipseLinkPolarisMetaStoreManagerFactory - extends LocalPolarisMetaStoreManagerFactory { - - @ConfigProperty(name = "polaris.eclipselink.conf-file") - private String confFile; - - @ConfigProperty(name = "polaris.eclipselink.persistence-unit", defaultValue = "polaris") - private String persistenceUnitName; - - @Override - protected PolarisEclipseLinkStore createBackingStore(PolarisDiagnostics diagnostics) { - return new PolarisEclipseLinkStore(diagnostics); - } - - @Override - protected PolarisMetaStoreSession createMetaStoreSession( - PolarisEclipseLinkStore store, RealmContext realmContext) { - return new PolarisEclipseLinkMetaStoreSessionImpl( - store, storageIntegration, realmContext, confFile, persistenceUnitName); - } -} diff --git a/extension/persistence/eclipselink-quarkus/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java b/extension/persistence/eclipselink-quarkus/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java deleted file mode 100644 index f1be685a7..000000000 --- a/extension/persistence/eclipselink-quarkus/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java +++ /dev/null @@ -1,746 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.extension.persistence.impl.eclipselink; - -import static org.eclipse.persistence.config.PersistenceUnitProperties.ECLIPSELINK_PERSISTENCE_XML; -import static org.eclipse.persistence.config.PersistenceUnitProperties.JDBC_URL; - -import com.google.common.base.Predicates; -import jakarta.persistence.EntityManager; -import jakarta.persistence.EntityManagerFactory; -import jakarta.persistence.EntityTransaction; -import jakarta.persistence.OptimisticLockException; -import jakarta.persistence.Persistence; -import jakarta.persistence.PersistenceException; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.net.URL; -import java.net.URLClassLoader; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.function.Supplier; -import java.util.stream.Collectors; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.xpath.XPath; -import javax.xml.xpath.XPathConstants; -import javax.xml.xpath.XPathExpressionException; -import javax.xml.xpath.XPathFactory; -import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.core.entity.PolarisBaseEntity; -import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; -import org.apache.polaris.core.entity.PolarisEntitiesActiveKey; -import org.apache.polaris.core.entity.PolarisEntityActiveRecord; -import org.apache.polaris.core.entity.PolarisEntityCore; -import org.apache.polaris.core.entity.PolarisEntityId; -import org.apache.polaris.core.entity.PolarisEntityType; -import org.apache.polaris.core.entity.PolarisGrantRecord; -import org.apache.polaris.core.entity.PolarisPrincipalSecrets; -import org.apache.polaris.core.exceptions.AlreadyExistsException; -import org.apache.polaris.core.persistence.PolarisMetaStoreManagerImpl; -import org.apache.polaris.core.persistence.PolarisMetaStoreSession; -import org.apache.polaris.core.persistence.RetryOnConcurrencyException; -import org.apache.polaris.core.persistence.models.ModelEntity; -import org.apache.polaris.core.persistence.models.ModelEntityActive; -import org.apache.polaris.core.persistence.models.ModelEntityChangeTracking; -import org.apache.polaris.core.persistence.models.ModelGrantRecord; -import org.apache.polaris.core.persistence.models.ModelPrincipalSecrets; -import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; -import org.apache.polaris.core.storage.PolarisStorageIntegration; -import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.w3c.dom.Document; -import org.w3c.dom.NamedNodeMap; -import org.w3c.dom.NodeList; -import org.xml.sax.SAXException; - -/** - * EclipseLink implementation of a Polaris metadata store supporting persisting and retrieving all - * Polaris metadata from/to the configured database systems. - */ -public class PolarisEclipseLinkMetaStoreSessionImpl implements PolarisMetaStoreSession { - private static final Logger LOGGER = - LoggerFactory.getLogger(PolarisEclipseLinkMetaStoreSessionImpl.class); - - // Cache to hold the EntityManagerFactory for each realm. Each realm needs a separate - // EntityManagerFactory since it connects to different databases - private static final ConcurrentHashMap realmFactories = - new ConcurrentHashMap<>(); - private final EntityManagerFactory emf; - private final ThreadLocal localSession = new ThreadLocal<>(); - private final PolarisEclipseLinkStore store; - private final PolarisStorageIntegrationProvider storageIntegrationProvider; - - /** - * Create a meta store session against provided realm. Each realm has its own database. - * - * @param store Backing store of EclipseLink implementation - * @param storageIntegrationProvider Storage integration provider - * @param realmContext Realm context used to communicate with different database. - * @param confFile Optional EclipseLink configuration file. Default to 'META-INF/persistence.xml'. - * @param persistenceUnitName Optional persistence-unit name in confFile. Default to 'polaris'. - */ - public PolarisEclipseLinkMetaStoreSessionImpl( - PolarisEclipseLinkStore store, - PolarisStorageIntegrationProvider storageIntegrationProvider, - RealmContext realmContext, - String confFile, - String persistenceUnitName) { - LOGGER.debug( - "Creating EclipseLink Meta Store Session for realm {}", realmContext.getRealmIdentifier()); - emf = createEntityManagerFactory(realmContext, confFile, persistenceUnitName); - - // init store - this.store = store; - try (EntityManager session = emf.createEntityManager()) { - this.store.initialize(session); - } - this.storageIntegrationProvider = storageIntegrationProvider; - } - - /** - * Create EntityManagerFactory. - * - *

The EntityManagerFactory creation is expensive, so we are caching and reusing it for each - * realm. - */ - private EntityManagerFactory createEntityManagerFactory( - RealmContext realmContext, String confFile, String persistenceUnitName) { - String realm = realmContext.getRealmIdentifier(); - EntityManagerFactory factory = realmFactories.getOrDefault(realm, null); - if (factory != null) { - return factory; - } - - ClassLoader prevClassLoader = Thread.currentThread().getContextClassLoader(); - try { - persistenceUnitName = persistenceUnitName == null ? "polaris" : persistenceUnitName; - confFile = confFile == null ? "META-INF/persistence.xml" : confFile; - - // Currently eclipseLink can only support configuration as a resource inside a jar. To support - // external configuration, persistence.xml needs be placed inside a jar and here is to add the - // jar to the classpath. - // Supported configuration file: META-INF/persistence.xml, /tmp/conf.jar!/persistence.xml - int splitPosition = confFile.indexOf("!/"); - if (splitPosition != -1) { - String jarPrefixPath = confFile.substring(0, splitPosition); - confFile = confFile.substring(splitPosition + 2); - URL prefixUrl = this.getClass().getClassLoader().getResource(jarPrefixPath); - if (prefixUrl == null) { - prefixUrl = new File(jarPrefixPath).toURI().toURL(); - } - - LOGGER.debug( - "Creating a new ClassLoader with the jar {} in classpath to load the config file", - prefixUrl); - - URLClassLoader currentClassLoader = - new URLClassLoader(new URL[] {prefixUrl}, this.getClass().getClassLoader()); - - LOGGER.debug("Update ClassLoader in current thread temporarily"); - Thread.currentThread().setContextClassLoader(currentClassLoader); - } - - Map properties = loadProperties(confFile, persistenceUnitName); - // Replace database name in JDBC URL with realm - if (properties.containsKey(JDBC_URL)) { - properties.put(JDBC_URL, properties.get(JDBC_URL).replace("{realm}", realm)); - } - properties.put(ECLIPSELINK_PERSISTENCE_XML, confFile); - - factory = Persistence.createEntityManagerFactory(persistenceUnitName, properties); - realmFactories.putIfAbsent(realm, factory); - - return factory; - } catch (IOException e) { - throw new RuntimeException(e); - } finally { - Thread.currentThread().setContextClassLoader(prevClassLoader); - } - } - - static void clearEntityManagerFactories() { - realmFactories.clear(); - } - - /** Load the persistence unit properties from a given configuration file */ - private Map loadProperties(String confFile, String persistenceUnitName) - throws IOException { - try { - InputStream input = - Thread.currentThread().getContextClassLoader().getResourceAsStream(confFile); - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - DocumentBuilder builder = factory.newDocumentBuilder(); - Document doc = builder.parse(input); - XPath xPath = XPathFactory.newInstance().newXPath(); - String expression = - "/persistence/persistence-unit[@name='" + persistenceUnitName + "']/properties/property"; - NodeList nodeList = - (NodeList) xPath.compile(expression).evaluate(doc, XPathConstants.NODESET); - Map properties = new HashMap<>(); - for (int i = 0; i < nodeList.getLength(); i++) { - NamedNodeMap nodeMap = nodeList.item(i).getAttributes(); - properties.put( - nodeMap.getNamedItem("name").getNodeValue(), - nodeMap.getNamedItem("value").getNodeValue()); - } - - return properties; - } catch (XPathExpressionException - | ParserConfigurationException - | SAXException - | IOException e) { - String str = - String.format( - "Cannot find or parse the configuration file %s for persistence-unit %s", - confFile, persistenceUnitName); - LOGGER.error(str, e); - throw new IOException(str); - } - } - - /** {@inheritDoc} */ - @Override - public T runInTransaction(PolarisCallContext callCtx, Supplier transactionCode) { - callCtx.getDiagServices().check(localSession.get() == null, "cannot nest transaction"); - - try (EntityManager session = emf.createEntityManager()) { - localSession.set(session); - EntityTransaction tr = session.getTransaction(); - try { - tr.begin(); - - T result = transactionCode.get(); - - // Commit when it's not rolled back by the client - if (session.getTransaction().isActive()) { - tr.commit(); - LOGGER.debug("transaction committed"); - } - - return result; - } catch (Exception e) { - tr.rollback(); - LOGGER.debug("transaction rolled back", e); - - if (e instanceof OptimisticLockException - || e.getCause() instanceof OptimisticLockException) { - throw new RetryOnConcurrencyException(e); - } - - throw e; - } finally { - localSession.remove(); - } - } catch (PersistenceException e) { - if (e.toString().toLowerCase(Locale.ROOT).contains("duplicate key")) { - throw new AlreadyExistsException("Duplicate key error when persisting entity", e); - } else { - throw e; - } - } - } - - /** {@inheritDoc} */ - @Override - public void runActionInTransaction(PolarisCallContext callCtx, Runnable transactionCode) { - callCtx.getDiagServices().check(localSession.get() == null, "cannot nest transaction"); - - try (EntityManager session = emf.createEntityManager()) { - localSession.set(session); - EntityTransaction tr = session.getTransaction(); - try { - tr.begin(); - - transactionCode.run(); - - // Commit when it's not rolled back by the client - if (session.getTransaction().isActive()) { - tr.commit(); - LOGGER.debug("transaction committed"); - } - } catch (Exception e) { - LOGGER.debug("Rolling back transaction due to an error", e); - tr.rollback(); - - if (e instanceof OptimisticLockException - || e.getCause() instanceof OptimisticLockException) { - throw new RetryOnConcurrencyException(e); - } - - throw e; - } finally { - localSession.remove(); - } - } - } - - /** {@inheritDoc} */ - @Override - public T runInReadTransaction(PolarisCallContext callCtx, Supplier transactionCode) { - // EclipseLink doesn't support readOnly transaction - return runInTransaction(callCtx, transactionCode); - } - - /** {@inheritDoc} */ - @Override - public void runActionInReadTransaction(PolarisCallContext callCtx, Runnable transactionCode) { - // EclipseLink doesn't support readOnly transaction - runActionInTransaction(callCtx, transactionCode); - } - - /** - * @return new unique entity identifier - */ - @Override - public long generateNewId(PolarisCallContext callCtx) { - // This function can be called within a transaction or out of transaction. - // If called out of transaction, create a new transaction, otherwise run in current transaction - return localSession.get() != null - ? this.store.getNextSequence(localSession.get()) - : runInReadTransaction(callCtx, () -> generateNewId(callCtx)); - } - - /** {@inheritDoc} */ - @Override - public void writeToEntities(PolarisCallContext callCtx, PolarisBaseEntity entity) { - this.store.writeToEntities(localSession.get(), entity); - } - - /** {@inheritDoc} */ - @Override - public void persistStorageIntegrationIfNeeded( - PolarisCallContext callContext, - PolarisBaseEntity entity, - PolarisStorageIntegration storageIntegration) { - // not implemented for eclipselink store - } - - /** {@inheritDoc} */ - @Override - public void writeToEntitiesActive(PolarisCallContext callCtx, PolarisBaseEntity entity) { - // write it - this.store.writeToEntitiesActive(localSession.get(), entity); - } - - /** {@inheritDoc} */ - @Override - public void writeToEntitiesDropped(PolarisCallContext callCtx, PolarisBaseEntity entity) { - // write it - this.store.writeToEntitiesDropped(localSession.get(), entity); - } - - /** {@inheritDoc} */ - @Override - public void writeToEntitiesChangeTracking(PolarisCallContext callCtx, PolarisBaseEntity entity) { - // write it - this.store.writeToEntitiesChangeTracking(localSession.get(), entity); - } - - /** {@inheritDoc} */ - @Override - public void writeToGrantRecords(PolarisCallContext callCtx, PolarisGrantRecord grantRec) { - // write it - this.store.writeToGrantRecords(localSession.get(), grantRec); - } - - /** {@inheritDoc} */ - @Override - public void deleteFromEntities(PolarisCallContext callCtx, PolarisEntityCore entity) { - - // delete it - this.store.deleteFromEntities(localSession.get(), entity.getCatalogId(), entity.getId()); - } - - /** {@inheritDoc} */ - @Override - public void deleteFromEntitiesActive(PolarisCallContext callCtx, PolarisEntityCore entity) { - // delete it - this.store.deleteFromEntitiesActive(localSession.get(), new PolarisEntitiesActiveKey(entity)); - } - - /** {@inheritDoc} */ - @Override - public void deleteFromEntitiesDropped(PolarisCallContext callCtx, PolarisBaseEntity entity) { - // delete it - this.store.deleteFromEntitiesDropped(localSession.get(), entity.getCatalogId(), entity.getId()); - } - - /** - * {@inheritDoc} - * - * @param callCtx - * @param entity entity record to delete - */ - @Override - public void deleteFromEntitiesChangeTracking( - PolarisCallContext callCtx, PolarisEntityCore entity) { - // delete it - this.store.deleteFromEntitiesChangeTracking(localSession.get(), entity); - } - - /** {@inheritDoc} */ - @Override - public void deleteFromGrantRecords(PolarisCallContext callCtx, PolarisGrantRecord grantRec) { - this.store.deleteFromGrantRecords(localSession.get(), grantRec); - } - - /** {@inheritDoc} */ - @Override - public void deleteAllEntityGrantRecords( - PolarisCallContext callCtx, - PolarisEntityCore entity, - List grantsOnGrantee, - List grantsOnSecurable) { - this.store.deleteAllEntityGrantRecords(localSession.get(), entity); - } - - /** {@inheritDoc} */ - @Override - public void deleteAll(PolarisCallContext callCtx) { - this.store.deleteAll(localSession.get()); - } - - /** {@inheritDoc} */ - @Override - public PolarisBaseEntity lookupEntity(PolarisCallContext callCtx, long catalogId, long entityId) { - return ModelEntity.toEntity(this.store.lookupEntity(localSession.get(), catalogId, entityId)); - } - - @Override - public List lookupEntities( - PolarisCallContext callCtx, List entityIds) { - return this.store.lookupEntities(localSession.get(), entityIds).stream() - .map(ModelEntity::toEntity) - .toList(); - } - - /** {@inheritDoc} */ - @Override - public int lookupEntityVersion(PolarisCallContext callCtx, long catalogId, long entityId) { - ModelEntity model = this.store.lookupEntity(localSession.get(), catalogId, entityId); - return model == null ? 0 : model.getEntityVersion(); - } - - /** {@inheritDoc} */ - @Override - public List lookupEntityVersions( - PolarisCallContext callCtx, List entityIds) { - Map idToEntityMap = - this.store.lookupEntities(localSession.get(), entityIds).stream() - .collect( - Collectors.toMap( - entry -> new PolarisEntityId(entry.getCatalogId(), entry.getId()), - entry -> entry)); - return entityIds.stream() - .map( - entityId -> { - ModelEntity entity = idToEntityMap.getOrDefault(entityId, null); - return entity == null - ? null - : new PolarisChangeTrackingVersions( - entity.getEntityVersion(), entity.getGrantRecordsVersion()); - }) - .collect(Collectors.toList()); - } - - /** {@inheritDoc} */ - @Override - public PolarisEntityActiveRecord lookupEntityActive( - PolarisCallContext callCtx, PolarisEntitiesActiveKey entityActiveKey) { - // lookup the active entity slice - return ModelEntityActive.toEntityActive( - this.store.lookupEntityActive(localSession.get(), entityActiveKey)); - } - - /** {@inheritDoc} */ - @Override - public List lookupEntityActiveBatch( - PolarisCallContext callCtx, List entityActiveKeys) { - // now build a list to quickly verify that nothing has changed - return entityActiveKeys.stream() - .map(entityActiveKey -> this.lookupEntityActive(callCtx, entityActiveKey)) - .collect(Collectors.toList()); - } - - /** {@inheritDoc} */ - @Override - public List listActiveEntities( - PolarisCallContext callCtx, long catalogId, long parentId, PolarisEntityType entityType) { - return listActiveEntities(callCtx, catalogId, parentId, entityType, Predicates.alwaysTrue()); - } - - @Override - public List listActiveEntities( - PolarisCallContext callCtx, - long catalogId, - long parentId, - PolarisEntityType entityType, - Predicate entityFilter) { - // full range scan under the parent for that type - return listActiveEntities( - callCtx, - catalogId, - parentId, - entityType, - Integer.MAX_VALUE, - entityFilter, - entity -> - new PolarisEntityActiveRecord( - entity.getCatalogId(), - entity.getId(), - entity.getParentId(), - entity.getName(), - entity.getTypeCode(), - entity.getSubTypeCode())); - } - - @Override - public List listActiveEntities( - PolarisCallContext callCtx, - long catalogId, - long parentId, - PolarisEntityType entityType, - int limit, - Predicate entityFilter, - Function transformer) { - // full range scan under the parent for that type - return this.store - .lookupFullEntitiesActive(localSession.get(), catalogId, parentId, entityType) - .stream() - .map(ModelEntity::toEntity) - .filter(entityFilter) - .limit(limit) - .map(transformer) - .collect(Collectors.toList()); - } - - /** {@inheritDoc} */ - @Override - public boolean hasChildren( - PolarisCallContext callContext, PolarisEntityType entityType, long catalogId, long parentId) { - // check if it has children - return this.store.countActiveChildEntities(localSession.get(), catalogId, parentId, entityType) - > 0; - } - - /** {@inheritDoc} */ - @Override - public int lookupEntityGrantRecordsVersion( - PolarisCallContext callCtx, long catalogId, long entityId) { - ModelEntityChangeTracking entity = - this.store.lookupEntityChangeTracking(localSession.get(), catalogId, entityId); - - // does not exist, 0 - return entity == null ? 0 : entity.getGrantRecordsVersion(); - } - - /** {@inheritDoc} */ - @Override - public PolarisGrantRecord lookupGrantRecord( - PolarisCallContext callCtx, - long securableCatalogId, - long securableId, - long granteeCatalogId, - long granteeId, - int privilegeCode) { - // lookup the grants records slice to find the usage role - return ModelGrantRecord.toGrantRecord( - this.store.lookupGrantRecord( - localSession.get(), - securableCatalogId, - securableId, - granteeCatalogId, - granteeId, - privilegeCode)); - } - - /** {@inheritDoc} */ - @Override - public List loadAllGrantRecordsOnSecurable( - PolarisCallContext callCtx, long securableCatalogId, long securableId) { - // now fetch all grants for this securable - return this.store - .lookupAllGrantRecordsOnSecurable(localSession.get(), securableCatalogId, securableId) - .stream() - .map(ModelGrantRecord::toGrantRecord) - .toList(); - } - - /** {@inheritDoc} */ - @Override - public List loadAllGrantRecordsOnGrantee( - PolarisCallContext callCtx, long granteeCatalogId, long granteeId) { - // now fetch all grants assigned to this grantee - return this.store - .lookupGrantRecordsOnGrantee(localSession.get(), granteeCatalogId, granteeId) - .stream() - .map(ModelGrantRecord::toGrantRecord) - .toList(); - } - - /** {@inheritDoc} */ - @Override - public PolarisPrincipalSecrets loadPrincipalSecrets(PolarisCallContext callCtx, String clientId) { - return ModelPrincipalSecrets.toPrincipalSecrets( - this.store.lookupPrincipalSecrets(localSession.get(), clientId)); - } - - /** {@inheritDoc} */ - @Override - public PolarisPrincipalSecrets generateNewPrincipalSecrets( - PolarisCallContext callCtx, String principalName, long principalId) { - // ensure principal client id is unique - PolarisPrincipalSecrets principalSecrets; - ModelPrincipalSecrets lookupPrincipalSecrets; - do { - // generate new random client id and secrets - principalSecrets = new PolarisPrincipalSecrets(principalId); - - // load the existing secrets - lookupPrincipalSecrets = - this.store.lookupPrincipalSecrets( - localSession.get(), principalSecrets.getPrincipalClientId()); - } while (lookupPrincipalSecrets != null); - - // write new principal secrets - this.store.writePrincipalSecrets(localSession.get(), principalSecrets); - - // if not found, return null - return principalSecrets; - } - - /** {@inheritDoc} */ - @Override - public PolarisPrincipalSecrets rotatePrincipalSecrets( - PolarisCallContext callCtx, - String clientId, - long principalId, - String mainSecretToRotate, - boolean reset) { - - // load the existing secrets - PolarisPrincipalSecrets principalSecrets = - ModelPrincipalSecrets.toPrincipalSecrets( - this.store.lookupPrincipalSecrets(localSession.get(), clientId)); - - // should be found - callCtx - .getDiagServices() - .checkNotNull( - principalSecrets, - "cannot_find_secrets", - "client_id={} principalId={}", - clientId, - principalId); - - // ensure principal id is matching - callCtx - .getDiagServices() - .check( - principalId == principalSecrets.getPrincipalId(), - "principal_id_mismatch", - "expectedId={} id={}", - principalId, - principalSecrets.getPrincipalId()); - - // rotate the secrets - principalSecrets.rotateSecrets(mainSecretToRotate); - if (reset) { - principalSecrets.rotateSecrets(principalSecrets.getMainSecret()); - } - - // write back new secrets - this.store.writePrincipalSecrets(localSession.get(), principalSecrets); - - // return those - return principalSecrets; - } - - /** {@inheritDoc} */ - @Override - public void deletePrincipalSecrets( - PolarisCallContext callCtx, String clientId, long principalId) { - // load the existing secrets - ModelPrincipalSecrets principalSecrets = - this.store.lookupPrincipalSecrets(localSession.get(), clientId); - - // should be found - callCtx - .getDiagServices() - .checkNotNull( - principalSecrets, - "cannot_find_secrets", - "client_id={} principalId={}", - clientId, - principalId); - - // ensure principal id is matching - callCtx - .getDiagServices() - .check( - principalId == principalSecrets.getPrincipalId(), - "principal_id_mismatch", - "expectedId={} id={}", - principalId, - principalSecrets.getPrincipalId()); - - // delete these secrets - this.store.deletePrincipalSecrets(localSession.get(), clientId); - } - - /** {@inheritDoc} */ - @Override - public - PolarisStorageIntegration createStorageIntegration( - PolarisCallContext callCtx, - long catalogId, - long entityId, - PolarisStorageConfigurationInfo polarisStorageConfigurationInfo) { - return storageIntegrationProvider.getStorageIntegrationForConfig( - polarisStorageConfigurationInfo); - } - - /** {@inheritDoc} */ - @Override - public - PolarisStorageIntegration loadPolarisStorageIntegration( - PolarisCallContext callCtx, PolarisBaseEntity entity) { - PolarisStorageConfigurationInfo storageConfig = - PolarisMetaStoreManagerImpl.readStorageConfiguration(callCtx, entity); - return storageIntegrationProvider.getStorageIntegrationForConfig(storageConfig); - } - - @Override - public void rollback() { - EntityManager session = localSession.get(); - if (session != null) { - session.getTransaction().rollback(); - } - } -} diff --git a/extension/persistence/eclipselink-quarkus/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java b/extension/persistence/eclipselink-quarkus/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java deleted file mode 100644 index 9a4c84e41..000000000 --- a/extension/persistence/eclipselink-quarkus/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java +++ /dev/null @@ -1,450 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.extension.persistence.impl.eclipselink; - -import jakarta.persistence.EntityManager; -import jakarta.persistence.TypedQuery; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.Collectors; -import org.apache.polaris.core.PolarisDiagnostics; -import org.apache.polaris.core.entity.PolarisBaseEntity; -import org.apache.polaris.core.entity.PolarisEntitiesActiveKey; -import org.apache.polaris.core.entity.PolarisEntityActiveRecord; -import org.apache.polaris.core.entity.PolarisEntityCore; -import org.apache.polaris.core.entity.PolarisEntityId; -import org.apache.polaris.core.entity.PolarisEntityType; -import org.apache.polaris.core.entity.PolarisGrantRecord; -import org.apache.polaris.core.entity.PolarisPrincipalSecrets; -import org.apache.polaris.core.persistence.models.ModelEntity; -import org.apache.polaris.core.persistence.models.ModelEntityActive; -import org.apache.polaris.core.persistence.models.ModelEntityChangeTracking; -import org.apache.polaris.core.persistence.models.ModelEntityDropped; -import org.apache.polaris.core.persistence.models.ModelGrantRecord; -import org.apache.polaris.core.persistence.models.ModelPrincipalSecrets; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Implements an EclipseLink based metastore for Polaris which can be configured for any database - * with EclipseLink support - */ -public class PolarisEclipseLinkStore { - private static final Logger LOGGER = LoggerFactory.getLogger(PolarisEclipseLinkStore.class); - - // diagnostic services - private final PolarisDiagnostics diagnosticServices; - - // Used to track when the store is initialized - private final AtomicBoolean initialized = new AtomicBoolean(false); - - /** - * Constructor, allocate everything at once - * - * @param diagnostics diagnostic services - */ - public PolarisEclipseLinkStore(PolarisDiagnostics diagnostics) { - this.diagnosticServices = diagnostics; - } - - /** Initialize the store. This should be called before other methods. */ - public void initialize(EntityManager session) { - PolarisSequenceUtil.initialize(session); - initialized.set(true); - } - - long getNextSequence(EntityManager session) { - diagnosticServices.check(session != null, "session_is_null"); - checkInitialized(); - - return PolarisSequenceUtil.getNewId(session); - } - - void writeToEntities(EntityManager session, PolarisBaseEntity entity) { - diagnosticServices.check(session != null, "session_is_null"); - checkInitialized(); - - ModelEntity model = lookupEntity(session, entity.getCatalogId(), entity.getId()); - if (model != null) { - // Update if the same entity already exists - model.update(entity); - } else { - model = ModelEntity.fromEntity(entity); - } - - session.persist(model); - } - - void writeToEntitiesActive(EntityManager session, PolarisBaseEntity entity) { - diagnosticServices.check(session != null, "session_is_null"); - checkInitialized(); - - ModelEntityActive model = lookupEntityActive(session, new PolarisEntitiesActiveKey(entity)); - if (model == null) { - session.persist(ModelEntityActive.fromEntityActive(new PolarisEntityActiveRecord(entity))); - } - } - - void writeToEntitiesDropped(EntityManager session, PolarisBaseEntity entity) { - diagnosticServices.check(session != null, "session_is_null"); - checkInitialized(); - - ModelEntityDropped entityDropped = - lookupEntityDropped(session, entity.getCatalogId(), entity.getId()); - if (entityDropped == null) { - session.persist(ModelEntityDropped.fromEntity(entity)); - } - } - - void writeToEntitiesChangeTracking(EntityManager session, PolarisBaseEntity entity) { - diagnosticServices.check(session != null, "session_is_null"); - checkInitialized(); - - // Update the existing change tracking if a record with the same ids exists; otherwise, persist - // a new one - ModelEntityChangeTracking entityChangeTracking = - lookupEntityChangeTracking(session, entity.getCatalogId(), entity.getId()); - if (entityChangeTracking != null) { - entityChangeTracking.update(entity); - } else { - entityChangeTracking = new ModelEntityChangeTracking(entity); - } - - session.persist(entityChangeTracking); - } - - void writeToGrantRecords(EntityManager session, PolarisGrantRecord grantRec) { - diagnosticServices.check(session != null, "session_is_null"); - checkInitialized(); - - session.persist(ModelGrantRecord.fromGrantRecord(grantRec)); - } - - void deleteFromEntities(EntityManager session, long catalogId, long entityId) { - diagnosticServices.check(session != null, "session_is_null"); - checkInitialized(); - - ModelEntity model = lookupEntity(session, catalogId, entityId); - diagnosticServices.check(model != null, "entity_not_found"); - - session.remove(model); - } - - void deleteFromEntitiesActive(EntityManager session, PolarisEntitiesActiveKey key) { - diagnosticServices.check(session != null, "session_is_null"); - checkInitialized(); - - ModelEntityActive entity = lookupEntityActive(session, key); - diagnosticServices.check(entity != null, "active_entity_not_found"); - session.remove(entity); - } - - void deleteFromEntitiesDropped(EntityManager session, long catalogId, long entityId) { - diagnosticServices.check(session != null, "session_is_null"); - checkInitialized(); - - ModelEntityDropped entity = lookupEntityDropped(session, catalogId, entityId); - diagnosticServices.check(entity != null, "dropped_entity_not_found"); - - session.remove(entity); - } - - void deleteFromEntitiesChangeTracking(EntityManager session, PolarisEntityCore entity) { - diagnosticServices.check(session != null, "session_is_null"); - checkInitialized(); - - ModelEntityChangeTracking entityChangeTracking = - lookupEntityChangeTracking(session, entity.getCatalogId(), entity.getId()); - diagnosticServices.check(entityChangeTracking != null, "change_tracking_entity_not_found"); - - session.remove(entityChangeTracking); - } - - void deleteFromGrantRecords(EntityManager session, PolarisGrantRecord grantRec) { - diagnosticServices.check(session != null, "session_is_null"); - checkInitialized(); - - ModelGrantRecord lookupGrantRecord = - lookupGrantRecord( - session, - grantRec.getSecurableCatalogId(), - grantRec.getSecurableId(), - grantRec.getGranteeCatalogId(), - grantRec.getGranteeId(), - grantRec.getPrivilegeCode()); - - diagnosticServices.check(lookupGrantRecord != null, "grant_record_not_found"); - - session.remove(lookupGrantRecord); - } - - void deleteAllEntityGrantRecords(EntityManager session, PolarisEntityCore entity) { - diagnosticServices.check(session != null, "session_is_null"); - checkInitialized(); - - // Delete grant records from grantRecords tables - lookupAllGrantRecordsOnSecurable(session, entity.getCatalogId(), entity.getId()) - .forEach(session::remove); - - // Delete grantee records from grantRecords tables - lookupGrantRecordsOnGrantee(session, entity.getCatalogId(), entity.getId()) - .forEach(session::remove); - } - - void deleteAll(EntityManager session) { - diagnosticServices.check(session != null, "session_is_null"); - checkInitialized(); - - session.createQuery("DELETE from ModelEntity").executeUpdate(); - session.createQuery("DELETE from ModelEntityActive").executeUpdate(); - session.createQuery("DELETE from ModelEntityDropped").executeUpdate(); - session.createQuery("DELETE from ModelEntityChangeTracking").executeUpdate(); - session.createQuery("DELETE from ModelGrantRecord").executeUpdate(); - session.createQuery("DELETE from ModelPrincipalSecrets").executeUpdate(); - - LOGGER.debug("All entities deleted."); - } - - ModelEntity lookupEntity(EntityManager session, long catalogId, long entityId) { - diagnosticServices.check(session != null, "session_is_null"); - checkInitialized(); - - return session - .createQuery( - "SELECT m from ModelEntity m where m.catalogId=:catalogId and m.id=:id", - ModelEntity.class) - .setParameter("catalogId", catalogId) - .setParameter("id", entityId) - .getResultStream() - .findFirst() - .orElse(null); - } - - @SuppressWarnings("unchecked") - List lookupEntities(EntityManager session, List entityIds) { - diagnosticServices.check(session != null, "session_is_null"); - checkInitialized(); - - if (entityIds == null || entityIds.isEmpty()) return new ArrayList<>(); - - // TODO Support paging - String inClause = - entityIds.stream() - .map(entityId -> "(" + entityId.getCatalogId() + "," + entityId.getId() + ")") - .collect(Collectors.joining(",")); - - String hql = "SELECT * from ENTITIES m where (m.catalogId, m.id) in (" + inClause + ")"; - return (List) session.createNativeQuery(hql, ModelEntity.class).getResultList(); - } - - ModelEntityActive lookupEntityActive( - EntityManager session, PolarisEntitiesActiveKey entityActiveKey) { - diagnosticServices.check(session != null, "session_is_null"); - checkInitialized(); - - return session - .createQuery( - "SELECT m from ModelEntityActive m where m.catalogId=:catalogId and m.parentId=:parentId and m.typeCode=:typeCode and m.name=:name", - ModelEntityActive.class) - .setParameter("catalogId", entityActiveKey.getCatalogId()) - .setParameter("parentId", entityActiveKey.getParentId()) - .setParameter("typeCode", entityActiveKey.getTypeCode()) - .setParameter("name", entityActiveKey.getName()) - .getResultStream() - .findFirst() - .orElse(null); - } - - long countActiveChildEntities( - EntityManager session, long catalogId, long parentId, PolarisEntityType entityType) { - diagnosticServices.check(session != null, "session_is_null"); - checkInitialized(); - - String hql = - "SELECT COUNT(m) from ModelEntityActive m where m.catalogId=:catalogId and m.parentId=:parentId"; - if (entityType != null) { - hql += " and m.typeCode=:typeCode"; - } - - TypedQuery query = - session - .createQuery(hql, Long.class) - .setParameter("catalogId", catalogId) - .setParameter("parentId", parentId); - if (entityType != null) { - query.setParameter("typeCode", entityType.getCode()); - } - - return query.getSingleResult(); - } - - List lookupFullEntitiesActive( - EntityManager session, long catalogId, long parentId, PolarisEntityType entityType) { - diagnosticServices.check(session != null, "session_is_null"); - checkInitialized(); - - // Currently check against ENTITIES not joining with ENTITIES_ACTIVE - String hql = - "SELECT m from ModelEntity m where m.catalogId=:catalogId and m.parentId=:parentId and m.typeCode=:typeCode"; - - TypedQuery query = - session - .createQuery(hql, ModelEntity.class) - .setParameter("catalogId", catalogId) - .setParameter("parentId", parentId) - .setParameter("typeCode", entityType.getCode()); - - return query.getResultList(); - } - - ModelEntityDropped lookupEntityDropped(EntityManager session, long catalogId, long entityId) { - diagnosticServices.check(session != null, "session_is_null"); - checkInitialized(); - - return session - .createQuery( - "SELECT m from ModelEntityDropped m where m.catalogId=:catalogId and m.id=:id", - ModelEntityDropped.class) - .setParameter("catalogId", catalogId) - .setParameter("id", entityId) - .getResultStream() - .findFirst() - .orElse(null); - } - - ModelEntityChangeTracking lookupEntityChangeTracking( - EntityManager session, long catalogId, long entityId) { - diagnosticServices.check(session != null, "session_is_null"); - checkInitialized(); - - return session - .createQuery( - "SELECT m from ModelEntityChangeTracking m where m.catalogId=:catalogId and m.id=:id", - ModelEntityChangeTracking.class) - .setParameter("catalogId", catalogId) - .setParameter("id", entityId) - .getResultStream() - .findFirst() - .orElse(null); - } - - ModelGrantRecord lookupGrantRecord( - EntityManager session, - long securableCatalogId, - long securableId, - long granteeCatalogId, - long granteeId, - int privilegeCode) { - diagnosticServices.check(session != null, "session_is_null"); - checkInitialized(); - - return session - .createQuery( - "SELECT m from ModelGrantRecord m where m.securableCatalogId=:securableCatalogId " - + "and m.securableId=:securableId " - + "and m.granteeCatalogId=:granteeCatalogId " - + "and m.granteeId=:granteeId " - + "and m.privilegeCode=:privilegeCode", - ModelGrantRecord.class) - .setParameter("securableCatalogId", securableCatalogId) - .setParameter("securableId", securableId) - .setParameter("granteeCatalogId", granteeCatalogId) - .setParameter("granteeId", granteeId) - .setParameter("privilegeCode", privilegeCode) - .getResultStream() - .findFirst() - .orElse(null); - } - - List lookupAllGrantRecordsOnSecurable( - EntityManager session, long securableCatalogId, long securableId) { - diagnosticServices.check(session != null, "session_is_null"); - checkInitialized(); - - return session - .createQuery( - "SELECT m from ModelGrantRecord m " - + "where m.securableCatalogId=:securableCatalogId " - + "and m.securableId=:securableId", - ModelGrantRecord.class) - .setParameter("securableCatalogId", securableCatalogId) - .setParameter("securableId", securableId) - .getResultList(); - } - - List lookupGrantRecordsOnGrantee( - EntityManager session, long granteeCatalogId, long granteeId) { - diagnosticServices.check(session != null, "session_is_null"); - checkInitialized(); - - return session - .createQuery( - "SELECT m from ModelGrantRecord m " - + "where m.granteeCatalogId=:granteeCatalogId " - + "and m.granteeId=:granteeId", - ModelGrantRecord.class) - .setParameter("granteeCatalogId", granteeCatalogId) - .setParameter("granteeId", granteeId) - .getResultList(); - } - - ModelPrincipalSecrets lookupPrincipalSecrets(EntityManager session, String clientId) { - diagnosticServices.check(session != null, "session_is_null"); - checkInitialized(); - - return session - .createQuery( - "SELECT m from ModelPrincipalSecrets m where m.principalClientId=:clientId", - ModelPrincipalSecrets.class) - .setParameter("clientId", clientId) - .getResultStream() - .findFirst() - .orElse(null); - } - - void writePrincipalSecrets(EntityManager session, PolarisPrincipalSecrets principalSecrets) { - diagnosticServices.check(session != null, "session_is_null"); - checkInitialized(); - - ModelPrincipalSecrets modelPrincipalSecrets = - lookupPrincipalSecrets(session, principalSecrets.getPrincipalClientId()); - if (modelPrincipalSecrets != null) { - modelPrincipalSecrets.update(principalSecrets); - } else { - modelPrincipalSecrets = ModelPrincipalSecrets.fromPrincipalSecrets(principalSecrets); - } - - session.persist(modelPrincipalSecrets); - } - - void deletePrincipalSecrets(EntityManager session, String clientId) { - diagnosticServices.check(session != null, "session_is_null"); - checkInitialized(); - - ModelPrincipalSecrets modelPrincipalSecrets = lookupPrincipalSecrets(session, clientId); - diagnosticServices.check(modelPrincipalSecrets != null, "principal_secretes_not_found"); - - session.remove(modelPrincipalSecrets); - } - - private void checkInitialized() { - diagnosticServices.check(this.initialized.get(), "store_not_initialized"); - } -} diff --git a/extension/persistence/eclipselink-quarkus/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisSequenceUtil.java b/extension/persistence/eclipselink-quarkus/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisSequenceUtil.java deleted file mode 100644 index 609eec525..000000000 --- a/extension/persistence/eclipselink-quarkus/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisSequenceUtil.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.extension.persistence.impl.eclipselink; - -import jakarta.persistence.*; -import java.util.Optional; -import java.util.concurrent.atomic.AtomicBoolean; -import org.apache.polaris.core.persistence.models.ModelSequenceId; -import org.eclipse.persistence.internal.jpa.EntityManagerImpl; -import org.eclipse.persistence.platform.database.DatabasePlatform; -import org.eclipse.persistence.platform.database.PostgreSQLPlatform; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Used to generate sequence IDs for Polaris entities. If the legacy `POLARIS_SEQ` generator is - * available it will be used then cleaned up. In all other cases the `POLARIS_SEQUENCE` table is - * used directly. - */ -class PolarisSequenceUtil { - private static final Logger LOGGER = LoggerFactory.getLogger(PolarisSequenceUtil.class); - - private static final AtomicBoolean initialized = new AtomicBoolean(false); - - private PolarisSequenceUtil() {} - - /* If `initialize` was never called, throw an exception */ - private static void throwIfNotInitialized() { - if (!initialized.get()) { - throw new IllegalStateException("Sequence util has not been initialized"); - } - } - - /* Get the database platform associated with the `EntityManager` */ - private static DatabasePlatform getDatabasePlatform(EntityManager session) { - EntityManagerImpl entityManagerImpl = session.unwrap(EntityManagerImpl.class); - return entityManagerImpl.getDatabaseSession().getPlatform(); - } - - private static void removeSequence(EntityManager session) { - LOGGER.info("Renaming legacy sequence `POLARIS_SEQ` to `POLARIS_SEQ_UNUSED`"); - String renameSequenceQuery = "ALTER SEQUENCE POLARIS_SEQ RENAME TO POLARIS_SEQ_UNUSED"; - session.createNativeQuery(renameSequenceQuery).executeUpdate(); - } - - /** - * Prepare the `PolarisSequenceUtil` to generate IDs. This may run a failing query, so it should - * be called for the first time outside the context of a transaction. This method should be called - * before any other methods. TODO: after a sufficient this can eventually be removed or altered - */ - public static void initialize(EntityManager session) { - // Trigger cleanup of the POLARIS_SEQ if it is present - DatabasePlatform databasePlatform = getDatabasePlatform(session); - if (!initialized.get()) { - if (databasePlatform instanceof PostgreSQLPlatform) { - Optional result = Optional.empty(); - LOGGER.info("Checking if the sequence POLARIS_SEQ exists"); - String checkSequenceQuery = - "SELECT COUNT(*) FROM information_schema.sequences WHERE sequence_name IN " - + "('polaris_seq', 'POLARIS_SEQ')"; - int sequenceExists = - ((Number) session.createNativeQuery(checkSequenceQuery).getSingleResult()).intValue(); - - if (sequenceExists > 0) { - LOGGER.info("POLARIS_SEQ exists, calling NEXTVAL"); - long queryResult = - (long) session.createNativeQuery("SELECT NEXTVAL('POLARIS_SEQ')").getSingleResult(); - result = Optional.of(queryResult); - } else { - LOGGER.info("POLARIS_SEQ does not exist, skipping NEXTVAL"); - } - result.ifPresent( - r -> { - ModelSequenceId modelSequenceId = new ModelSequenceId(); - modelSequenceId.setId(r); - - // Persist the new ID: - session.persist(modelSequenceId); - session.flush(); - - // Clean the sequence: - removeSequence(session); - }); - } - initialized.set(true); - } - } - - /** - * Generates a new ID from `POLARIS_SEQUENCE` or `POLARIS_SEQ` depending on availability. - * `initialize` should be called before this method. - */ - public static Long getNewId(EntityManager session) { - throwIfNotInitialized(); - - ModelSequenceId modelSequenceId = new ModelSequenceId(); - - // Persist the new ID: - session.persist(modelSequenceId); - session.flush(); - - return modelSequenceId.getId(); - } -} diff --git a/extension/persistence/eclipselink-quarkus/src/main/resources/META-INF/persistence.xml b/extension/persistence/eclipselink-quarkus/src/main/resources/META-INF/persistence.xml deleted file mode 100644 index 913713796..000000000 --- a/extension/persistence/eclipselink-quarkus/src/main/resources/META-INF/persistence.xml +++ /dev/null @@ -1,47 +0,0 @@ - - - - - - org.eclipse.persistence.jpa.PersistenceProvider - org.apache.polaris.core.persistence.models.ModelEntity - org.apache.polaris.core.persistence.models.ModelEntityActive - org.apache.polaris.core.persistence.models.ModelEntityChangeTracking - org.apache.polaris.core.persistence.models.ModelEntityDropped - org.apache.polaris.core.persistence.models.ModelGrantRecord - org.apache.polaris.core.persistence.models.ModelPrincipalSecrets - org.apache.polaris.core.persistence.models.ModelSequenceId - NONE - - - - - - - - - - - \ No newline at end of file diff --git a/extension/persistence/eclipselink-quarkus/src/test/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreManagerTest.java b/extension/persistence/eclipselink-quarkus/src/test/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreManagerTest.java deleted file mode 100644 index 73c90347c..000000000 --- a/extension/persistence/eclipselink-quarkus/src/test/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreManagerTest.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.extension.persistence.impl.eclipselink; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.time.ZoneId; -import java.util.stream.Stream; -import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.PolarisConfigurationStore; -import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; -import org.apache.polaris.core.PolarisDiagnostics; -import org.apache.polaris.core.persistence.BasePolarisMetaStoreManagerTest; -import org.apache.polaris.core.persistence.PolarisMetaStoreManagerImpl; -import org.apache.polaris.core.persistence.PolarisTestMetaStoreManager; -import org.junit.jupiter.api.extension.ExtensionContext; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.ArgumentsProvider; -import org.junit.jupiter.params.provider.ArgumentsSource; -import org.mockito.Mockito; - -/** - * Integration test for EclipseLink based metastore implementation - * - * @author aixu - */ -public class PolarisEclipseLinkMetaStoreManagerTest extends BasePolarisMetaStoreManagerTest { - - @Override - protected PolarisTestMetaStoreManager createPolarisTestMetaStoreManager() { - PolarisDiagnostics diagServices = new PolarisDefaultDiagServiceImpl(); - PolarisEclipseLinkStore store = new PolarisEclipseLinkStore(diagServices); - PolarisEclipseLinkMetaStoreSessionImpl session = - new PolarisEclipseLinkMetaStoreSessionImpl( - store, Mockito.mock(), () -> "realm", null, "polaris"); - return new PolarisTestMetaStoreManager( - new PolarisMetaStoreManagerImpl(), - new PolarisCallContext( - session, - diagServices, - new PolarisConfigurationStore() {}, - timeSource.withZone(ZoneId.systemDefault()))); - } - - @ParameterizedTest() - @ArgumentsSource(CreateStoreSessionArgs.class) - void testCreateStoreSession(String confFile, boolean success) { - // Clear cache to prevent reuse EntityManagerFactory - PolarisEclipseLinkMetaStoreSessionImpl.clearEntityManagerFactories(); - - PolarisDiagnostics diagServices = new PolarisDefaultDiagServiceImpl(); - PolarisEclipseLinkStore store = new PolarisEclipseLinkStore(diagServices); - try { - var session = - new PolarisEclipseLinkMetaStoreSessionImpl( - store, Mockito.mock(), () -> "realm", confFile, "polaris"); - assertNotNull(session); - assertTrue(success); - } catch (Exception e) { - assertFalse(success); - } - } - - private static class CreateStoreSessionArgs implements ArgumentsProvider { - @Override - public Stream provideArguments(ExtensionContext extensionContext) { - return Stream.of( - Arguments.of("META-INF/persistence.xml", true), - Arguments.of("/dummy_path/conf.jar!/persistence.xml", false)); - } - } -} diff --git a/extension/persistence/eclipselink/build.gradle.kts b/extension/persistence/eclipselink/build.gradle.kts index 5c8333213..03bacd58e 100644 --- a/extension/persistence/eclipselink/build.gradle.kts +++ b/extension/persistence/eclipselink/build.gradle.kts @@ -23,6 +23,7 @@ fun isValidDep(dep: String): Boolean { } plugins { + alias(libs.plugins.quarkus) id("polaris-server") `java-library` } @@ -30,8 +31,8 @@ plugins { dependencies { implementation(project(":polaris-core")) implementation(libs.eclipselink) - implementation(platform(libs.dropwizard.bom)) - implementation("io.dropwizard:dropwizard-jackson") + implementation(platform(libs.quarkus.bom)) + val eclipseLinkDeps: String? = project.findProperty("eclipseLinkDeps") as String? eclipseLinkDeps?.let { val dependenciesList = it.split(",") @@ -46,6 +47,10 @@ dependencies { } compileOnly(libs.jetbrains.annotations) + compileOnly(libs.jakarta.enterprise.cdi.api) + compileOnly(libs.jakarta.inject.api) + compileOnly("io.quarkus:quarkus-arc") + compileOnly("org.eclipse.microprofile.config:microprofile-config-api") testImplementation(libs.h2) testImplementation(testFixtures(project(":polaris-core"))) diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java index 0415c0d5f..b9206050f 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java @@ -18,13 +18,15 @@ */ package org.apache.polaris.extension.persistence.impl.eclipselink; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; +import io.quarkus.arc.lookup.LookupIfProperty; +import jakarta.enterprise.context.ApplicationScoped; import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.config.RuntimeCandidate; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.LocalPolarisMetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.PolarisMetaStoreSession; +import org.eclipse.microprofile.config.inject.ConfigProperty; import org.jetbrains.annotations.NotNull; /** @@ -32,14 +34,17 @@ * using an EclipseLink based meta store to store and retrieve all Polaris metadata. It can be * configured through persistence.xml to use supported RDBMS as the meta store. */ -@JsonTypeName("eclipse-link") +@ApplicationScoped +@RuntimeCandidate +@LookupIfProperty(name = "polaris.persistence.metastore-manager.type", stringValue = "eclipse-link") public class EclipseLinkPolarisMetaStoreManagerFactory extends LocalPolarisMetaStoreManagerFactory { - @JsonProperty("conf-file") - private String confFile; - @JsonProperty("persistence-unit") - private String persistenceUnitName; + @ConfigProperty(name = "polaris.eclipselink.conf-file") + String confFile; + + @ConfigProperty(name = "polaris.eclipselink.persistence-unit", defaultValue = "polaris") + String persistenceUnitName; @Override protected PolarisEclipseLinkStore createBackingStore(@NotNull PolarisDiagnostics diagnostics) { diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java index 534d32a0b..1091fcd97 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java @@ -21,7 +21,6 @@ import static org.eclipse.persistence.config.PersistenceUnitProperties.ECLIPSELINK_PERSISTENCE_XML; import static org.eclipse.persistence.config.PersistenceUnitProperties.JDBC_URL; -import com.google.common.base.Predicates; import jakarta.persistence.EntityManager; import jakarta.persistence.EntityManagerFactory; import jakarta.persistence.EntityTransaction; @@ -519,7 +518,7 @@ public List lookupEntityActiveBatch( long catalogId, long parentId, @NotNull PolarisEntityType entityType) { - return listActiveEntities(callCtx, catalogId, parentId, entityType, Predicates.alwaysTrue()); + return listActiveEntities(callCtx, catalogId, parentId, entityType, x -> true); } @Override diff --git a/gradle.properties b/gradle.properties index e366b713e..9b9f97026 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,3 @@ -# Gradle properties - # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index dd3edd1bf..1ce99f648 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -18,7 +18,6 @@ # [versions] -dropwizard = "4.0.8" hadoop = "3.4.0" iceberg = "1.6.1" quarkus = "3.16.1" @@ -42,7 +41,6 @@ bouncycastle-bcprov = { module = "org.bouncycastle:bcprov-jdk18on", version = "1 caffeine = { module = "com.github.ben-manes.caffeine:caffeine", version = "3.1.8" } commons-codec1 = { module = "commons-codec:commons-codec", version = "1.17.1" } commons-lang3 = { module = "org.apache.commons:commons-lang3", version = "3.17.0" } -dropwizard-bom = { module = "io.dropwizard:dropwizard-bom", version.ref = "dropwizard" } eclipselink = { module = "org.eclipse.persistence:eclipselink", version = "4.0.4" } errorprone = { module = "com.google.errorprone:error_prone_core", version = "2.29.2" } google-cloud-storage-bom = { module = "com.google.cloud:google-cloud-storage-bom", version = "2.42.0" } @@ -58,11 +56,12 @@ jakarta-annotation-api = { module = "jakarta.annotation:jakarta.annotation-api", jakarta-enterprise-cdi-api = { module = "jakarta.enterprise:jakarta.enterprise.cdi-api", version = "4.1.0" } jakarta-inject-api = { module = "jakarta.inject:jakarta.inject-api", version = "2.0.1" } jakarta-persistence-api = { module = "jakarta.persistence:jakarta.persistence-api", version = "3.1.0" } +jakarta-servlet-api = { module = "jakarta.servlet:jakarta.servlet-api", version = "6.1.0" } jakarta-validation-api = { module = "jakarta.validation:jakarta.validation-api", version = "3.0.2" } jakarta-ws-rs-api = { module = "jakarta.ws.rs:jakarta.ws.rs-api", version = "3.1.0" } javax-annotation-api = { module = "javax.annotation:javax.annotation-api", version = "1.3.2" } -javax-servlet-api = { module = "javax.servlet:javax.servlet-api", version = "4.0.1" } javax-inject = { module = "javax.inject:javax.inject", version = "1" } +javax-servlet-api = { module = "javax.servlet:javax.servlet-api", version = "4.0.1" } jetbrains-annotations = { module = "org.jetbrains:annotations", version = "24.1.0" } junit-bom = { module = "org.junit:junit-bom", version = "5.10.3" } junit-jupiter-api = { module = "org.junit.jupiter:junit-jupiter-api", version = "5.10.3" } diff --git a/gradle/projects.main.properties b/gradle/projects.main.properties index 28170949e..64ac2e00c 100644 --- a/gradle/projects.main.properties +++ b/gradle/projects.main.properties @@ -20,8 +20,5 @@ polaris-core=polaris-core polaris-service=polaris-service -polaris-service-quarkus=polaris-service-quarkus polaris-eclipselink=extension/persistence/eclipselink -# Workaround to https://github.com/quarkusio/quarkus/issues/44367 for full build (probably a conflict between polaris-service-quarkus and polaris-eclipselink-quarkus) -#polaris-eclipselink-quarkus=extension/persistence/eclipselink-quarkus aggregated-license-report=aggregated-license-report diff --git a/polaris-core/build.gradle.kts b/polaris-core/build.gradle.kts index e48dedd07..736f39312 100644 --- a/polaris-core/build.gradle.kts +++ b/polaris-core/build.gradle.kts @@ -34,13 +34,6 @@ dependencies { constraints { implementation("io.airlift:aircompressor:0.27") { because("Vulnerability detected in 0.25") } } - // TODO - this is only here for the Discoverable interface, it should be remove when - // polaris-service will use Quarkus - // For now, I keep the Discoverable interface for now, to have both polaris-service (with - // Dropwizard) and polaris-service-quarkus (with Quarkus) - // We should use a different mechanism to discover the plugin implementations - implementation(platform(libs.dropwizard.bom)) - implementation("io.dropwizard:dropwizard-jackson") implementation(platform(libs.jackson.bom)) implementation("com.fasterxml.jackson.core:jackson-annotations") @@ -53,6 +46,7 @@ dependencies { implementation(libs.slf4j.api) compileOnly(libs.jetbrains.annotations) compileOnly(libs.spotbugs.annotations) + compileOnly(libs.jakarta.inject.api) // FIXME remove when RuntimeCandidate is removed constraints { implementation("org.xerial.snappy:snappy-java:1.1.10.4") { diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/config/RuntimeCandidate.java b/polaris-core/src/main/java/org/apache/polaris/core/config/RuntimeCandidate.java similarity index 95% rename from polaris-service-quarkus/src/main/java/org/apache/polaris/service/config/RuntimeCandidate.java rename to polaris-core/src/main/java/org/apache/polaris/core/config/RuntimeCandidate.java index b60fe37e0..c7513b443 100644 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/config/RuntimeCandidate.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/config/RuntimeCandidate.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.config; +package org.apache.polaris.core.config; import static java.lang.annotation.ElementType.FIELD; import static java.lang.annotation.ElementType.METHOD; @@ -34,4 +34,5 @@ @Retention(RUNTIME) @Target({TYPE, METHOD, FIELD, PARAMETER}) @Documented +// FIXME remove or replace public @interface RuntimeCandidate {} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java index a4ad38140..0f2e24a73 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java @@ -18,8 +18,6 @@ */ package org.apache.polaris.core.persistence; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import io.dropwizard.jackson.Discoverable; import java.util.List; import java.util.Map; import java.util.function.Supplier; @@ -33,10 +31,7 @@ * Configuration interface for configuring the {@link PolarisMetaStoreManager} via Dropwizard * configuration */ -// TODO for now, I keep the Discoverable interface in order to have both polaris-service (with -// Dropwizard) and polaris-service-quarkus (with Quarkus) -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") -public interface MetaStoreManagerFactory extends Discoverable { +public interface MetaStoreManagerFactory { PolarisMetaStoreManager getOrCreateMetaStoreManager(RealmContext realmContext); diff --git a/polaris-service-quarkus/build.gradle.kts b/polaris-service-quarkus/build.gradle.kts deleted file mode 100644 index cbcadacbf..000000000 --- a/polaris-service-quarkus/build.gradle.kts +++ /dev/null @@ -1,264 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import org.openapitools.generator.gradle.plugin.tasks.GenerateTask - -plugins { - alias(libs.plugins.quarkus) - alias(libs.plugins.openapi.generator) - id("polaris-server") - id("application") -} - -dependencies { - implementation(project(":polaris-core")) - - implementation(platform(libs.iceberg.bom)) - implementation("org.apache.iceberg:iceberg-api") - implementation("org.apache.iceberg:iceberg-core") - implementation("org.apache.iceberg:iceberg-aws") - - implementation(platform(libs.quarkus.bom)) - implementation("io.quarkus:quarkus-logging-json") - implementation("io.quarkus:quarkus-rest") - implementation("io.quarkus:quarkus-rest-jackson") - implementation("io.quarkus:quarkus-hibernate-validator") - implementation("io.quarkus:quarkus-smallrye-health") - implementation("io.quarkus:quarkus-micrometer") - implementation("io.quarkus:quarkus-opentelemetry") - implementation("io.quarkus:quarkus-container-image-docker") - - implementation("org.apache.commons:commons-lang3:3.17.0") - - compileOnly(libs.jakarta.enterprise.cdi.api) - compileOnly(libs.jakarta.inject.api) - compileOnly(libs.jakarta.validation.api) - compileOnly(libs.jakarta.ws.rs.api) - - // TODO this will removed as soon as dropwizard will be removed - compileOnly(platform(libs.dropwizard.bom)) - compileOnly("io.dropwizard:dropwizard-core") - testCompileOnly(platform(libs.dropwizard.bom)) - testCompileOnly("io.dropwizard:dropwizard-core") - - implementation(platform(libs.jackson.bom)) - implementation("com.fasterxml.jackson.core:jackson-annotations") - implementation("com.fasterxml.jackson.core:jackson-core") - implementation("com.fasterxml.jackson.core:jackson-databind") - - implementation(libs.caffeine) - implementation(libs.guava) - implementation(libs.slf4j.api) - - implementation(libs.hadoop.client.api) - implementation(libs.hadoop.client.runtime) - - implementation(libs.auth0.jwt) - - implementation(libs.bouncycastle.bcprov) - - implementation(platform(libs.google.cloud.storage.bom)) - implementation("com.google.cloud:google-cloud-storage") - implementation(platform(libs.awssdk.bom)) - implementation("software.amazon.awssdk:sts") - implementation("software.amazon.awssdk:iam-policy-builder") - implementation("software.amazon.awssdk:s3") - implementation(platform(libs.azuresdk.bom)) - implementation("com.azure:azure-core") - - implementation("io.quarkus:quarkus-micrometer-registry-prometheus") - - compileOnly(libs.swagger.annotations) - - testImplementation("org.apache.iceberg:iceberg-api:${libs.versions.iceberg.get()}:tests") - testImplementation("org.apache.iceberg:iceberg-core:${libs.versions.iceberg.get()}:tests") - - testImplementation("org.apache.iceberg:iceberg-spark-3.5_2.12") - testImplementation("org.apache.iceberg:iceberg-spark-extensions-3.5_2.12") - testImplementation("org.apache.spark:spark-sql_2.12:3.5.1") { - // exclude log4j dependencies - exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") - exclude("org.apache.logging.log4j", "log4j-api") - exclude("org.apache.logging.log4j", "log4j-1.2-api") - exclude("org.slf4j", "jul-to-slf4j") - } - - testImplementation("software.amazon.awssdk:glue") - testImplementation("software.amazon.awssdk:kms") - testImplementation("software.amazon.awssdk:dynamodb") - - testImplementation(platform(libs.junit.bom)) - testImplementation(libs.bundles.junit.testing) - testImplementation(libs.assertj.core) - testImplementation(libs.mockito.core) - - testImplementation(platform(libs.quarkus.bom)) - testImplementation("io.quarkus:quarkus-junit5") - testImplementation("io.quarkus:quarkus-junit5-mockito") - testImplementation("io.quarkus:quarkus-rest-client") - testImplementation("io.quarkus:quarkus-rest-client-jackson") - testImplementation("io.rest-assured:rest-assured") - - testImplementation(platform(libs.testcontainers.bom)) - testImplementation("org.testcontainers:testcontainers") - testImplementation(libs.s3mock.testcontainers) - - // required for PolarisSparkIntegrationTest - testImplementation(enforcedPlatform("org.scala-lang:scala-library:2.12.18")) - testImplementation(enforcedPlatform("org.scala-lang:scala-reflect:2.12.18")) - testImplementation(libs.javax.servlet.api) - testImplementation( - enforcedPlatform("org.antlr:antlr4-runtime:4.9.3") - ) // cannot be higher than 4.9.3 -} - -openApiGenerate { - inputSpec = "$rootDir/spec/rest-catalog-open-api.yaml" - generatorName = "jaxrs-resteasy" - outputDir = "$projectDir/build/generated" - apiPackage = "org.apache.polaris.service.catalog.api" - ignoreFileOverride = "$rootDir/.openapi-generator-ignore" - removeOperationIdPrefix = true - templateDir = "$rootDir/server-templates" - globalProperties.put("apis", "") - globalProperties.put("models", "false") - globalProperties.put("apiDocs", "false") - globalProperties.put("modelTests", "false") - configOptions.put("resourceName", "catalog") - configOptions.put("useTags", "true") - configOptions.put("useBeanValidation", "false") - configOptions.put("sourceFolder", "src/main/java") - configOptions.put("useJakartaEe", "true") - openapiNormalizer.put("REFACTOR_ALLOF_WITH_PROPERTIES_ONLY", "true") - additionalProperties.put("apiNamePrefix", "IcebergRest") - additionalProperties.put("apiNameSuffix", "") - additionalProperties.put("metricsPrefix", "polaris") - serverVariables.put("basePath", "api/catalog") - importMappings = - mapOf( - "CatalogConfig" to "org.apache.iceberg.rest.responses.ConfigResponse", - "CommitTableResponse" to "org.apache.iceberg.rest.responses.LoadTableResponse", - "CreateNamespaceRequest" to "org.apache.iceberg.rest.requests.CreateNamespaceRequest", - "CreateNamespaceResponse" to "org.apache.iceberg.rest.responses.CreateNamespaceResponse", - "CreateTableRequest" to "org.apache.iceberg.rest.requests.CreateTableRequest", - "ErrorModel" to "org.apache.iceberg.rest.responses.ErrorResponse", - "GetNamespaceResponse" to "org.apache.iceberg.rest.responses.GetNamespaceResponse", - "ListNamespacesResponse" to "org.apache.iceberg.rest.responses.ListNamespacesResponse", - "ListTablesResponse" to "org.apache.iceberg.rest.responses.ListTablesResponse", - "LoadTableResult" to "org.apache.iceberg.rest.responses.LoadTableResponse", - "LoadViewResult" to "org.apache.iceberg.rest.responses.LoadTableResponse", - "OAuthTokenResponse" to "org.apache.iceberg.rest.responses.OAuthTokenResponse", - "OAuthErrorResponse" to "org.apache.iceberg.rest.responses.OAuthErrorResponse", - "RenameTableRequest" to "org.apache.iceberg.rest.requests.RenameTableRequest", - "ReportMetricsRequest" to "org.apache.iceberg.rest.requests.ReportMetricsRequest", - "UpdateNamespacePropertiesRequest" to - "org.apache.iceberg.rest.requests.UpdateNamespacePropertiesRequest", - "UpdateNamespacePropertiesResponse" to - "org.apache.iceberg.rest.responses.UpdateNamespacePropertiesResponse", - "CommitTransactionRequest" to "org.apache.iceberg.rest.requests.CommitTransactionRequest", - "CreateViewRequest" to "org.apache.iceberg.rest.requests.CreateViewRequest", - "RegisterTableRequest" to "org.apache.iceberg.rest.requests.RegisterTableRequest", - "IcebergErrorResponse" to "org.apache.iceberg.rest.responses.ErrorResponse", - "OAuthError" to "org.apache.iceberg.rest.responses.ErrorResponse", - - // Custom types defined below - "CommitViewRequest" to "org.apache.polaris.service.types.CommitViewRequest", - "TokenType" to "org.apache.polaris.service.types.TokenType", - "CommitTableRequest" to "org.apache.polaris.service.types.CommitTableRequest", - "NotificationRequest" to "org.apache.polaris.service.types.NotificationRequest", - "TableUpdateNotification" to "org.apache.polaris.service.types.TableUpdateNotification", - "NotificationType" to "org.apache.polaris.service.types.NotificationType" - ) -} - -val generatePolarisService by - tasks.registering(GenerateTask::class) { - inputSpec = "$rootDir/spec/polaris-management-service.yml" - generatorName = "jaxrs-resteasy" - outputDir = "$projectDir/build/generated" - apiPackage = "org.apache.polaris.service.admin.api" - modelPackage = "org.apache.polaris.core.admin.model" - ignoreFileOverride = "$rootDir/.openapi-generator-ignore" - removeOperationIdPrefix = true - templateDir = "$rootDir/server-templates" - globalProperties.put("apis", "") - globalProperties.put("models", "false") - globalProperties.put("apiDocs", "false") - globalProperties.put("modelTests", "false") - configOptions.put("useBeanValidation", "true") - configOptions.put("sourceFolder", "src/main/java") - configOptions.put("useJakartaEe", "true") - configOptions.put("generateBuilders", "true") - configOptions.put("generateConstructorWithAllArgs", "true") - additionalProperties.put("apiNamePrefix", "Polaris") - additionalProperties.put("apiNameSuffix", "Api") - additionalProperties.put("metricsPrefix", "polaris") - serverVariables.put("basePath", "api/v1") - } - -listOf("sourcesJar", "compileJava").forEach { task -> - tasks.named(task) { dependsOn("openApiGenerate", generatePolarisService) } -} - -sourceSets { - main { java { srcDir(project.layout.buildDirectory.dir("generated/src/main/java")) } } -} - -tasks.withType(Test::class.java).configureEach { - systemProperty("java.util.logging.manager", "org.jboss.logmanager.LogManager") - addSparkJvmOptions() -} - -tasks.named("test").configure { - if (System.getenv("AWS_REGION") == null) { - environment("AWS_REGION", "us-west-2") - } - jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") - useJUnitPlatform() - maxParallelForks = 4 -} - -/** - * Adds the JPMS options required for Spark to run on Java 17, taken from the - * `DEFAULT_MODULE_OPTIONS` constant in `org.apache.spark.launcher.JavaModuleOptions`. - */ -fun JavaForkOptions.addSparkJvmOptions() { - jvmArgs = - (jvmArgs ?: emptyList()) + - listOf( - // Spark 3.3+ - "-XX:+IgnoreUnrecognizedVMOptions", - "--add-opens=java.base/java.lang=ALL-UNNAMED", - "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED", - "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED", - "--add-opens=java.base/java.io=ALL-UNNAMED", - "--add-opens=java.base/java.net=ALL-UNNAMED", - "--add-opens=java.base/java.nio=ALL-UNNAMED", - "--add-opens=java.base/java.util=ALL-UNNAMED", - "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED", - "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED", - "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", - "--add-opens=java.base/sun.nio.cs=ALL-UNNAMED", - "--add-opens=java.base/sun.security.action=ALL-UNNAMED", - "--add-opens=java.base/sun.util.calendar=ALL-UNNAMED", - "--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED", - // Spark 3.4+ - "-Djdk.reflect.useDirectMethodHandle=false" - ) -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java deleted file mode 100644 index 65d903930..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java +++ /dev/null @@ -1,1719 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.admin; - -import jakarta.annotation.Nonnull; -import jakarta.annotation.Nullable; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.function.Function; -import org.apache.commons.lang3.StringUtils; -import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.exceptions.AlreadyExistsException; -import org.apache.iceberg.exceptions.BadRequestException; -import org.apache.iceberg.exceptions.CommitFailedException; -import org.apache.iceberg.exceptions.NoSuchNamespaceException; -import org.apache.iceberg.exceptions.NoSuchTableException; -import org.apache.iceberg.exceptions.NoSuchViewException; -import org.apache.iceberg.exceptions.NotFoundException; -import org.apache.iceberg.exceptions.ValidationException; -import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.PolarisConfiguration; -import org.apache.polaris.core.admin.model.CatalogGrant; -import org.apache.polaris.core.admin.model.CatalogPrivilege; -import org.apache.polaris.core.admin.model.GrantResource; -import org.apache.polaris.core.admin.model.NamespaceGrant; -import org.apache.polaris.core.admin.model.NamespacePrivilege; -import org.apache.polaris.core.admin.model.PrincipalWithCredentials; -import org.apache.polaris.core.admin.model.PrincipalWithCredentialsCredentials; -import org.apache.polaris.core.admin.model.TableGrant; -import org.apache.polaris.core.admin.model.TablePrivilege; -import org.apache.polaris.core.admin.model.UpdateCatalogRequest; -import org.apache.polaris.core.admin.model.UpdateCatalogRoleRequest; -import org.apache.polaris.core.admin.model.UpdatePrincipalRequest; -import org.apache.polaris.core.admin.model.UpdatePrincipalRoleRequest; -import org.apache.polaris.core.admin.model.ViewGrant; -import org.apache.polaris.core.admin.model.ViewPrivilege; -import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; -import org.apache.polaris.core.auth.PolarisAuthorizableOperation; -import org.apache.polaris.core.auth.PolarisAuthorizer; -import org.apache.polaris.core.auth.PolarisGrantManager.LoadGrantsResult; -import org.apache.polaris.core.catalog.PolarisCatalogHelpers; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.entity.CatalogEntity; -import org.apache.polaris.core.entity.CatalogRoleEntity; -import org.apache.polaris.core.entity.NamespaceEntity; -import org.apache.polaris.core.entity.PolarisBaseEntity; -import org.apache.polaris.core.entity.PolarisEntity; -import org.apache.polaris.core.entity.PolarisEntitySubType; -import org.apache.polaris.core.entity.PolarisEntityType; -import org.apache.polaris.core.entity.PolarisGrantRecord; -import org.apache.polaris.core.entity.PolarisPrincipalSecrets; -import org.apache.polaris.core.entity.PolarisPrivilege; -import org.apache.polaris.core.entity.PrincipalEntity; -import org.apache.polaris.core.entity.PrincipalRoleEntity; -import org.apache.polaris.core.entity.TableLikeEntity; -import org.apache.polaris.core.persistence.PolarisEntityManager; -import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; -import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest; -import org.apache.polaris.core.persistence.resolver.ResolverPath; -import org.apache.polaris.core.persistence.resolver.ResolverStatus; -import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; -import org.apache.polaris.core.storage.StorageLocation; -import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo; -import org.apache.polaris.core.storage.azure.AzureStorageConfigurationInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Just as an Iceberg Catalog represents the logical model of Iceberg business logic to manage - * Namespaces, Tables and Views, abstracted away from Iceberg REST objects, this class represents - * the logical model for managing realm-level Catalogs, Principals, Roles, and Grants. - * - *

Different API implementors could expose different REST, gRPC, etc., interfaces that delegate - * to this logical model without being tightly coupled to a single frontend protocol, and can - * provide different implementations of PolarisEntityManager to abstract away the implementation of - * the persistence layer. - */ -public class PolarisAdminService { - private static final Logger LOGGER = LoggerFactory.getLogger(PolarisAdminService.class); - - private final CallContext callContext; - private final PolarisEntityManager entityManager; - private final AuthenticatedPolarisPrincipal authenticatedPrincipal; - private final PolarisAuthorizer authorizer; - private final PolarisMetaStoreManager metaStoreManager; - - // Initialized in the authorize methods. - private PolarisResolutionManifest resolutionManifest = null; - - public PolarisAdminService( - CallContext callContext, - PolarisEntityManager entityManager, - PolarisMetaStoreManager metaStoreManager, - AuthenticatedPolarisPrincipal authenticatedPrincipal, - PolarisAuthorizer authorizer) { - this.callContext = callContext; - this.entityManager = entityManager; - this.metaStoreManager = metaStoreManager; - this.authenticatedPrincipal = authenticatedPrincipal; - this.authorizer = authorizer; - } - - private PolarisCallContext getCurrentPolarisContext() { - return callContext.getPolarisCallContext(); - } - - private Optional findCatalogByName(String name) { - return Optional.ofNullable(resolutionManifest.getResolvedReferenceCatalogEntity()) - .map(path -> CatalogEntity.of(path.getRawLeafEntity())); - } - - private Optional findPrincipalByName(String name) { - return Optional.ofNullable( - resolutionManifest.getResolvedTopLevelEntity(name, PolarisEntityType.PRINCIPAL)) - .map(path -> PrincipalEntity.of(path.getRawLeafEntity())); - } - - private Optional findPrincipalRoleByName(String name) { - return Optional.ofNullable( - resolutionManifest.getResolvedTopLevelEntity(name, PolarisEntityType.PRINCIPAL_ROLE)) - .map(path -> PrincipalRoleEntity.of(path.getRawLeafEntity())); - } - - private Optional findCatalogRoleByName(String catalogName, String name) { - return Optional.ofNullable(resolutionManifest.getResolvedPath(name)) - .map(path -> CatalogRoleEntity.of(path.getRawLeafEntity())); - } - - private void authorizeBasicRootOperationOrThrow(PolarisAuthorizableOperation op) { - resolutionManifest = - entityManager.prepareResolutionManifest( - callContext, authenticatedPrincipal, null /* referenceCatalogName */); - resolutionManifest.resolveAll(); - PolarisResolvedPathWrapper rootContainerWrapper = - resolutionManifest.getResolvedRootContainerEntityAsPath(); - authorizer.authorizeOrThrow( - authenticatedPrincipal, - resolutionManifest.getAllActivatedPrincipalRoleEntities(), - op, - rootContainerWrapper, - null /* secondary */); - } - - private void authorizeBasicTopLevelEntityOperationOrThrow( - PolarisAuthorizableOperation op, String topLevelEntityName, PolarisEntityType entityType) { - String referenceCatalogName = - entityType == PolarisEntityType.CATALOG ? topLevelEntityName : null; - authorizeBasicTopLevelEntityOperationOrThrow( - op, topLevelEntityName, entityType, referenceCatalogName); - } - - private void authorizeBasicTopLevelEntityOperationOrThrow( - PolarisAuthorizableOperation op, - String topLevelEntityName, - PolarisEntityType entityType, - @Nullable String referenceCatalogName) { - resolutionManifest = - entityManager.prepareResolutionManifest( - callContext, authenticatedPrincipal, referenceCatalogName); - resolutionManifest.addTopLevelName(topLevelEntityName, entityType, false /* isOptional */); - ResolverStatus status = resolutionManifest.resolveAll(); - if (status.getStatus() == ResolverStatus.StatusEnum.ENTITY_COULD_NOT_BE_RESOLVED) { - throw new NotFoundException( - "TopLevelEntity of type %s does not exist: %s", entityType, topLevelEntityName); - } - PolarisResolvedPathWrapper topLevelEntityWrapper = - resolutionManifest.getResolvedTopLevelEntity(topLevelEntityName, entityType); - - // TODO: If we do add more "self" privilege operations for PRINCIPAL targets this should - // be extracted into an EnumSet and/or pushed down into PolarisAuthorizer. - if (topLevelEntityWrapper.getResolvedLeafEntity().getEntity().getId() - == authenticatedPrincipal.getPrincipalEntity().getId() - && (op.equals(PolarisAuthorizableOperation.ROTATE_CREDENTIALS) - || op.equals(PolarisAuthorizableOperation.RESET_CREDENTIALS))) { - LOGGER - .atDebug() - .addKeyValue("principalName", topLevelEntityName) - .log("Allowing rotate own credentials"); - return; - } - authorizer.authorizeOrThrow( - authenticatedPrincipal, - resolutionManifest.getAllActivatedCatalogRoleAndPrincipalRoles(), - op, - topLevelEntityWrapper, - null /* secondary */); - } - - private void authorizeBasicCatalogRoleOperationOrThrow( - PolarisAuthorizableOperation op, String catalogName, String catalogRoleName) { - resolutionManifest = - entityManager.prepareResolutionManifest(callContext, authenticatedPrincipal, catalogName); - resolutionManifest.addPath( - new ResolverPath(List.of(catalogRoleName), PolarisEntityType.CATALOG_ROLE), - catalogRoleName); - resolutionManifest.resolveAll(); - PolarisResolvedPathWrapper target = resolutionManifest.getResolvedPath(catalogRoleName, true); - if (target == null) { - throw new NotFoundException("CatalogRole does not exist: %s", catalogRoleName); - } - authorizer.authorizeOrThrow( - authenticatedPrincipal, - resolutionManifest.getAllActivatedCatalogRoleAndPrincipalRoles(), - op, - target, - null /* secondary */); - } - - private void authorizeGrantOnRootContainerToPrincipalRoleOperationOrThrow( - PolarisAuthorizableOperation op, String principalRoleName) { - resolutionManifest = - entityManager.prepareResolutionManifest(callContext, authenticatedPrincipal, null); - resolutionManifest.addTopLevelName( - principalRoleName, PolarisEntityType.PRINCIPAL_ROLE, false /* isOptional */); - ResolverStatus status = resolutionManifest.resolveAll(); - - if (status.getStatus() == ResolverStatus.StatusEnum.ENTITY_COULD_NOT_BE_RESOLVED) { - throw new NotFoundException( - "Entity %s not found when trying to grant on root to %s", - status.getFailedToResolvedEntityName(), principalRoleName); - } - - // TODO: Merge this method into authorizeGrantOnTopLevelEntityToPrincipalRoleOperationOrThrow - // once we remove any special handling logic for the rootContainer. - PolarisResolvedPathWrapper rootContainerWrapper = - resolutionManifest.getResolvedRootContainerEntityAsPath(); - PolarisResolvedPathWrapper principalRoleWrapper = - resolutionManifest.getResolvedTopLevelEntity( - principalRoleName, PolarisEntityType.PRINCIPAL_ROLE); - - authorizer.authorizeOrThrow( - authenticatedPrincipal, - resolutionManifest.getAllActivatedCatalogRoleAndPrincipalRoles(), - op, - rootContainerWrapper, - principalRoleWrapper); - } - - private void authorizeGrantOnTopLevelEntityToPrincipalRoleOperationOrThrow( - PolarisAuthorizableOperation op, - String topLevelEntityName, - PolarisEntityType topLevelEntityType, - String principalRoleName) { - resolutionManifest = - entityManager.prepareResolutionManifest(callContext, authenticatedPrincipal, null); - resolutionManifest.addTopLevelName( - topLevelEntityName, topLevelEntityType, false /* isOptional */); - resolutionManifest.addTopLevelName( - principalRoleName, PolarisEntityType.PRINCIPAL_ROLE, false /* isOptional */); - ResolverStatus status = resolutionManifest.resolveAll(); - - if (status.getStatus() == ResolverStatus.StatusEnum.ENTITY_COULD_NOT_BE_RESOLVED) { - throw new NotFoundException( - "Entity %s not found when trying to assign %s of type %s to %s", - status.getFailedToResolvedEntityName(), - topLevelEntityName, - topLevelEntityType, - principalRoleName); - } - - PolarisResolvedPathWrapper topLevelEntityWrapper = - resolutionManifest.getResolvedTopLevelEntity(topLevelEntityName, topLevelEntityType); - PolarisResolvedPathWrapper principalRoleWrapper = - resolutionManifest.getResolvedTopLevelEntity( - principalRoleName, PolarisEntityType.PRINCIPAL_ROLE); - - authorizer.authorizeOrThrow( - authenticatedPrincipal, - resolutionManifest.getAllActivatedCatalogRoleAndPrincipalRoles(), - op, - topLevelEntityWrapper, - principalRoleWrapper); - } - - private void authorizeGrantOnPrincipalRoleToPrincipalOperationOrThrow( - PolarisAuthorizableOperation op, String principalRoleName, String principalName) { - resolutionManifest = - entityManager.prepareResolutionManifest(callContext, authenticatedPrincipal, null); - resolutionManifest.addTopLevelName( - principalRoleName, PolarisEntityType.PRINCIPAL_ROLE, false /* isOptional */); - resolutionManifest.addTopLevelName( - principalName, PolarisEntityType.PRINCIPAL, false /* isOptional */); - ResolverStatus status = resolutionManifest.resolveAll(); - - if (status.getStatus() == ResolverStatus.StatusEnum.ENTITY_COULD_NOT_BE_RESOLVED) { - throw new NotFoundException( - "Entity %s not found when trying to assign %s to %s", - status.getFailedToResolvedEntityName(), principalRoleName, principalName); - } - - PolarisResolvedPathWrapper principalRoleWrapper = - resolutionManifest.getResolvedTopLevelEntity( - principalRoleName, PolarisEntityType.PRINCIPAL_ROLE); - PolarisResolvedPathWrapper principalWrapper = - resolutionManifest.getResolvedTopLevelEntity(principalName, PolarisEntityType.PRINCIPAL); - - authorizer.authorizeOrThrow( - authenticatedPrincipal, - resolutionManifest.getAllActivatedCatalogRoleAndPrincipalRoles(), - op, - principalRoleWrapper, - principalWrapper); - } - - private void authorizeGrantOnCatalogRoleToPrincipalRoleOperationOrThrow( - PolarisAuthorizableOperation op, - String catalogName, - String catalogRoleName, - String principalRoleName) { - resolutionManifest = - entityManager.prepareResolutionManifest(callContext, authenticatedPrincipal, catalogName); - resolutionManifest.addPath( - new ResolverPath(List.of(catalogRoleName), PolarisEntityType.CATALOG_ROLE), - catalogRoleName); - resolutionManifest.addTopLevelName( - principalRoleName, PolarisEntityType.PRINCIPAL_ROLE, false /* isOptional */); - ResolverStatus status = resolutionManifest.resolveAll(); - - if (status.getStatus() == ResolverStatus.StatusEnum.ENTITY_COULD_NOT_BE_RESOLVED) { - throw new NotFoundException( - "Entity %s not found when trying to assign %s.%s to %s", - status.getFailedToResolvedEntityName(), catalogName, catalogRoleName, principalRoleName); - } else if (status.getStatus() == ResolverStatus.StatusEnum.PATH_COULD_NOT_BE_FULLY_RESOLVED) { - throw new NotFoundException( - "Entity %s not found when trying to assign %s.%s to %s", - status.getFailedToResolvePath(), catalogName, catalogRoleName, principalRoleName); - } - - PolarisResolvedPathWrapper principalRoleWrapper = - resolutionManifest.getResolvedTopLevelEntity( - principalRoleName, PolarisEntityType.PRINCIPAL_ROLE); - PolarisResolvedPathWrapper catalogRoleWrapper = - resolutionManifest.getResolvedPath(catalogRoleName, true); - - authorizer.authorizeOrThrow( - authenticatedPrincipal, - resolutionManifest.getAllActivatedCatalogRoleAndPrincipalRoles(), - op, - catalogRoleWrapper, - principalRoleWrapper); - } - - private void authorizeGrantOnCatalogOperationOrThrow( - PolarisAuthorizableOperation op, String catalogName, String catalogRoleName) { - resolutionManifest = - entityManager.prepareResolutionManifest(callContext, authenticatedPrincipal, catalogName); - resolutionManifest.addTopLevelName( - catalogName, PolarisEntityType.CATALOG, false /* isOptional */); - resolutionManifest.addPath( - new ResolverPath(List.of(catalogRoleName), PolarisEntityType.CATALOG_ROLE), - catalogRoleName); - ResolverStatus status = resolutionManifest.resolveAll(); - - if (status.getStatus() == ResolverStatus.StatusEnum.ENTITY_COULD_NOT_BE_RESOLVED) { - throw new NotFoundException("Catalog not found: %s", catalogName); - } else if (status.getStatus() == ResolverStatus.StatusEnum.PATH_COULD_NOT_BE_FULLY_RESOLVED) { - throw new NotFoundException("CatalogRole not found: %s.%s", catalogName, catalogRoleName); - } - - PolarisResolvedPathWrapper catalogWrapper = - resolutionManifest.getResolvedTopLevelEntity(catalogName, PolarisEntityType.CATALOG); - PolarisResolvedPathWrapper catalogRoleWrapper = - resolutionManifest.getResolvedPath(catalogRoleName, true); - authorizer.authorizeOrThrow( - authenticatedPrincipal, - resolutionManifest.getAllActivatedCatalogRoleAndPrincipalRoles(), - op, - catalogWrapper, - catalogRoleWrapper); - } - - private void authorizeGrantOnNamespaceOperationOrThrow( - PolarisAuthorizableOperation op, - String catalogName, - Namespace namespace, - String catalogRoleName) { - resolutionManifest = - entityManager.prepareResolutionManifest(callContext, authenticatedPrincipal, catalogName); - resolutionManifest.addPath( - new ResolverPath(Arrays.asList(namespace.levels()), PolarisEntityType.NAMESPACE), - namespace); - resolutionManifest.addPath( - new ResolverPath(List.of(catalogRoleName), PolarisEntityType.CATALOG_ROLE), - catalogRoleName); - ResolverStatus status = resolutionManifest.resolveAll(); - - if (status.getStatus() == ResolverStatus.StatusEnum.ENTITY_COULD_NOT_BE_RESOLVED) { - throw new NotFoundException("Catalog not found: %s", catalogName); - } else if (status.getStatus() == ResolverStatus.StatusEnum.PATH_COULD_NOT_BE_FULLY_RESOLVED) { - if (status.getFailedToResolvePath().getLastEntityType() == PolarisEntityType.NAMESPACE) { - throw new NoSuchNamespaceException( - "Namespace does not exist: %s", status.getFailedToResolvePath().getEntityNames()); - } else { - throw new NotFoundException("CatalogRole not found: %s.%s", catalogName, catalogRoleName); - } - } - - PolarisResolvedPathWrapper namespaceWrapper = - resolutionManifest.getResolvedPath(namespace, true); - PolarisResolvedPathWrapper catalogRoleWrapper = - resolutionManifest.getResolvedPath(catalogRoleName, true); - - authorizer.authorizeOrThrow( - authenticatedPrincipal, - resolutionManifest.getAllActivatedCatalogRoleAndPrincipalRoles(), - op, - namespaceWrapper, - catalogRoleWrapper); - } - - private void authorizeGrantOnTableLikeOperationOrThrow( - PolarisAuthorizableOperation op, - String catalogName, - PolarisEntitySubType subType, - TableIdentifier identifier, - String catalogRoleName) { - resolutionManifest = - entityManager.prepareResolutionManifest(callContext, authenticatedPrincipal, catalogName); - resolutionManifest.addPath( - new ResolverPath( - PolarisCatalogHelpers.tableIdentifierToList(identifier), PolarisEntityType.TABLE_LIKE), - identifier); - resolutionManifest.addPath( - new ResolverPath(List.of(catalogRoleName), PolarisEntityType.CATALOG_ROLE), - catalogRoleName); - ResolverStatus status = resolutionManifest.resolveAll(); - - if (status.getStatus() == ResolverStatus.StatusEnum.ENTITY_COULD_NOT_BE_RESOLVED) { - throw new NotFoundException("Catalog not found: %s", catalogName); - } else if (status.getStatus() == ResolverStatus.StatusEnum.PATH_COULD_NOT_BE_FULLY_RESOLVED) { - if (status.getFailedToResolvePath().getLastEntityType() == PolarisEntityType.TABLE_LIKE) { - if (subType == PolarisEntitySubType.TABLE) { - throw new NoSuchTableException("Table does not exist: %s", identifier); - } else { - throw new NoSuchViewException("View does not exist: %s", identifier); - } - } else { - throw new NotFoundException("CatalogRole not found: %s.%s", catalogName, catalogRoleName); - } - } - - PolarisResolvedPathWrapper tableLikeWrapper = - resolutionManifest.getResolvedPath(identifier, subType, true); - PolarisResolvedPathWrapper catalogRoleWrapper = - resolutionManifest.getResolvedPath(catalogRoleName, true); - - authorizer.authorizeOrThrow( - authenticatedPrincipal, - resolutionManifest.getAllActivatedCatalogRoleAndPrincipalRoles(), - op, - tableLikeWrapper, - catalogRoleWrapper); - } - - /** Get all locations where data for a `CatalogEntity` may be stored */ - private Set getCatalogLocations(CatalogEntity catalogEntity) { - HashSet catalogLocations = new HashSet<>(); - catalogLocations.add(terminateWithSlash(catalogEntity.getDefaultBaseLocation())); - if (catalogEntity.getStorageConfigurationInfo() != null) { - catalogLocations.addAll( - catalogEntity.getStorageConfigurationInfo().getAllowedLocations().stream() - .map(this::terminateWithSlash) - .toList()); - } - return catalogLocations; - } - - /** Ensure a path is terminated with a `/` */ - private String terminateWithSlash(String path) { - if (path == null) { - return null; - } else if (path.endsWith("/")) { - return path; - } - return path + "/"; - } - - /** - * True if the `CatalogEntity` has a default base location or allowed location that overlaps with - * that of any existing catalog. If `ALLOW_OVERLAPPING_CATALOG_URLS` is set to true, this check - * will be skipped. - */ - private boolean catalogOverlapsWithExistingCatalog(CatalogEntity catalogEntity) { - boolean allowOverlappingCatalogUrls = - getCurrentPolarisContext() - .getConfigurationStore() - .getConfiguration( - getCurrentPolarisContext(), PolarisConfiguration.ALLOW_OVERLAPPING_CATALOG_URLS); - - if (allowOverlappingCatalogUrls) { - return false; - } - - Set newCatalogLocations = getCatalogLocations(catalogEntity); - return listCatalogsUnsafe().stream() - .map(CatalogEntity::new) - .anyMatch( - existingCatalog -> { - if (existingCatalog.getName().equals(catalogEntity.getName())) { - return false; - } - return getCatalogLocations(existingCatalog).stream() - .map(StorageLocation::of) - .anyMatch( - existingLocation -> { - return newCatalogLocations.stream() - .anyMatch( - newLocationString -> { - StorageLocation newLocation = - StorageLocation.of(newLocationString); - return newLocation.isChildOf(existingLocation) - || existingLocation.isChildOf(newLocation); - }); - }); - }); - } - - public PolarisEntity createCatalog(PolarisEntity entity) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.CREATE_CATALOG; - authorizeBasicRootOperationOrThrow(op); - - if (catalogOverlapsWithExistingCatalog((CatalogEntity) entity)) { - throw new ValidationException( - "Cannot create Catalog %s. One or more of its locations overlaps with an existing catalog", - entity.getName()); - } - - long id = - entity.getId() <= 0 - ? metaStoreManager.generateNewEntityId(getCurrentPolarisContext()).getId() - : entity.getId(); - PolarisEntity polarisEntity = - new PolarisEntity.Builder(entity) - .setId(id) - .setCreateTimestamp(System.currentTimeMillis()) - .build(); - PolarisMetaStoreManager.CreateCatalogResult catalogResult = - metaStoreManager.createCatalog(getCurrentPolarisContext(), polarisEntity, List.of()); - if (catalogResult.alreadyExists()) { - throw new AlreadyExistsException( - "Cannot create Catalog %s. Catalog already exists or resolution failed", - entity.getName()); - } - return PolarisEntity.of(catalogResult.getCatalog()); - } - - public void deleteCatalog(String name) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.DELETE_CATALOG; - authorizeBasicTopLevelEntityOperationOrThrow(op, name, PolarisEntityType.CATALOG); - - PolarisEntity entity = - findCatalogByName(name) - .orElseThrow(() -> new NotFoundException("Catalog %s not found", name)); - // TODO: Handle return value in case of concurrent modification - PolarisCallContext polarisCallContext = callContext.getPolarisCallContext(); - boolean cleanup = - polarisCallContext - .getConfigurationStore() - .getConfiguration(polarisCallContext, PolarisConfiguration.CLEANUP_ON_CATALOG_DROP); - PolarisMetaStoreManager.DropEntityResult dropEntityResult = - metaStoreManager.dropEntityIfExists( - getCurrentPolarisContext(), null, entity, Map.of(), cleanup); - - // at least some handling of error - if (!dropEntityResult.isSuccess()) { - if (dropEntityResult.failedBecauseNotEmpty()) { - throw new BadRequestException( - "Catalog '%s' cannot be dropped, it is not empty", entity.getName()); - } else { - throw new BadRequestException( - "Catalog '%s' cannot be dropped, concurrent modification detected. Please try " - + "again", - entity.getName()); - } - } - } - - public CatalogEntity getCatalog(String name) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.GET_CATALOG; - authorizeBasicTopLevelEntityOperationOrThrow(op, name, PolarisEntityType.CATALOG); - - return findCatalogByName(name) - .orElseThrow(() -> new NotFoundException("Catalog %s not found", name)); - } - - /** - * Helper to validate business logic of what is allowed to be updated or throw a - * BadRequestException. - */ - private void validateUpdateCatalogDiffOrThrow( - CatalogEntity currentEntity, CatalogEntity newEntity) { - // TODO: Expand the set of validations if there are other fields for other cloud providers - // that we can't successfully apply changes to. - PolarisStorageConfigurationInfo currentStorageConfig = - currentEntity.getStorageConfigurationInfo(); - PolarisStorageConfigurationInfo newStorageConfig = newEntity.getStorageConfigurationInfo(); - - if (currentStorageConfig == null || newStorageConfig == null) { - return; - } - - if (!currentStorageConfig.getClass().equals(newStorageConfig.getClass())) { - throw new BadRequestException( - "Cannot modify storage type of storage config from %s to %s", - currentStorageConfig, newStorageConfig); - } - - if (currentStorageConfig instanceof AwsStorageConfigurationInfo currentAwsConfig - && newStorageConfig instanceof AwsStorageConfigurationInfo newAwsConfig) { - - if (!currentAwsConfig.getRoleARN().equals(newAwsConfig.getRoleARN()) - || !newAwsConfig.getRoleARN().equals(currentAwsConfig.getRoleARN())) { - throw new BadRequestException( - "Cannot modify Role ARN in storage config from %s to %s", - currentStorageConfig, newStorageConfig); - } - - if ((currentAwsConfig.getExternalId() != null - && !currentAwsConfig.getExternalId().equals(newAwsConfig.getExternalId())) - || (newAwsConfig.getExternalId() != null - && !newAwsConfig.getExternalId().equals(currentAwsConfig.getExternalId()))) { - throw new BadRequestException( - "Cannot modify ExternalId in storage config from %s to %s", - currentStorageConfig, newStorageConfig); - } - } else if (currentStorageConfig instanceof AzureStorageConfigurationInfo currentAzureConfig - && newStorageConfig instanceof AzureStorageConfigurationInfo newAzureConfig) { - - if (!currentAzureConfig.getTenantId().equals(newAzureConfig.getTenantId()) - || !newAzureConfig.getTenantId().equals(currentAzureConfig.getTenantId())) { - throw new BadRequestException( - "Cannot modify TenantId in storage config from %s to %s", - currentStorageConfig, newStorageConfig); - } - } - } - - public @Nonnull CatalogEntity updateCatalog(String name, UpdateCatalogRequest updateRequest) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.UPDATE_CATALOG; - authorizeBasicTopLevelEntityOperationOrThrow(op, name, PolarisEntityType.CATALOG); - - CatalogEntity currentCatalogEntity = - findCatalogByName(name) - .orElseThrow(() -> new NotFoundException("Catalog %s not found", name)); - - if (currentCatalogEntity.getEntityVersion() != updateRequest.getCurrentEntityVersion()) { - throw new CommitFailedException( - "Failed to update Catalog; currentEntityVersion '%s', expected '%s'", - currentCatalogEntity.getEntityVersion(), updateRequest.getCurrentEntityVersion()); - } - - CatalogEntity.Builder updateBuilder = new CatalogEntity.Builder(currentCatalogEntity); - String defaultBaseLocation = currentCatalogEntity.getDefaultBaseLocation(); - if (updateRequest.getProperties() != null) { - updateBuilder.setProperties(updateRequest.getProperties()); - String newDefaultBaseLocation = - updateRequest.getProperties().get(CatalogEntity.DEFAULT_BASE_LOCATION_KEY); - // Since defaultBaseLocation is a required field during construction of a catalog, and the - // syntax of the Catalog API model splits default-base-location out from other keys in - // additionalProperties, it's easy for client libraries to focus on adding/merging - // additionalProperties while neglecting to "echo" the default-base-location from the - // fetched catalog, it's most user-friendly to treat a null or empty default-base-location - // as meaning no intended change to the default-base-location. - if (StringUtils.isNotEmpty(newDefaultBaseLocation)) { - // New base location is already in the updated properties; we'll also potentially - // plumb it into the logic for setting an updated StorageConfigurationInfo. - defaultBaseLocation = newDefaultBaseLocation; - } else { - // No default-base-location present at all in the properties of the update request, - // so we must restore it explicitly in the updateBuilder. - updateBuilder.setDefaultBaseLocation(defaultBaseLocation); - } - } - if (updateRequest.getStorageConfigInfo() != null) { - updateBuilder.setStorageConfigurationInfo( - updateRequest.getStorageConfigInfo(), defaultBaseLocation); - } - CatalogEntity updatedEntity = updateBuilder.build(); - - validateUpdateCatalogDiffOrThrow(currentCatalogEntity, updatedEntity); - - if (catalogOverlapsWithExistingCatalog(updatedEntity)) { - throw new ValidationException( - "Cannot update Catalog %s. One or more of its new locations overlaps with an existing catalog", - updatedEntity.getName()); - } - - CatalogEntity returnedEntity = - Optional.ofNullable( - CatalogEntity.of( - PolarisEntity.of( - metaStoreManager.updateEntityPropertiesIfNotChanged( - getCurrentPolarisContext(), null, updatedEntity)))) - .orElseThrow( - () -> - new CommitFailedException( - "Concurrent modification on Catalog '%s'; retry later", name)); - return returnedEntity; - } - - public List listCatalogs() { - authorizeBasicRootOperationOrThrow(PolarisAuthorizableOperation.LIST_CATALOGS); - return listCatalogsUnsafe(); - } - - /** List all catalogs without checking for permission */ - private List listCatalogsUnsafe() { - return metaStoreManager - .listEntities( - getCurrentPolarisContext(), - null, - PolarisEntityType.CATALOG, - PolarisEntitySubType.ANY_SUBTYPE) - .getEntities() - .stream() - .map( - nameAndId -> - PolarisEntity.of( - metaStoreManager.loadEntity(getCurrentPolarisContext(), 0, nameAndId.getId()))) - .toList(); - } - - public PrincipalWithCredentials createPrincipal(PolarisEntity entity) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.CREATE_PRINCIPAL; - authorizeBasicRootOperationOrThrow(op); - - long id = - entity.getId() <= 0 - ? metaStoreManager.generateNewEntityId(getCurrentPolarisContext()).getId() - : entity.getId(); - PolarisMetaStoreManager.CreatePrincipalResult principalResult = - metaStoreManager.createPrincipal( - getCurrentPolarisContext(), - new PolarisEntity.Builder(entity) - .setId(id) - .setCreateTimestamp(System.currentTimeMillis()) - .build()); - if (principalResult.alreadyExists()) { - throw new AlreadyExistsException( - "Cannot create Principal %s. Principal already exists or resolution failed", - entity.getName()); - } - return new PrincipalWithCredentials( - new PrincipalEntity(principalResult.getPrincipal()).asPrincipal(), - new PrincipalWithCredentialsCredentials( - principalResult.getPrincipalSecrets().getPrincipalClientId(), - principalResult.getPrincipalSecrets().getMainSecret())); - } - - public void deletePrincipal(String name) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.DELETE_PRINCIPAL; - authorizeBasicTopLevelEntityOperationOrThrow(op, name, PolarisEntityType.PRINCIPAL); - - PolarisEntity entity = - findPrincipalByName(name) - .orElseThrow(() -> new NotFoundException("Principal %s not found", name)); - // TODO: Handle return value in case of concurrent modification - PolarisMetaStoreManager.DropEntityResult dropEntityResult = - metaStoreManager.dropEntityIfExists( - getCurrentPolarisContext(), null, entity, Map.of(), false); - - // at least some handling of error - if (!dropEntityResult.isSuccess()) { - if (dropEntityResult.isEntityUnDroppable()) { - throw new BadRequestException("Root principal cannot be dropped"); - } else { - throw new BadRequestException( - "Root principal cannot be dropped, concurrent modification " - + "detected. Please try again"); - } - } - } - - public @Nonnull PrincipalEntity getPrincipal(String name) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.GET_PRINCIPAL; - authorizeBasicTopLevelEntityOperationOrThrow(op, name, PolarisEntityType.PRINCIPAL); - - return findPrincipalByName(name) - .orElseThrow(() -> new NotFoundException("Principal %s not found", name)); - } - - public @Nonnull PrincipalEntity updatePrincipal( - String name, UpdatePrincipalRequest updateRequest) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.UPDATE_PRINCIPAL; - authorizeBasicTopLevelEntityOperationOrThrow(op, name, PolarisEntityType.PRINCIPAL); - - PrincipalEntity currentPrincipalEntity = - findPrincipalByName(name) - .orElseThrow(() -> new NotFoundException("Principal %s not found", name)); - - if (currentPrincipalEntity.getEntityVersion() != updateRequest.getCurrentEntityVersion()) { - throw new CommitFailedException( - "Failed to update Principal; currentEntityVersion '%s', expected '%s'", - currentPrincipalEntity.getEntityVersion(), updateRequest.getCurrentEntityVersion()); - } - - PrincipalEntity.Builder updateBuilder = new PrincipalEntity.Builder(currentPrincipalEntity); - if (updateRequest.getProperties() != null) { - updateBuilder.setProperties(updateRequest.getProperties()); - } - PrincipalEntity updatedEntity = updateBuilder.build(); - PrincipalEntity returnedEntity = - Optional.ofNullable( - PrincipalEntity.of( - PolarisEntity.of( - metaStoreManager.updateEntityPropertiesIfNotChanged( - getCurrentPolarisContext(), null, updatedEntity)))) - .orElseThrow( - () -> - new CommitFailedException( - "Concurrent modification on Principal '%s'; retry later", name)); - return returnedEntity; - } - - private @Nonnull PrincipalWithCredentials rotateOrResetCredentialsHelper( - String principalName, boolean shouldReset) { - PrincipalEntity currentPrincipalEntity = - findPrincipalByName(principalName) - .orElseThrow(() -> new NotFoundException("Principal %s not found", principalName)); - - PolarisPrincipalSecrets currentSecrets = - metaStoreManager - .loadPrincipalSecrets(getCurrentPolarisContext(), currentPrincipalEntity.getClientId()) - .getPrincipalSecrets(); - if (currentSecrets == null) { - throw new IllegalArgumentException( - String.format("Failed to load current secrets for principal '%s'", principalName)); - } - PolarisPrincipalSecrets newSecrets = - metaStoreManager - .rotatePrincipalSecrets( - getCurrentPolarisContext(), - currentPrincipalEntity.getClientId(), - currentPrincipalEntity.getId(), - shouldReset, - currentSecrets.getMainSecret()) - .getPrincipalSecrets(); - if (newSecrets == null) { - throw new IllegalStateException( - String.format( - "Failed to %s secrets for principal '%s'", - shouldReset ? "reset" : "rotate", principalName)); - } - PolarisEntity newPrincipal = - PolarisEntity.of( - metaStoreManager.loadEntity( - getCurrentPolarisContext(), 0L, currentPrincipalEntity.getId())); - return new PrincipalWithCredentials( - PrincipalEntity.of(newPrincipal).asPrincipal(), - new PrincipalWithCredentialsCredentials( - newSecrets.getPrincipalClientId(), newSecrets.getMainSecret())); - } - - public @Nonnull PrincipalWithCredentials rotateCredentials(String principalName) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.ROTATE_CREDENTIALS; - authorizeBasicTopLevelEntityOperationOrThrow(op, principalName, PolarisEntityType.PRINCIPAL); - - return rotateOrResetCredentialsHelper(principalName, false); - } - - public @Nonnull PrincipalWithCredentials resetCredentials(String principalName) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.RESET_CREDENTIALS; - authorizeBasicTopLevelEntityOperationOrThrow(op, principalName, PolarisEntityType.PRINCIPAL); - - return rotateOrResetCredentialsHelper(principalName, true); - } - - public List listPrincipals() { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.LIST_PRINCIPALS; - authorizeBasicRootOperationOrThrow(op); - - return metaStoreManager - .listEntities( - getCurrentPolarisContext(), - null, - PolarisEntityType.PRINCIPAL, - PolarisEntitySubType.NULL_SUBTYPE) - .getEntities() - .stream() - .map( - nameAndId -> - PolarisEntity.of( - metaStoreManager.loadEntity(getCurrentPolarisContext(), 0, nameAndId.getId()))) - .toList(); - } - - public PolarisEntity createPrincipalRole(PolarisEntity entity) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.CREATE_PRINCIPAL_ROLE; - authorizeBasicRootOperationOrThrow(op); - - long id = - entity.getId() <= 0 - ? metaStoreManager.generateNewEntityId(getCurrentPolarisContext()).getId() - : entity.getId(); - PolarisEntity returnedEntity = - PolarisEntity.of( - metaStoreManager.createEntityIfNotExists( - getCurrentPolarisContext(), - null, - new PolarisEntity.Builder(entity) - .setId(id) - .setCreateTimestamp(System.currentTimeMillis()) - .build())); - if (returnedEntity == null) { - throw new AlreadyExistsException( - "Cannot create PrincipalRole %s. PrincipalRole already exists or resolution failed", - entity.getName()); - } - return returnedEntity; - } - - public void deletePrincipalRole(String name) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.DELETE_PRINCIPAL_ROLE; - authorizeBasicTopLevelEntityOperationOrThrow(op, name, PolarisEntityType.PRINCIPAL_ROLE); - - PolarisEntity entity = - findPrincipalRoleByName(name) - .orElseThrow(() -> new NotFoundException("PrincipalRole %s not found", name)); - // TODO: Handle return value in case of concurrent modification - PolarisMetaStoreManager.DropEntityResult dropEntityResult = - metaStoreManager.dropEntityIfExists( - getCurrentPolarisContext(), null, entity, Map.of(), true); // cleanup grants - - // at least some handling of error - if (!dropEntityResult.isSuccess()) { - if (dropEntityResult.isEntityUnDroppable()) { - throw new BadRequestException("Polaris service admin principal role cannot be dropped"); - } else { - throw new BadRequestException( - "Polaris service admin principal role cannot be dropped, " - + "concurrent modification detected. Please try again"); - } - } - } - - public @Nonnull PrincipalRoleEntity getPrincipalRole(String name) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.GET_PRINCIPAL_ROLE; - authorizeBasicTopLevelEntityOperationOrThrow(op, name, PolarisEntityType.PRINCIPAL_ROLE); - - return findPrincipalRoleByName(name) - .orElseThrow(() -> new NotFoundException("PrincipalRole %s not found", name)); - } - - public @Nonnull PrincipalRoleEntity updatePrincipalRole( - String name, UpdatePrincipalRoleRequest updateRequest) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.UPDATE_PRINCIPAL_ROLE; - authorizeBasicTopLevelEntityOperationOrThrow(op, name, PolarisEntityType.PRINCIPAL_ROLE); - - PrincipalRoleEntity currentPrincipalRoleEntity = - findPrincipalRoleByName(name) - .orElseThrow(() -> new NotFoundException("PrincipalRole %s not found", name)); - - if (currentPrincipalRoleEntity.getEntityVersion() != updateRequest.getCurrentEntityVersion()) { - throw new CommitFailedException( - "Failed to update PrincipalRole; currentEntityVersion '%s', expected '%s'", - currentPrincipalRoleEntity.getEntityVersion(), updateRequest.getCurrentEntityVersion()); - } - - PrincipalRoleEntity.Builder updateBuilder = - new PrincipalRoleEntity.Builder(currentPrincipalRoleEntity); - if (updateRequest.getProperties() != null) { - updateBuilder.setProperties(updateRequest.getProperties()); - } - PrincipalRoleEntity updatedEntity = updateBuilder.build(); - PrincipalRoleEntity returnedEntity = - Optional.ofNullable( - PrincipalRoleEntity.of( - PolarisEntity.of( - metaStoreManager.updateEntityPropertiesIfNotChanged( - getCurrentPolarisContext(), null, updatedEntity)))) - .orElseThrow( - () -> - new CommitFailedException( - "Concurrent modification on PrincipalRole '%s'; retry later", name)); - return returnedEntity; - } - - public List listPrincipalRoles() { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.LIST_PRINCIPAL_ROLES; - authorizeBasicRootOperationOrThrow(op); - - return metaStoreManager - .listEntities( - getCurrentPolarisContext(), - null, - PolarisEntityType.PRINCIPAL_ROLE, - PolarisEntitySubType.NULL_SUBTYPE) - .getEntities() - .stream() - .map( - nameAndId -> - PolarisEntity.of( - metaStoreManager.loadEntity(getCurrentPolarisContext(), 0, nameAndId.getId()))) - .toList(); - } - - public PolarisEntity createCatalogRole(String catalogName, PolarisEntity entity) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.CREATE_CATALOG_ROLE; - authorizeBasicTopLevelEntityOperationOrThrow(op, catalogName, PolarisEntityType.CATALOG); - - PolarisEntity catalogEntity = - findCatalogByName(catalogName) - .orElseThrow(() -> new NotFoundException("Parent catalog %s not found", catalogName)); - - long id = - entity.getId() <= 0 - ? metaStoreManager.generateNewEntityId(getCurrentPolarisContext()).getId() - : entity.getId(); - PolarisEntity returnedEntity = - PolarisEntity.of( - metaStoreManager.createEntityIfNotExists( - getCurrentPolarisContext(), - PolarisEntity.toCoreList(List.of(catalogEntity)), - new PolarisEntity.Builder(entity) - .setId(id) - .setCatalogId(catalogEntity.getId()) - .setParentId(catalogEntity.getId()) - .setCreateTimestamp(System.currentTimeMillis()) - .build())); - if (returnedEntity == null) { - throw new AlreadyExistsException( - "Cannot create CatalogRole %s in %s. CatalogRole already exists or resolution failed", - entity.getName(), catalogName); - } - return returnedEntity; - } - - public void deleteCatalogRole(String catalogName, String name) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.DELETE_CATALOG_ROLE; - authorizeBasicCatalogRoleOperationOrThrow(op, catalogName, name); - - PolarisResolvedPathWrapper resolvedCatalogRoleEntity = resolutionManifest.getResolvedPath(name); - if (resolvedCatalogRoleEntity == null) { - throw new NotFoundException("CatalogRole %s not found in catalog %s", name, catalogName); - } - // TODO: Handle return value in case of concurrent modification - PolarisMetaStoreManager.DropEntityResult dropEntityResult = - metaStoreManager.dropEntityIfExists( - getCurrentPolarisContext(), - PolarisEntity.toCoreList(resolvedCatalogRoleEntity.getRawParentPath()), - resolvedCatalogRoleEntity.getRawLeafEntity(), - Map.of(), - true); // cleanup grants - - // at least some handling of error - if (!dropEntityResult.isSuccess()) { - if (dropEntityResult.isEntityUnDroppable()) { - throw new BadRequestException("Catalog admin role cannot be dropped"); - } else { - throw new BadRequestException( - "Catalog admin role cannot be dropped, concurrent " - + "modification detected. Please try again"); - } - } - } - - public @Nonnull CatalogRoleEntity getCatalogRole(String catalogName, String name) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.GET_CATALOG_ROLE; - authorizeBasicCatalogRoleOperationOrThrow(op, catalogName, name); - - return findCatalogRoleByName(catalogName, name) - .orElseThrow(() -> new NotFoundException("CatalogRole %s not found", name)); - } - - public @Nonnull CatalogRoleEntity updateCatalogRole( - String catalogName, String name, UpdateCatalogRoleRequest updateRequest) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.UPDATE_CATALOG_ROLE; - authorizeBasicCatalogRoleOperationOrThrow(op, catalogName, name); - - CatalogEntity catalogEntity = - findCatalogByName(catalogName) - .orElseThrow(() -> new NotFoundException("Catalog %s not found", catalogName)); - CatalogRoleEntity currentCatalogRoleEntity = - findCatalogRoleByName(catalogName, name) - .orElseThrow(() -> new NotFoundException("CatalogRole %s not found", name)); - - if (currentCatalogRoleEntity.getEntityVersion() != updateRequest.getCurrentEntityVersion()) { - throw new CommitFailedException( - "Failed to update CatalogRole; currentEntityVersion '%s', expected '%s'", - currentCatalogRoleEntity.getEntityVersion(), updateRequest.getCurrentEntityVersion()); - } - - CatalogRoleEntity.Builder updateBuilder = - new CatalogRoleEntity.Builder(currentCatalogRoleEntity); - if (updateRequest.getProperties() != null) { - updateBuilder.setProperties(updateRequest.getProperties()); - } - CatalogRoleEntity updatedEntity = updateBuilder.build(); - CatalogRoleEntity returnedEntity = - Optional.ofNullable( - CatalogRoleEntity.of( - PolarisEntity.of( - metaStoreManager.updateEntityPropertiesIfNotChanged( - getCurrentPolarisContext(), - PolarisEntity.toCoreList(List.of(catalogEntity)), - updatedEntity)))) - .orElseThrow( - () -> - new CommitFailedException( - "Concurrent modification on CatalogRole '%s'; retry later", name)); - return returnedEntity; - } - - public List listCatalogRoles(String catalogName) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.LIST_CATALOG_ROLES; - authorizeBasicTopLevelEntityOperationOrThrow(op, catalogName, PolarisEntityType.CATALOG); - - PolarisEntity catalogEntity = - findCatalogByName(catalogName) - .orElseThrow(() -> new NotFoundException("Parent catalog %s not found", catalogName)); - return metaStoreManager - .listEntities( - getCurrentPolarisContext(), - PolarisEntity.toCoreList(List.of(catalogEntity)), - PolarisEntityType.CATALOG_ROLE, - PolarisEntitySubType.NULL_SUBTYPE) - .getEntities() - .stream() - .map( - nameAndId -> - PolarisEntity.of( - metaStoreManager.loadEntity( - getCurrentPolarisContext(), catalogEntity.getId(), nameAndId.getId()))) - .toList(); - } - - public boolean assignPrincipalRole(String principalName, String principalRoleName) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.ASSIGN_PRINCIPAL_ROLE; - authorizeGrantOnPrincipalRoleToPrincipalOperationOrThrow(op, principalRoleName, principalName); - - PolarisEntity principalEntity = - findPrincipalByName(principalName) - .orElseThrow(() -> new NotFoundException("Principal %s not found", principalName)); - PolarisEntity principalRoleEntity = - findPrincipalRoleByName(principalRoleName) - .orElseThrow( - () -> new NotFoundException("PrincipalRole %s not found", principalRoleName)); - - return metaStoreManager - .grantUsageOnRoleToGrantee( - getCurrentPolarisContext(), null, principalRoleEntity, principalEntity) - .isSuccess(); - } - - public boolean revokePrincipalRole(String principalName, String principalRoleName) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.REVOKE_PRINCIPAL_ROLE; - authorizeGrantOnPrincipalRoleToPrincipalOperationOrThrow(op, principalRoleName, principalName); - - PolarisEntity principalEntity = - findPrincipalByName(principalName) - .orElseThrow(() -> new NotFoundException("Principal %s not found", principalName)); - PolarisEntity principalRoleEntity = - findPrincipalRoleByName(principalRoleName) - .orElseThrow( - () -> new NotFoundException("PrincipalRole %s not found", principalRoleName)); - return metaStoreManager - .revokeUsageOnRoleFromGrantee( - getCurrentPolarisContext(), null, principalRoleEntity, principalEntity) - .isSuccess(); - } - - public List listPrincipalRolesAssigned(String principalName) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.LIST_PRINCIPAL_ROLES_ASSIGNED; - - authorizeBasicTopLevelEntityOperationOrThrow(op, principalName, PolarisEntityType.PRINCIPAL); - - PolarisEntity principalEntity = - findPrincipalByName(principalName) - .orElseThrow(() -> new NotFoundException("Principal %s not found", principalName)); - LoadGrantsResult grantList = - metaStoreManager.loadGrantsToGrantee( - getCurrentPolarisContext(), principalEntity.getCatalogId(), principalEntity.getId()); - return buildEntitiesFromGrantResults(grantList, false, null); - } - - public boolean assignCatalogRoleToPrincipalRole( - String principalRoleName, String catalogName, String catalogRoleName) { - PolarisAuthorizableOperation op = - PolarisAuthorizableOperation.ASSIGN_CATALOG_ROLE_TO_PRINCIPAL_ROLE; - authorizeGrantOnCatalogRoleToPrincipalRoleOperationOrThrow( - op, catalogName, catalogRoleName, principalRoleName); - - PolarisEntity principalRoleEntity = - findPrincipalRoleByName(principalRoleName) - .orElseThrow( - () -> new NotFoundException("PrincipalRole %s not found", principalRoleName)); - PolarisEntity catalogEntity = - findCatalogByName(catalogName) - .orElseThrow(() -> new NotFoundException("Parent catalog %s not found", catalogName)); - PolarisEntity catalogRoleEntity = - findCatalogRoleByName(catalogName, catalogRoleName) - .orElseThrow(() -> new NotFoundException("CatalogRole %s not found", catalogRoleName)); - - return metaStoreManager - .grantUsageOnRoleToGrantee( - getCurrentPolarisContext(), catalogEntity, catalogRoleEntity, principalRoleEntity) - .isSuccess(); - } - - public boolean revokeCatalogRoleFromPrincipalRole( - String principalRoleName, String catalogName, String catalogRoleName) { - PolarisAuthorizableOperation op = - PolarisAuthorizableOperation.REVOKE_CATALOG_ROLE_FROM_PRINCIPAL_ROLE; - authorizeGrantOnCatalogRoleToPrincipalRoleOperationOrThrow( - op, catalogName, catalogRoleName, principalRoleName); - - PolarisEntity principalRoleEntity = - findPrincipalRoleByName(principalRoleName) - .orElseThrow( - () -> new NotFoundException("PrincipalRole %s not found", principalRoleName)); - PolarisEntity catalogEntity = - findCatalogByName(catalogName) - .orElseThrow(() -> new NotFoundException("Parent catalog %s not found", catalogName)); - PolarisEntity catalogRoleEntity = - findCatalogRoleByName(catalogName, catalogRoleName) - .orElseThrow(() -> new NotFoundException("CatalogRole %s not found", catalogRoleName)); - return metaStoreManager - .revokeUsageOnRoleFromGrantee( - getCurrentPolarisContext(), catalogEntity, catalogRoleEntity, principalRoleEntity) - .isSuccess(); - } - - public List listAssigneePrincipalsForPrincipalRole(String principalRoleName) { - PolarisAuthorizableOperation op = - PolarisAuthorizableOperation.LIST_ASSIGNEE_PRINCIPALS_FOR_PRINCIPAL_ROLE; - - authorizeBasicTopLevelEntityOperationOrThrow( - op, principalRoleName, PolarisEntityType.PRINCIPAL_ROLE); - - PolarisEntity principalRoleEntity = - findPrincipalRoleByName(principalRoleName) - .orElseThrow( - () -> new NotFoundException("PrincipalRole %s not found", principalRoleName)); - LoadGrantsResult grantList = - metaStoreManager.loadGrantsOnSecurable( - getCurrentPolarisContext(), - principalRoleEntity.getCatalogId(), - principalRoleEntity.getId()); - return buildEntitiesFromGrantResults(grantList, true, null); - } - - /** - * Build the list of entities matching the set of grant records returned by a grant lookup - * request. - * - * @param grantList result of a load grants on a securable or to a grantee - * @param grantees if true, return the list of grantee entities, else the list of securable - * entities - * @param grantFilter filter on the grant records, use null for all - * @return list of grantees or securables matching the filter - */ - private List buildEntitiesFromGrantResults( - @Nonnull LoadGrantsResult grantList, - boolean grantees, - @Nullable Function grantFilter) { - Map granteeMap = grantList.getEntitiesAsMap(); - List toReturn = new ArrayList<>(grantList.getGrantRecords().size()); - for (PolarisGrantRecord grantRecord : grantList.getGrantRecords()) { - if (grantFilter == null || grantFilter.apply(grantRecord)) { - long catalogId = - grantees ? grantRecord.getGranteeCatalogId() : grantRecord.getSecurableCatalogId(); - long entityId = grantees ? grantRecord.getGranteeId() : grantRecord.getSecurableId(); - // get the entity associated with the grantee - PolarisBaseEntity entity = this.getOrLoadEntity(granteeMap, catalogId, entityId); - if (entity != null) { - toReturn.add(PolarisEntity.of(entity)); - } - } - } - return toReturn; - } - - public List listCatalogRolesForPrincipalRole( - String principalRoleName, String catalogName) { - PolarisAuthorizableOperation op = - PolarisAuthorizableOperation.LIST_CATALOG_ROLES_FOR_PRINCIPAL_ROLE; - authorizeBasicTopLevelEntityOperationOrThrow( - op, principalRoleName, PolarisEntityType.PRINCIPAL_ROLE, catalogName); - - PolarisEntity catalogEntity = - findCatalogByName(catalogName) - .orElseThrow(() -> new NotFoundException("Parent catalog %s not found", catalogName)); - PolarisEntity principalRoleEntity = - findPrincipalRoleByName(principalRoleName) - .orElseThrow( - () -> new NotFoundException("PrincipalRole %s not found", principalRoleName)); - LoadGrantsResult grantList = - metaStoreManager.loadGrantsToGrantee( - getCurrentPolarisContext(), - principalRoleEntity.getCatalogId(), - principalRoleEntity.getId()); - return buildEntitiesFromGrantResults( - grantList, false, grantRec -> grantRec.getSecurableCatalogId() == catalogEntity.getId()); - } - - /** Adds a grant on the root container of this realm to {@code principalRoleName}. */ - public boolean grantPrivilegeOnRootContainerToPrincipalRole( - String principalRoleName, PolarisPrivilege privilege) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.ADD_ROOT_GRANT_TO_PRINCIPAL_ROLE; - authorizeGrantOnRootContainerToPrincipalRoleOperationOrThrow(op, principalRoleName); - - PolarisEntity rootContainerEntity = - resolutionManifest.getResolvedRootContainerEntityAsPath().getRawLeafEntity(); - PolarisEntity principalRoleEntity = - findPrincipalRoleByName(principalRoleName) - .orElseThrow( - () -> new NotFoundException("PrincipalRole %s not found", principalRoleName)); - - return metaStoreManager - .grantPrivilegeOnSecurableToRole( - getCurrentPolarisContext(), principalRoleEntity, null, rootContainerEntity, privilege) - .isSuccess(); - } - - /** Revokes a grant on the root container of this realm from {@code principalRoleName}. */ - public boolean revokePrivilegeOnRootContainerFromPrincipalRole( - String principalRoleName, PolarisPrivilege privilege) { - PolarisAuthorizableOperation op = - PolarisAuthorizableOperation.REVOKE_ROOT_GRANT_FROM_PRINCIPAL_ROLE; - authorizeGrantOnRootContainerToPrincipalRoleOperationOrThrow(op, principalRoleName); - - PolarisEntity rootContainerEntity = - resolutionManifest.getResolvedRootContainerEntityAsPath().getRawLeafEntity(); - PolarisEntity principalRoleEntity = - findPrincipalRoleByName(principalRoleName) - .orElseThrow( - () -> new NotFoundException("PrincipalRole %s not found", principalRoleName)); - - return metaStoreManager - .revokePrivilegeOnSecurableFromRole( - getCurrentPolarisContext(), principalRoleEntity, null, rootContainerEntity, privilege) - .isSuccess(); - } - - /** - * Adds a catalog-level grant on {@code catalogName} to {@code catalogRoleName} which resides - * within the same catalog on which it is being granted the privilege. - */ - public boolean grantPrivilegeOnCatalogToRole( - String catalogName, String catalogRoleName, PolarisPrivilege privilege) { - PolarisAuthorizableOperation op = - PolarisAuthorizableOperation.ADD_CATALOG_GRANT_TO_CATALOG_ROLE; - - authorizeGrantOnCatalogOperationOrThrow(op, catalogName, catalogRoleName); - - PolarisEntity catalogEntity = - findCatalogByName(catalogName) - .orElseThrow(() -> new NotFoundException("Parent catalog %s not found", catalogName)); - PolarisEntity catalogRoleEntity = - findCatalogRoleByName(catalogName, catalogRoleName) - .orElseThrow(() -> new NotFoundException("CatalogRole %s not found", catalogRoleName)); - - return metaStoreManager - .grantPrivilegeOnSecurableToRole( - getCurrentPolarisContext(), - catalogRoleEntity, - PolarisEntity.toCoreList(List.of(catalogEntity)), - catalogEntity, - privilege) - .isSuccess(); - } - - /** Removes a catalog-level grant on {@code catalogName} from {@code catalogRoleName}. */ - public boolean revokePrivilegeOnCatalogFromRole( - String catalogName, String catalogRoleName, PolarisPrivilege privilege) { - PolarisAuthorizableOperation op = - PolarisAuthorizableOperation.REVOKE_CATALOG_GRANT_FROM_CATALOG_ROLE; - authorizeGrantOnCatalogOperationOrThrow(op, catalogName, catalogRoleName); - - PolarisEntity catalogEntity = - findCatalogByName(catalogName) - .orElseThrow(() -> new NotFoundException("Parent catalog %s not found", catalogName)); - PolarisEntity catalogRoleEntity = - findCatalogRoleByName(catalogName, catalogRoleName) - .orElseThrow(() -> new NotFoundException("CatalogRole %s not found", catalogRoleName)); - - return metaStoreManager - .revokePrivilegeOnSecurableFromRole( - getCurrentPolarisContext(), - catalogRoleEntity, - PolarisEntity.toCoreList(List.of(catalogEntity)), - catalogEntity, - privilege) - .isSuccess(); - } - - /** Adds a namespace-level grant on {@code namespace} to {@code catalogRoleName}. */ - public boolean grantPrivilegeOnNamespaceToRole( - String catalogName, String catalogRoleName, Namespace namespace, PolarisPrivilege privilege) { - PolarisAuthorizableOperation op = - PolarisAuthorizableOperation.ADD_NAMESPACE_GRANT_TO_CATALOG_ROLE; - authorizeGrantOnNamespaceOperationOrThrow(op, catalogName, namespace, catalogRoleName); - - PolarisEntity catalogRoleEntity = - findCatalogRoleByName(catalogName, catalogRoleName) - .orElseThrow(() -> new NotFoundException("CatalogRole %s not found", catalogRoleName)); - - PolarisResolvedPathWrapper resolvedPathWrapper = resolutionManifest.getResolvedPath(namespace); - if (resolvedPathWrapper == null) { - throw new NotFoundException("Namespace %s not found", namespace); - } - List catalogPath = resolvedPathWrapper.getRawParentPath(); - PolarisEntity namespaceEntity = resolvedPathWrapper.getRawLeafEntity(); - - return metaStoreManager - .grantPrivilegeOnSecurableToRole( - getCurrentPolarisContext(), - catalogRoleEntity, - PolarisEntity.toCoreList(catalogPath), - namespaceEntity, - privilege) - .isSuccess(); - } - - /** Removes a namespace-level grant on {@code namespace} from {@code catalogRoleName}. */ - public boolean revokePrivilegeOnNamespaceFromRole( - String catalogName, String catalogRoleName, Namespace namespace, PolarisPrivilege privilege) { - PolarisAuthorizableOperation op = - PolarisAuthorizableOperation.REVOKE_NAMESPACE_GRANT_FROM_CATALOG_ROLE; - authorizeGrantOnNamespaceOperationOrThrow(op, catalogName, namespace, catalogRoleName); - - PolarisEntity catalogRoleEntity = - findCatalogRoleByName(catalogName, catalogRoleName) - .orElseThrow(() -> new NotFoundException("CatalogRole %s not found", catalogRoleName)); - - PolarisResolvedPathWrapper resolvedPathWrapper = resolutionManifest.getResolvedPath(namespace); - if (resolvedPathWrapper == null) { - throw new NotFoundException("Namespace %s not found", namespace); - } - List catalogPath = resolvedPathWrapper.getRawParentPath(); - PolarisEntity namespaceEntity = resolvedPathWrapper.getRawLeafEntity(); - - return metaStoreManager - .revokePrivilegeOnSecurableFromRole( - getCurrentPolarisContext(), - catalogRoleEntity, - PolarisEntity.toCoreList(catalogPath), - namespaceEntity, - privilege) - .isSuccess(); - } - - public boolean grantPrivilegeOnTableToRole( - String catalogName, - String catalogRoleName, - TableIdentifier identifier, - PolarisPrivilege privilege) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.ADD_TABLE_GRANT_TO_CATALOG_ROLE; - - authorizeGrantOnTableLikeOperationOrThrow( - op, catalogName, PolarisEntitySubType.TABLE, identifier, catalogRoleName); - - return grantPrivilegeOnTableLikeToRole( - catalogName, catalogRoleName, identifier, PolarisEntitySubType.TABLE, privilege); - } - - public boolean revokePrivilegeOnTableFromRole( - String catalogName, - String catalogRoleName, - TableIdentifier identifier, - PolarisPrivilege privilege) { - PolarisAuthorizableOperation op = - PolarisAuthorizableOperation.REVOKE_TABLE_GRANT_FROM_CATALOG_ROLE; - - authorizeGrantOnTableLikeOperationOrThrow( - op, catalogName, PolarisEntitySubType.TABLE, identifier, catalogRoleName); - - return revokePrivilegeOnTableLikeFromRole( - catalogName, catalogRoleName, identifier, PolarisEntitySubType.TABLE, privilege); - } - - public boolean grantPrivilegeOnViewToRole( - String catalogName, - String catalogRoleName, - TableIdentifier identifier, - PolarisPrivilege privilege) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.ADD_VIEW_GRANT_TO_CATALOG_ROLE; - - authorizeGrantOnTableLikeOperationOrThrow( - op, catalogName, PolarisEntitySubType.VIEW, identifier, catalogRoleName); - - return grantPrivilegeOnTableLikeToRole( - catalogName, catalogRoleName, identifier, PolarisEntitySubType.VIEW, privilege); - } - - public boolean revokePrivilegeOnViewFromRole( - String catalogName, - String catalogRoleName, - TableIdentifier identifier, - PolarisPrivilege privilege) { - PolarisAuthorizableOperation op = - PolarisAuthorizableOperation.REVOKE_VIEW_GRANT_FROM_CATALOG_ROLE; - - authorizeGrantOnTableLikeOperationOrThrow( - op, catalogName, PolarisEntitySubType.VIEW, identifier, catalogRoleName); - - return revokePrivilegeOnTableLikeFromRole( - catalogName, catalogRoleName, identifier, PolarisEntitySubType.VIEW, privilege); - } - - public List listAssigneePrincipalRolesForCatalogRole( - String catalogName, String catalogRoleName) { - PolarisAuthorizableOperation op = - PolarisAuthorizableOperation.LIST_ASSIGNEE_PRINCIPAL_ROLES_FOR_CATALOG_ROLE; - authorizeBasicCatalogRoleOperationOrThrow(op, catalogName, catalogRoleName); - - if (findCatalogByName(catalogName).isEmpty()) { - throw new NotFoundException("Parent catalog %s not found", catalogName); - } - PolarisEntity catalogRoleEntity = - findCatalogRoleByName(catalogName, catalogRoleName) - .orElseThrow(() -> new NotFoundException("CatalogRole %s not found", catalogRoleName)); - LoadGrantsResult grantList = - metaStoreManager.loadGrantsOnSecurable( - getCurrentPolarisContext(), - catalogRoleEntity.getCatalogId(), - catalogRoleEntity.getId()); - return buildEntitiesFromGrantResults(grantList, true, null); - } - - /** - * Lists all grants on Catalog-level resources (Catalog/Namespace/Table/View) granted to the - * specified catalogRole. - */ - public List listGrantsForCatalogRole(String catalogName, String catalogRoleName) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.LIST_GRANTS_FOR_CATALOG_ROLE; - authorizeBasicCatalogRoleOperationOrThrow(op, catalogName, catalogRoleName); - - PolarisEntity catalogRoleEntity = - findCatalogRoleByName(catalogName, catalogRoleName) - .orElseThrow(() -> new NotFoundException("CatalogRole %s not found", catalogRoleName)); - LoadGrantsResult grantList = - metaStoreManager.loadGrantsToGrantee( - getCurrentPolarisContext(), - catalogRoleEntity.getCatalogId(), - catalogRoleEntity.getId()); - List catalogGrants = new ArrayList<>(); - List namespaceGrants = new ArrayList<>(); - List tableGrants = new ArrayList<>(); - List viewGrants = new ArrayList<>(); - Map entityMap = grantList.getEntitiesAsMap(); - for (PolarisGrantRecord record : grantList.getGrantRecords()) { - PolarisPrivilege privilege = PolarisPrivilege.fromCode(record.getPrivilegeCode()); - PolarisBaseEntity baseEntity = - this.getOrLoadEntity(entityMap, record.getSecurableCatalogId(), record.getSecurableId()); - if (baseEntity != null) { - switch (baseEntity.getType()) { - case CATALOG: - { - CatalogGrant grant = - new CatalogGrant( - CatalogPrivilege.valueOf(privilege.toString()), - GrantResource.TypeEnum.CATALOG); - catalogGrants.add(grant); - break; - } - case NAMESPACE: - { - NamespaceGrant grant = - new NamespaceGrant( - List.of(NamespaceEntity.of(baseEntity).asNamespace().levels()), - NamespacePrivilege.valueOf(privilege.toString()), - GrantResource.TypeEnum.NAMESPACE); - namespaceGrants.add(grant); - break; - } - case TABLE_LIKE: - { - if (baseEntity.getSubType() == PolarisEntitySubType.TABLE) { - TableIdentifier identifier = TableLikeEntity.of(baseEntity).getTableIdentifier(); - TableGrant grant = - new TableGrant( - List.of(identifier.namespace().levels()), - identifier.name(), - TablePrivilege.valueOf(privilege.toString()), - GrantResource.TypeEnum.TABLE); - tableGrants.add(grant); - } else { - TableIdentifier identifier = TableLikeEntity.of(baseEntity).getTableIdentifier(); - ViewGrant grant = - new ViewGrant( - List.of(identifier.namespace().levels()), - identifier.name(), - ViewPrivilege.valueOf(privilege.toString()), - GrantResource.TypeEnum.VIEW); - viewGrants.add(grant); - } - break; - } - default: - throw new IllegalArgumentException( - String.format( - "Unexpected entity type '%s' listing grants for catalogRole '%s' in catalog '%s'", - baseEntity.getType(), catalogRoleName, catalogName)); - } - } - } - // Assemble these at the end so that they're grouped by type. - List allGrants = new ArrayList<>(); - allGrants.addAll(catalogGrants); - allGrants.addAll(namespaceGrants); - allGrants.addAll(tableGrants); - allGrants.addAll(viewGrants); - return allGrants; - } - - /** - * Get the specified entity from the input map or load it from backend if the input map is null. - * Normally the input map is not expected to be null, except for backward compatibility issue. - * - * @param entitiesMap map of entities - * @param catalogId the id of the catalog of the entity we are looking for - * @param id id of the entity we are looking for - * @return null if the entity does not exist - */ - private @Nullable PolarisBaseEntity getOrLoadEntity( - @Nullable Map entitiesMap, long catalogId, long id) { - return (entitiesMap == null) - ? metaStoreManager.loadEntity(getCurrentPolarisContext(), catalogId, id).getEntity() - : entitiesMap.get(id); - } - - /** Adds a table-level or view-level grant on {@code identifier} to {@code catalogRoleName}. */ - private boolean grantPrivilegeOnTableLikeToRole( - String catalogName, - String catalogRoleName, - TableIdentifier identifier, - PolarisEntitySubType subType, - PolarisPrivilege privilege) { - if (findCatalogByName(catalogName).isEmpty()) { - throw new NotFoundException("Parent catalog %s not found", catalogName); - } - PolarisEntity catalogRoleEntity = - findCatalogRoleByName(catalogName, catalogRoleName) - .orElseThrow(() -> new NotFoundException("CatalogRole %s not found", catalogRoleName)); - - PolarisResolvedPathWrapper resolvedPathWrapper = - resolutionManifest.getResolvedPath(identifier, subType); - if (resolvedPathWrapper == null) { - if (subType == PolarisEntitySubType.VIEW) { - throw new NotFoundException("View %s not found", identifier); - } else { - throw new NotFoundException("Table %s not found", identifier); - } - } - List catalogPath = resolvedPathWrapper.getRawParentPath(); - PolarisEntity tableLikeEntity = resolvedPathWrapper.getRawLeafEntity(); - - return metaStoreManager - .grantPrivilegeOnSecurableToRole( - getCurrentPolarisContext(), - catalogRoleEntity, - PolarisEntity.toCoreList(catalogPath), - tableLikeEntity, - privilege) - .isSuccess(); - } - - /** - * Removes a table-level or view-level grant on {@code identifier} from {@code catalogRoleName}. - */ - private boolean revokePrivilegeOnTableLikeFromRole( - String catalogName, - String catalogRoleName, - TableIdentifier identifier, - PolarisEntitySubType subType, - PolarisPrivilege privilege) { - if (findCatalogByName(catalogName).isEmpty()) { - throw new NotFoundException("Parent catalog %s not found", catalogName); - } - PolarisEntity catalogRoleEntity = - findCatalogRoleByName(catalogName, catalogRoleName) - .orElseThrow(() -> new NotFoundException("CatalogRole %s not found", catalogRoleName)); - - PolarisResolvedPathWrapper resolvedPathWrapper = - resolutionManifest.getResolvedPath(identifier, subType); - if (resolvedPathWrapper == null) { - if (subType == PolarisEntitySubType.VIEW) { - throw new NotFoundException("View %s not found", identifier); - } else { - throw new NotFoundException("Table %s not found", identifier); - } - } - List catalogPath = resolvedPathWrapper.getRawParentPath(); - PolarisEntity tableLikeEntity = resolvedPathWrapper.getRawLeafEntity(); - - return metaStoreManager - .revokePrivilegeOnSecurableFromRole( - getCurrentPolarisContext(), - catalogRoleEntity, - PolarisEntity.toCoreList(catalogPath), - tableLikeEntity, - privilege) - .isSuccess(); - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java deleted file mode 100644 index 5bf286047..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java +++ /dev/null @@ -1,633 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.admin; - -import jakarta.enterprise.context.RequestScoped; -import jakarta.inject.Inject; -import jakarta.ws.rs.core.Response; -import jakarta.ws.rs.core.SecurityContext; -import java.util.List; -import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.exceptions.NotAuthorizedException; -import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.PolarisConfiguration; -import org.apache.polaris.core.admin.model.AddGrantRequest; -import org.apache.polaris.core.admin.model.Catalog; -import org.apache.polaris.core.admin.model.CatalogGrant; -import org.apache.polaris.core.admin.model.CatalogRole; -import org.apache.polaris.core.admin.model.CatalogRoles; -import org.apache.polaris.core.admin.model.Catalogs; -import org.apache.polaris.core.admin.model.CreateCatalogRequest; -import org.apache.polaris.core.admin.model.CreateCatalogRoleRequest; -import org.apache.polaris.core.admin.model.CreatePrincipalRequest; -import org.apache.polaris.core.admin.model.CreatePrincipalRoleRequest; -import org.apache.polaris.core.admin.model.GrantCatalogRoleRequest; -import org.apache.polaris.core.admin.model.GrantPrincipalRoleRequest; -import org.apache.polaris.core.admin.model.GrantResource; -import org.apache.polaris.core.admin.model.GrantResources; -import org.apache.polaris.core.admin.model.NamespaceGrant; -import org.apache.polaris.core.admin.model.Principal; -import org.apache.polaris.core.admin.model.PrincipalRole; -import org.apache.polaris.core.admin.model.PrincipalRoles; -import org.apache.polaris.core.admin.model.PrincipalWithCredentials; -import org.apache.polaris.core.admin.model.Principals; -import org.apache.polaris.core.admin.model.RevokeGrantRequest; -import org.apache.polaris.core.admin.model.StorageConfigInfo; -import org.apache.polaris.core.admin.model.TableGrant; -import org.apache.polaris.core.admin.model.UpdateCatalogRequest; -import org.apache.polaris.core.admin.model.UpdateCatalogRoleRequest; -import org.apache.polaris.core.admin.model.UpdatePrincipalRequest; -import org.apache.polaris.core.admin.model.UpdatePrincipalRoleRequest; -import org.apache.polaris.core.admin.model.ViewGrant; -import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; -import org.apache.polaris.core.auth.PolarisAuthorizer; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.entity.CatalogEntity; -import org.apache.polaris.core.entity.CatalogRoleEntity; -import org.apache.polaris.core.entity.PolarisPrivilege; -import org.apache.polaris.core.entity.PrincipalEntity; -import org.apache.polaris.core.entity.PrincipalRoleEntity; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.core.persistence.PolarisEntityManager; -import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.service.admin.api.PolarisCatalogsApiService; -import org.apache.polaris.service.admin.api.PolarisPrincipalRolesApiService; -import org.apache.polaris.service.admin.api.PolarisPrincipalsApiService; -import org.apache.polaris.service.config.RealmEntityManagerFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** Concrete implementation of the Polaris API services */ -@RequestScoped -public class PolarisServiceImpl - implements PolarisCatalogsApiService, - PolarisPrincipalsApiService, - PolarisPrincipalRolesApiService { - private static final Logger LOGGER = LoggerFactory.getLogger(PolarisServiceImpl.class); - private final RealmEntityManagerFactory entityManagerFactory; - private final PolarisAuthorizer polarisAuthorizer; - private final MetaStoreManagerFactory metaStoreManagerFactory; - - @Inject - public PolarisServiceImpl( - RealmEntityManagerFactory entityManagerFactory, - MetaStoreManagerFactory metaStoreManagerFactory, - PolarisAuthorizer polarisAuthorizer) { - this.entityManagerFactory = entityManagerFactory; - this.metaStoreManagerFactory = metaStoreManagerFactory; - this.polarisAuthorizer = polarisAuthorizer; - } - - private PolarisAdminService newAdminService(SecurityContext securityContext) { - CallContext callContext = CallContext.getCurrentContext(); - AuthenticatedPolarisPrincipal authenticatedPrincipal = - (AuthenticatedPolarisPrincipal) securityContext.getUserPrincipal(); - if (authenticatedPrincipal == null) { - throw new NotAuthorizedException("Failed to find authenticatedPrincipal in SecurityContext"); - } - - PolarisEntityManager entityManager = - entityManagerFactory.getOrCreateEntityManager(callContext.getRealmContext()); - PolarisMetaStoreManager metaStoreManager = - metaStoreManagerFactory.getOrCreateMetaStoreManager(callContext.getRealmContext()); - return new PolarisAdminService( - callContext, entityManager, metaStoreManager, authenticatedPrincipal, polarisAuthorizer); - } - - /** From PolarisCatalogsApiService */ - @Override - public Response createCatalog(CreateCatalogRequest request, SecurityContext securityContext) { - PolarisAdminService adminService = newAdminService(securityContext); - Catalog catalog = request.getCatalog(); - validateStorageConfig(catalog.getStorageConfigInfo()); - Catalog newCatalog = - new CatalogEntity(adminService.createCatalog(CatalogEntity.fromCatalog(catalog))) - .asCatalog(); - LOGGER.info("Created new catalog {}", newCatalog); - return Response.status(Response.Status.CREATED).build(); - } - - private void validateStorageConfig(StorageConfigInfo storageConfigInfo) { - CallContext callContext = CallContext.getCurrentContext(); - PolarisCallContext polarisCallContext = callContext.getPolarisCallContext(); - List allowedStorageTypes = - polarisCallContext - .getConfigurationStore() - .getConfiguration( - polarisCallContext, PolarisConfiguration.SUPPORTED_CATALOG_STORAGE_TYPES); - if (!allowedStorageTypes.contains(storageConfigInfo.getStorageType().name())) { - LOGGER - .atWarn() - .addKeyValue("storageConfig", storageConfigInfo) - .log("Disallowed storage type in catalog"); - throw new IllegalArgumentException( - "Unsupported storage type: " + storageConfigInfo.getStorageType()); - } - } - - /** From PolarisCatalogsApiService */ - @Override - public Response deleteCatalog(String catalogName, SecurityContext securityContext) { - PolarisAdminService adminService = newAdminService(securityContext); - adminService.deleteCatalog(catalogName); - return Response.status(Response.Status.NO_CONTENT).build(); - } - - /** From PolarisCatalogsApiService */ - @Override - public Response getCatalog(String catalogName, SecurityContext securityContext) { - PolarisAdminService adminService = newAdminService(securityContext); - return Response.ok(adminService.getCatalog(catalogName).asCatalog()).build(); - } - - /** From PolarisCatalogsApiService */ - @Override - public Response updateCatalog( - String catalogName, UpdateCatalogRequest updateRequest, SecurityContext securityContext) { - PolarisAdminService adminService = newAdminService(securityContext); - if (updateRequest.getStorageConfigInfo() != null) { - validateStorageConfig(updateRequest.getStorageConfigInfo()); - } - return Response.ok(adminService.updateCatalog(catalogName, updateRequest).asCatalog()).build(); - } - - /** From PolarisCatalogsApiService */ - @Override - public Response listCatalogs(SecurityContext securityContext) { - PolarisAdminService adminService = newAdminService(securityContext); - List catalogList = - adminService.listCatalogs().stream() - .map(CatalogEntity::new) - .map(CatalogEntity::asCatalog) - .toList(); - Catalogs catalogs = new Catalogs(catalogList); - LOGGER.debug("listCatalogs returning: {}", catalogs); - return Response.ok(catalogs).build(); - } - - /** From PolarisPrincipalsApiService */ - @Override - public Response createPrincipal(CreatePrincipalRequest request, SecurityContext securityContext) { - PolarisAdminService adminService = newAdminService(securityContext); - PrincipalEntity principal = PrincipalEntity.fromPrincipal(request.getPrincipal()); - if (Boolean.TRUE.equals(request.getCredentialRotationRequired())) { - principal = - new PrincipalEntity.Builder(principal).setCredentialRotationRequiredState().build(); - } - PrincipalWithCredentials createdPrincipal = adminService.createPrincipal(principal); - LOGGER.info("Created new principal {}", createdPrincipal); - return Response.status(Response.Status.CREATED).entity(createdPrincipal).build(); - } - - /** From PolarisPrincipalsApiService */ - @Override - public Response deletePrincipal(String principalName, SecurityContext securityContext) { - PolarisAdminService adminService = newAdminService(securityContext); - adminService.deletePrincipal(principalName); - return Response.status(Response.Status.NO_CONTENT).build(); - } - - /** From PolarisPrincipalsApiService */ - @Override - public Response getPrincipal(String principalName, SecurityContext securityContext) { - PolarisAdminService adminService = newAdminService(securityContext); - return Response.ok(adminService.getPrincipal(principalName).asPrincipal()).build(); - } - - /** From PolarisPrincipalsApiService */ - @Override - public Response updatePrincipal( - String principalName, UpdatePrincipalRequest updateRequest, SecurityContext securityContext) { - PolarisAdminService adminService = newAdminService(securityContext); - return Response.ok(adminService.updatePrincipal(principalName, updateRequest).asPrincipal()) - .build(); - } - - /** From PolarisPrincipalsApiService */ - @Override - public Response rotateCredentials(String principalName, SecurityContext securityContext) { - PolarisAdminService adminService = newAdminService(securityContext); - return Response.ok(adminService.rotateCredentials(principalName)).build(); - } - - /** From PolarisPrincipalsApiService */ - @Override - public Response listPrincipals(SecurityContext securityContext) { - PolarisAdminService adminService = newAdminService(securityContext); - List principalList = - adminService.listPrincipals().stream() - .map(PrincipalEntity::new) - .map(PrincipalEntity::asPrincipal) - .toList(); - Principals principals = new Principals(principalList); - LOGGER.debug("listPrincipals returning: {}", principals); - return Response.ok(principals).build(); - } - - /** From PolarisPrincipalRolesApiService */ - @Override - public Response createPrincipalRole( - CreatePrincipalRoleRequest request, SecurityContext securityContext) { - PolarisAdminService adminService = newAdminService(securityContext); - PrincipalRole newPrincipalRole = - new PrincipalRoleEntity( - adminService.createPrincipalRole( - PrincipalRoleEntity.fromPrincipalRole(request.getPrincipalRole()))) - .asPrincipalRole(); - LOGGER.info("Created new principalRole {}", newPrincipalRole); - return Response.status(Response.Status.CREATED).build(); - } - - /** From PolarisPrincipalRolesApiService */ - @Override - public Response deletePrincipalRole(String principalRoleName, SecurityContext securityContext) { - PolarisAdminService adminService = newAdminService(securityContext); - adminService.deletePrincipalRole(principalRoleName); - return Response.status(Response.Status.NO_CONTENT).build(); - } - - /** From PolarisPrincipalRolesApiService */ - @Override - public Response getPrincipalRole(String principalRoleName, SecurityContext securityContext) { - PolarisAdminService adminService = newAdminService(securityContext); - return Response.ok(adminService.getPrincipalRole(principalRoleName).asPrincipalRole()).build(); - } - - /** From PolarisPrincipalRolesApiService */ - @Override - public Response updatePrincipalRole( - String principalRoleName, - UpdatePrincipalRoleRequest updateRequest, - SecurityContext securityContext) { - PolarisAdminService adminService = newAdminService(securityContext); - return Response.ok( - adminService.updatePrincipalRole(principalRoleName, updateRequest).asPrincipalRole()) - .build(); - } - - /** From PolarisPrincipalRolesApiService */ - @Override - public Response listPrincipalRoles(SecurityContext securityContext) { - PolarisAdminService adminService = newAdminService(securityContext); - List principalRoleList = - adminService.listPrincipalRoles().stream() - .map(PrincipalRoleEntity::new) - .map(PrincipalRoleEntity::asPrincipalRole) - .toList(); - PrincipalRoles principalRoles = new PrincipalRoles(principalRoleList); - LOGGER.debug("listPrincipalRoles returning: {}", principalRoles); - return Response.ok(principalRoles).build(); - } - - /** From PolarisCatalogsApiService */ - @Override - public Response createCatalogRole( - String catalogName, CreateCatalogRoleRequest request, SecurityContext securityContext) { - PolarisAdminService adminService = newAdminService(securityContext); - CatalogRole newCatalogRole = - new CatalogRoleEntity( - adminService.createCatalogRole( - catalogName, CatalogRoleEntity.fromCatalogRole(request.getCatalogRole()))) - .asCatalogRole(); - LOGGER.info("Created new catalogRole {}", newCatalogRole); - return Response.status(Response.Status.CREATED).build(); - } - - /** From PolarisCatalogsApiService */ - @Override - public Response deleteCatalogRole( - String catalogName, String catalogRoleName, SecurityContext securityContext) { - PolarisAdminService adminService = newAdminService(securityContext); - adminService.deleteCatalogRole(catalogName, catalogRoleName); - return Response.status(Response.Status.NO_CONTENT).build(); - } - - /** From PolarisCatalogsApiService */ - @Override - public Response getCatalogRole( - String catalogName, String catalogRoleName, SecurityContext securityContext) { - PolarisAdminService adminService = newAdminService(securityContext); - return Response.ok(adminService.getCatalogRole(catalogName, catalogRoleName).asCatalogRole()) - .build(); - } - - /** From PolarisCatalogsApiService */ - @Override - public Response updateCatalogRole( - String catalogName, - String catalogRoleName, - UpdateCatalogRoleRequest updateRequest, - SecurityContext securityContext) { - PolarisAdminService adminService = newAdminService(securityContext); - return Response.ok( - adminService - .updateCatalogRole(catalogName, catalogRoleName, updateRequest) - .asCatalogRole()) - .build(); - } - - /** From PolarisCatalogsApiService */ - @Override - public Response listCatalogRoles(String catalogName, SecurityContext securityContext) { - PolarisAdminService adminService = newAdminService(securityContext); - List catalogRoleList = - adminService.listCatalogRoles(catalogName).stream() - .map(CatalogRoleEntity::new) - .map(CatalogRoleEntity::asCatalogRole) - .toList(); - CatalogRoles catalogRoles = new CatalogRoles(catalogRoleList); - LOGGER.debug("listCatalogRoles returning: {}", catalogRoles); - return Response.ok(catalogRoles).build(); - } - - /** From PolarisPrincipalsApiService */ - @Override - public Response assignPrincipalRole( - String principalName, GrantPrincipalRoleRequest request, SecurityContext securityContext) { - LOGGER.info( - "Assigning principalRole {} to principal {}", - request.getPrincipalRole().getName(), - principalName); - PolarisAdminService adminService = newAdminService(securityContext); - adminService.assignPrincipalRole(principalName, request.getPrincipalRole().getName()); - return Response.status(Response.Status.CREATED).build(); - } - - /** From PolarisPrincipalsApiService */ - @Override - public Response revokePrincipalRole( - String principalName, String principalRoleName, SecurityContext securityContext) { - LOGGER.info("Revoking principalRole {} from principal {}", principalRoleName, principalName); - PolarisAdminService adminService = newAdminService(securityContext); - adminService.revokePrincipalRole(principalName, principalRoleName); - return Response.status(Response.Status.NO_CONTENT).build(); - } - - /** From PolarisPrincipalsApiService */ - @Override - public Response listPrincipalRolesAssigned( - String principalName, SecurityContext securityContext) { - PolarisAdminService adminService = newAdminService(securityContext); - List principalRoleList = - adminService.listPrincipalRolesAssigned(principalName).stream() - .map(PrincipalRoleEntity::new) - .map(PrincipalRoleEntity::asPrincipalRole) - .toList(); - PrincipalRoles principalRoles = new PrincipalRoles(principalRoleList); - LOGGER.debug("listPrincipalRolesAssigned returning: {}", principalRoles); - return Response.ok(principalRoles).build(); - } - - /** From PolarisPrincipalRolesApiService */ - @Override - public Response assignCatalogRoleToPrincipalRole( - String principalRoleName, - String catalogName, - GrantCatalogRoleRequest request, - SecurityContext securityContext) { - LOGGER.info( - "Assigning catalogRole {} in catalog {} to principalRole {}", - request.getCatalogRole().getName(), - catalogName, - principalRoleName); - PolarisAdminService adminService = newAdminService(securityContext); - adminService.assignCatalogRoleToPrincipalRole( - principalRoleName, catalogName, request.getCatalogRole().getName()); - return Response.status(Response.Status.CREATED).build(); - } - - /** From PolarisPrincipalRolesApiService */ - @Override - public Response revokeCatalogRoleFromPrincipalRole( - String principalRoleName, - String catalogName, - String catalogRoleName, - SecurityContext securityContext) { - LOGGER.info( - "Revoking catalogRole {} in catalog {} from principalRole {}", - catalogRoleName, - catalogName, - principalRoleName); - PolarisAdminService adminService = newAdminService(securityContext); - adminService.revokeCatalogRoleFromPrincipalRole( - principalRoleName, catalogName, catalogRoleName); - return Response.status(Response.Status.NO_CONTENT).build(); - } - - /** From PolarisPrincipalRolesApiService */ - @Override - public Response listAssigneePrincipalsForPrincipalRole( - String principalRoleName, SecurityContext securityContext) { - PolarisAdminService adminService = newAdminService(securityContext); - List principalList = - adminService.listAssigneePrincipalsForPrincipalRole(principalRoleName).stream() - .map(PrincipalEntity::new) - .map(PrincipalEntity::asPrincipal) - .toList(); - Principals principals = new Principals(principalList); - LOGGER.debug("listAssigneePrincipalsForPrincipalRole returning: {}", principals); - return Response.ok(principals).build(); - } - - /** From PolarisPrincipalRolesApiService */ - @Override - public Response listCatalogRolesForPrincipalRole( - String principalRoleName, String catalogName, SecurityContext securityContext) { - PolarisAdminService adminService = newAdminService(securityContext); - List catalogRoleList = - adminService.listCatalogRolesForPrincipalRole(principalRoleName, catalogName).stream() - .map(CatalogRoleEntity::new) - .map(CatalogRoleEntity::asCatalogRole) - .toList(); - CatalogRoles catalogRoles = new CatalogRoles(catalogRoleList); - LOGGER.debug("listCatalogRolesForPrincipalRole returning: {}", catalogRoles); - return Response.ok(catalogRoles).build(); - } - - /** From PolarisCatalogsApiService */ - @Override - public Response addGrantToCatalogRole( - String catalogName, - String catalogRoleName, - AddGrantRequest grantRequest, - SecurityContext securityContext) { - LOGGER.info( - "Adding grant {} to catalogRole {} in catalog {}", - grantRequest, - catalogRoleName, - catalogName); - PolarisAdminService adminService = newAdminService(securityContext); - switch (grantRequest.getGrant()) { - // The per-securable-type Privilege enums must be exact String match for a subset of all - // PolarisPrivilege values. - case ViewGrant viewGrant: - { - PolarisPrivilege privilege = - PolarisPrivilege.valueOf(viewGrant.getPrivilege().toString()); - String viewName = viewGrant.getViewName(); - String[] namespaceParts = viewGrant.getNamespace().toArray(new String[0]); - adminService.grantPrivilegeOnViewToRole( - catalogName, - catalogRoleName, - TableIdentifier.of(Namespace.of(namespaceParts), viewName), - privilege); - break; - } - case TableGrant tableGrant: - { - PolarisPrivilege privilege = - PolarisPrivilege.valueOf(tableGrant.getPrivilege().toString()); - String tableName = tableGrant.getTableName(); - String[] namespaceParts = tableGrant.getNamespace().toArray(new String[0]); - adminService.grantPrivilegeOnTableToRole( - catalogName, - catalogRoleName, - TableIdentifier.of(Namespace.of(namespaceParts), tableName), - privilege); - break; - } - case NamespaceGrant namespaceGrant: - { - PolarisPrivilege privilege = - PolarisPrivilege.valueOf(namespaceGrant.getPrivilege().toString()); - String[] namespaceParts = namespaceGrant.getNamespace().toArray(new String[0]); - adminService.grantPrivilegeOnNamespaceToRole( - catalogName, catalogRoleName, Namespace.of(namespaceParts), privilege); - break; - } - case CatalogGrant catalogGrant: - { - PolarisPrivilege privilege = - PolarisPrivilege.valueOf(catalogGrant.getPrivilege().toString()); - adminService.grantPrivilegeOnCatalogToRole(catalogName, catalogRoleName, privilege); - break; - } - default: - LOGGER - .atWarn() - .addKeyValue("catalog", catalogName) - .addKeyValue("role", catalogRoleName) - .log("Don't know how to handle privilege grant: {}", grantRequest); - return Response.status(Response.Status.BAD_REQUEST).build(); - } - return Response.status(Response.Status.CREATED).build(); - } - - /** From PolarisCatalogsApiService */ - @Override - public Response revokeGrantFromCatalogRole( - String catalogName, - String catalogRoleName, - Boolean cascade, - RevokeGrantRequest grantRequest, - SecurityContext securityContext) { - LOGGER.info( - "Revoking grant {} from catalogRole {} in catalog {}", - grantRequest, - catalogRoleName, - catalogName); - if (cascade != null && cascade) { - LOGGER.warn("Tried to use unimplemented 'cascade' feature when revoking grants."); - return Response.status(501).build(); // not implemented - } - - PolarisAdminService adminService = newAdminService(securityContext); - switch (grantRequest.getGrant()) { - // The per-securable-type Privilege enums must be exact String match for a subset of all - // PolarisPrivilege values. - case ViewGrant viewGrant: - { - PolarisPrivilege privilege = - PolarisPrivilege.valueOf(viewGrant.getPrivilege().toString()); - String viewName = viewGrant.getViewName(); - String[] namespaceParts = viewGrant.getNamespace().toArray(new String[0]); - adminService.revokePrivilegeOnViewFromRole( - catalogName, - catalogRoleName, - TableIdentifier.of(Namespace.of(namespaceParts), viewName), - privilege); - break; - } - case TableGrant tableGrant: - { - PolarisPrivilege privilege = - PolarisPrivilege.valueOf(tableGrant.getPrivilege().toString()); - String tableName = tableGrant.getTableName(); - String[] namespaceParts = tableGrant.getNamespace().toArray(new String[0]); - adminService.revokePrivilegeOnTableFromRole( - catalogName, - catalogRoleName, - TableIdentifier.of(Namespace.of(namespaceParts), tableName), - privilege); - break; - } - case NamespaceGrant namespaceGrant: - { - PolarisPrivilege privilege = - PolarisPrivilege.valueOf(namespaceGrant.getPrivilege().toString()); - String[] namespaceParts = namespaceGrant.getNamespace().toArray(new String[0]); - adminService.revokePrivilegeOnNamespaceFromRole( - catalogName, catalogRoleName, Namespace.of(namespaceParts), privilege); - break; - } - case CatalogGrant catalogGrant: - { - PolarisPrivilege privilege = - PolarisPrivilege.valueOf(catalogGrant.getPrivilege().toString()); - adminService.revokePrivilegeOnCatalogFromRole(catalogName, catalogRoleName, privilege); - break; - } - default: - LOGGER - .atWarn() - .addKeyValue("catalog", catalogName) - .addKeyValue("role", catalogRoleName) - .log("Don't know how to handle privilege revocation: {}", grantRequest); - return Response.status(Response.Status.BAD_REQUEST).build(); - } - return Response.status(Response.Status.CREATED).build(); - } - - /** From PolarisCatalogsApiService */ - @Override - public Response listAssigneePrincipalRolesForCatalogRole( - String catalogName, String catalogRoleName, SecurityContext securityContext) { - PolarisAdminService adminService = newAdminService(securityContext); - List principalRoleList = - adminService.listAssigneePrincipalRolesForCatalogRole(catalogName, catalogRoleName).stream() - .map(PrincipalRoleEntity::new) - .map(PrincipalRoleEntity::asPrincipalRole) - .toList(); - PrincipalRoles principalRoles = new PrincipalRoles(principalRoleList); - LOGGER.debug("listAssigneePrincipalRolesForCatalogRole returning: {}", principalRoles); - return Response.ok(principalRoles).build(); - } - - /** From PolarisCatalogsApiService */ - @Override - public Response listGrantsForCatalogRole( - String catalogName, String catalogRoleName, SecurityContext securityContext) { - PolarisAdminService adminService = newAdminService(securityContext); - List grantList = - adminService.listGrantsForCatalogRole(catalogName, catalogRoleName); - GrantResources grantResources = new GrantResources(grantList); - return Response.ok(grantResources).build(); - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/BasePolarisAuthenticator.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/BasePolarisAuthenticator.java deleted file mode 100644 index ac51a2179..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/BasePolarisAuthenticator.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.auth; - -import java.util.Arrays; -import java.util.HashSet; -import java.util.Optional; -import java.util.Set; -import org.apache.commons.lang3.exception.ExceptionUtils; -import org.apache.iceberg.exceptions.NotAuthorizedException; -import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.core.entity.PolarisEntity; -import org.apache.polaris.core.entity.PolarisEntitySubType; -import org.apache.polaris.core.entity.PolarisEntityType; -import org.apache.polaris.core.entity.PrincipalEntity; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Base implementation of {@link Authenticator} constructs a {@link AuthenticatedPolarisPrincipal} - * from the token parsed by subclasses. The {@link AuthenticatedPolarisPrincipal} is read from the - * {@link PolarisMetaStoreManager} for the current {@link RealmContext}. If the token defines a - * non-empty set of scopes, only the principal roles specified in the scopes will be active for the - * current principal. Only the grants assigned to these roles will be active in the current request. - */ -public abstract class BasePolarisAuthenticator - implements Authenticator { - public static final String PRINCIPAL_ROLE_ALL = "PRINCIPAL_ROLE:ALL"; - public static final String PRINCIPAL_ROLE_PREFIX = "PRINCIPAL_ROLE:"; - private static final Logger LOGGER = LoggerFactory.getLogger(BasePolarisAuthenticator.class); - - protected final MetaStoreManagerFactory metaStoreManagerFactory; - - protected BasePolarisAuthenticator(MetaStoreManagerFactory metaStoreManagerFactory) { - this.metaStoreManagerFactory = metaStoreManagerFactory; - } - - public PolarisCallContext getCurrentPolarisContext() { - return CallContext.getCurrentContext().getPolarisCallContext(); - } - - protected Optional getPrincipal(DecodedToken tokenInfo) { - LOGGER.debug("Resolving principal for tokenInfo client_id={}", tokenInfo.getClientId()); - RealmContext realmContext = CallContext.getCurrentContext().getRealmContext(); - PolarisMetaStoreManager metaStoreManager = - metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext); - PolarisEntity principal; - try { - principal = - tokenInfo.getPrincipalId() > 0 - ? PolarisEntity.of( - metaStoreManager.loadEntity( - getCurrentPolarisContext(), 0L, tokenInfo.getPrincipalId())) - : PolarisEntity.of( - metaStoreManager.readEntityByName( - getCurrentPolarisContext(), - null, - PolarisEntityType.PRINCIPAL, - PolarisEntitySubType.NULL_SUBTYPE, - tokenInfo.getSub())); - } catch (Exception e) { - LOGGER - .atError() - .addKeyValue("errMsg", e.getMessage()) - .addKeyValue("stackTrace", ExceptionUtils.getStackTrace(e)) - .log("Unable to authenticate user with token"); - throw new NotAuthorizedException("Unable to authenticate"); - } - if (principal == null) { - LOGGER.warn( - "Failed to resolve principal from tokenInfo client_id={}", tokenInfo.getClientId()); - throw new NotAuthorizedException("Unable to authenticate"); - } - - Set activatedPrincipalRoles = new HashSet<>(); - // TODO: Consolidate the divergent "scopes" logic between test-bearer-token and token-exchange. - if (tokenInfo.getScope() != null && !tokenInfo.getScope().equals(PRINCIPAL_ROLE_ALL)) { - activatedPrincipalRoles.addAll( - Arrays.stream(tokenInfo.getScope().split(" ")) - .map( - s -> // strip the principal_role prefix, if present - s.startsWith(PRINCIPAL_ROLE_PREFIX) - ? s.substring(PRINCIPAL_ROLE_PREFIX.length()) - : s) - .toList()); - } - - LOGGER.debug("Resolved principal: {}", principal); - - AuthenticatedPolarisPrincipal authenticatedPrincipal = - new AuthenticatedPolarisPrincipal(new PrincipalEntity(principal), activatedPrincipalRoles); - LOGGER.debug("Populating authenticatedPrincipal into CallContext: {}", authenticatedPrincipal); - CallContext.getCurrentContext() - .contextVariables() - .put(CallContext.AUTHENTICATED_PRINCIPAL, authenticatedPrincipal); - return Optional.of(authenticatedPrincipal); - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/DecodedToken.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/DecodedToken.java deleted file mode 100644 index 487173f34..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/DecodedToken.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.auth; - -public interface DecodedToken { - Long getPrincipalId(); - - String getClientId(); - - String getSub(); - - String getScope(); -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/DefaultOAuth2ApiService.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/DefaultOAuth2ApiService.java deleted file mode 100644 index 9bb567c62..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/DefaultOAuth2ApiService.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.auth; - -import static java.nio.charset.StandardCharsets.UTF_8; - -import io.quarkus.arc.lookup.LookupIfProperty; -import jakarta.enterprise.context.RequestScoped; -import jakarta.inject.Inject; -import jakarta.ws.rs.core.Response; -import jakarta.ws.rs.core.SecurityContext; -import org.apache.commons.codec.binary.Base64; -import org.apache.hadoop.hdfs.web.oauth2.OAuth2Constants; -import org.apache.iceberg.rest.auth.OAuth2Properties; -import org.apache.iceberg.rest.responses.OAuthTokenResponse; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.service.catalog.api.IcebergRestOAuth2ApiService; -import org.apache.polaris.service.config.RuntimeCandidate; -import org.apache.polaris.service.types.TokenType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Default implementation of the {@link IcebergRestOAuth2ApiService} that generates a JWT token for - * the client if the client secret matches. - */ -@RequestScoped -@RuntimeCandidate -@LookupIfProperty(name = "polaris.authentication.oauth2-service.type", stringValue = "default") -public class DefaultOAuth2ApiService implements IcebergRestOAuth2ApiService { - private static final Logger LOGGER = LoggerFactory.getLogger(DefaultOAuth2ApiService.class); - private final TokenBrokerFactory tokenBrokerFactory; - private final CallContext callContext; - - @Inject - public DefaultOAuth2ApiService(TokenBrokerFactory tokenBrokerFactory, CallContext callContext) { - this.tokenBrokerFactory = tokenBrokerFactory; - this.callContext = callContext; - CallContext.setCurrentContext(callContext); - } - - @Override - public Response getToken( - String authHeader, - String grantType, - String scope, - String clientId, - String clientSecret, - TokenType requestedTokenType, - String subjectToken, - TokenType subjectTokenType, - String actorToken, - TokenType actorTokenType, - SecurityContext securityContext) { - - TokenBroker tokenBroker = tokenBrokerFactory.apply(callContext.getRealmContext()); - if (!tokenBroker.supportsGrantType(grantType)) { - return OAuthUtils.getResponseFromError(OAuthTokenErrorResponse.Error.unsupported_grant_type); - } - if (!tokenBroker.supportsRequestedTokenType(requestedTokenType)) { - return OAuthUtils.getResponseFromError(OAuthTokenErrorResponse.Error.invalid_request); - } - if (authHeader == null && clientId == null) { - return OAuthUtils.getResponseFromError(OAuthTokenErrorResponse.Error.invalid_client); - } - if (authHeader != null && clientId == null && authHeader.startsWith("Basic ")) { - String credentials = new String(Base64.decodeBase64(authHeader.substring(6)), UTF_8); - if (!credentials.contains(":")) { - return OAuthUtils.getResponseFromError(OAuthTokenErrorResponse.Error.invalid_client); - } - LOGGER.debug("Found credentials in auth header - treating as client_credentials"); - String[] parts = credentials.split(":", 2); - clientId = parts[0]; - clientSecret = parts[1]; - } - TokenResponse tokenResponse = - switch (subjectTokenType) { - case TokenType.ID_TOKEN, - TokenType.REFRESH_TOKEN, - TokenType.JWT, - TokenType.SAML1, - TokenType.SAML2 -> - new TokenResponse(OAuthTokenErrorResponse.Error.invalid_request); - case TokenType.ACCESS_TOKEN -> { - // token exchange with client id and client secret means the client has previously - // attempted to refresh - // an access token, but refreshing was not supported by the token broker. Accept the - // client id and - // secret and treat it as a new token request - if (clientId != null && clientSecret != null) { - yield tokenBroker.generateFromClientSecrets( - clientId, clientSecret, OAuth2Constants.CLIENT_CREDENTIALS, scope); - } else { - yield tokenBroker.generateFromToken(subjectTokenType, subjectToken, grantType, scope); - } - } - case null -> - tokenBroker.generateFromClientSecrets(clientId, clientSecret, grantType, scope); - }; - if (tokenResponse == null) { - return OAuthUtils.getResponseFromError(OAuthTokenErrorResponse.Error.unsupported_grant_type); - } - if (!tokenResponse.isValid()) { - return OAuthUtils.getResponseFromError(tokenResponse.getError()); - } - return Response.ok( - OAuthTokenResponse.builder() - .withToken(tokenResponse.getAccessToken()) - .withTokenType(OAuth2Constants.BEARER) - .withIssuedTokenType(OAuth2Properties.ACCESS_TOKEN_TYPE) - .setExpirationInSeconds(tokenResponse.getExpiresIn()) - .build()) - .build(); - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/DefaultPolarisAuthenticator.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/DefaultPolarisAuthenticator.java deleted file mode 100644 index 60ff49b25..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/DefaultPolarisAuthenticator.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.auth; - -import io.quarkus.arc.lookup.LookupIfProperty; -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.inject.Inject; -import java.util.Optional; -import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.service.config.RuntimeCandidate; - -@ApplicationScoped -@RuntimeCandidate -@LookupIfProperty(name = "polaris.authentication.type", stringValue = "default") -public class DefaultPolarisAuthenticator extends BasePolarisAuthenticator { - - private final TokenBrokerFactory tokenBrokerFactory; - - // Required for CDI - public DefaultPolarisAuthenticator() { - this(null, null); - } - - @Inject - public DefaultPolarisAuthenticator( - MetaStoreManagerFactory metaStoreManagerFactory, TokenBrokerFactory tokenBrokerFactory) { - super(metaStoreManagerFactory); - this.tokenBrokerFactory = tokenBrokerFactory; - } - - @Override - public Optional authenticate(String credentials) { - TokenBroker handler = - tokenBrokerFactory.apply(CallContext.getCurrentContext().getRealmContext()); - DecodedToken decodedToken = handler.verify(credentials); - return getPrincipal(decodedToken); - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/JWTBroker.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/JWTBroker.java deleted file mode 100644 index 3ac173f37..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/JWTBroker.java +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.auth; - -import com.auth0.jwt.JWT; -import com.auth0.jwt.algorithms.Algorithm; -import com.auth0.jwt.exceptions.JWTVerificationException; -import com.auth0.jwt.interfaces.DecodedJWT; -import com.auth0.jwt.interfaces.JWTVerifier; -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.Optional; -import java.util.UUID; -import org.apache.commons.lang3.StringUtils; -import org.apache.iceberg.exceptions.NotAuthorizedException; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.entity.PolarisEntityType; -import org.apache.polaris.core.entity.PrincipalEntity; -import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.service.types.TokenType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** Generates a JWT Token. */ -abstract class JWTBroker implements TokenBroker { - private static final Logger LOGGER = LoggerFactory.getLogger(JWTBroker.class); - - private static final String ISSUER_KEY = "polaris"; - private static final String CLAIM_KEY_ACTIVE = "active"; - private static final String CLAIM_KEY_CLIENT_ID = "client_id"; - private static final String CLAIM_KEY_PRINCIPAL_ID = "principalId"; - private static final String CLAIM_KEY_SCOPE = "scope"; - - private final PolarisMetaStoreManager metaStoreManager; - private final int maxTokenGenerationInSeconds; - - JWTBroker(PolarisMetaStoreManager metaStoreManager, int maxTokenGenerationInSeconds) { - this.metaStoreManager = metaStoreManager; - this.maxTokenGenerationInSeconds = maxTokenGenerationInSeconds; - } - - abstract Algorithm getAlgorithm(); - - @Override - public DecodedToken verify(String token) { - JWTVerifier verifier = JWT.require(getAlgorithm()).withClaim(CLAIM_KEY_ACTIVE, true).build(); - - try { - DecodedJWT decodedJWT = verifier.verify(token); - return new DecodedToken() { - @Override - public Long getPrincipalId() { - return decodedJWT.getClaim("principalId").asLong(); - } - - @Override - public String getClientId() { - return decodedJWT.getClaim("client_id").asString(); - } - - @Override - public String getSub() { - return decodedJWT.getSubject(); - } - - @Override - public String getScope() { - return decodedJWT.getClaim("scope").asString(); - } - }; - - } catch (JWTVerificationException e) { - LOGGER.error("Failed to verify the token with error", e); - throw new NotAuthorizedException("Failed to verify the token"); - } - } - - @Override - public TokenResponse generateFromToken( - TokenType tokenType, String subjectToken, String grantType, String scope) { - if (!TokenType.ACCESS_TOKEN.equals(tokenType)) { - return new TokenResponse(OAuthTokenErrorResponse.Error.invalid_request); - } - if (StringUtils.isBlank(subjectToken)) { - return new TokenResponse(OAuthTokenErrorResponse.Error.invalid_request); - } - DecodedToken decodedToken = verify(subjectToken); - PolarisMetaStoreManager.EntityResult principalLookup = - metaStoreManager.loadEntity( - CallContext.getCurrentContext().getPolarisCallContext(), - 0L, - decodedToken.getPrincipalId()); - if (!principalLookup.isSuccess() - || principalLookup.getEntity().getType() != PolarisEntityType.PRINCIPAL) { - return new TokenResponse(OAuthTokenErrorResponse.Error.unauthorized_client); - } - String tokenString = - generateTokenString( - decodedToken.getClientId(), decodedToken.getScope(), decodedToken.getPrincipalId()); - return new TokenResponse( - tokenString, TokenType.ACCESS_TOKEN.getValue(), maxTokenGenerationInSeconds); - } - - @Override - public TokenResponse generateFromClientSecrets( - String clientId, String clientSecret, String grantType, String scope) { - // Initial sanity checks - TokenRequestValidator validator = new TokenRequestValidator(); - Optional initialValidationResponse = - validator.validateForClientCredentialsFlow(clientId, clientSecret, grantType, scope); - if (initialValidationResponse.isPresent()) { - return new TokenResponse(initialValidationResponse.get()); - } - - Optional principal = - TokenBroker.findPrincipalEntity(metaStoreManager, clientId, clientSecret); - if (principal.isEmpty()) { - return new TokenResponse(OAuthTokenErrorResponse.Error.unauthorized_client); - } - String tokenString = generateTokenString(clientId, scope, principal.get().getId()); - return new TokenResponse( - tokenString, TokenType.ACCESS_TOKEN.getValue(), maxTokenGenerationInSeconds); - } - - private String generateTokenString(String clientId, String scope, Long principalId) { - Instant now = Instant.now(); - return JWT.create() - .withIssuer(ISSUER_KEY) - .withSubject(String.valueOf(principalId)) - .withIssuedAt(now) - .withExpiresAt(now.plus(maxTokenGenerationInSeconds, ChronoUnit.SECONDS)) - .withJWTId(UUID.randomUUID().toString()) - .withClaim(CLAIM_KEY_ACTIVE, true) - .withClaim(CLAIM_KEY_CLIENT_ID, clientId) - .withClaim(CLAIM_KEY_PRINCIPAL_ID, principalId) - .withClaim(CLAIM_KEY_SCOPE, scopes(scope)) - .sign(getAlgorithm()); - } - - @Override - public boolean supportsGrantType(String grantType) { - return TokenRequestValidator.ALLOWED_GRANT_TYPES.contains(grantType); - } - - @Override - public boolean supportsRequestedTokenType(TokenType tokenType) { - return tokenType == null || TokenType.ACCESS_TOKEN.equals(tokenType); - } - - private String scopes(String scope) { - return StringUtils.isNotBlank(scope) ? scope : BasePolarisAuthenticator.PRINCIPAL_ROLE_ALL; - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/JWTRSAKeyPair.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/JWTRSAKeyPair.java deleted file mode 100644 index 43b886f86..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/JWTRSAKeyPair.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.auth; - -import com.auth0.jwt.algorithms.Algorithm; -import java.security.interfaces.RSAPrivateKey; -import java.security.interfaces.RSAPublicKey; -import org.apache.polaris.core.persistence.PolarisMetaStoreManager; - -/** Generates a JWT using a Public/Private RSA Key */ -public class JWTRSAKeyPair extends JWTBroker { - - JWTRSAKeyPair(PolarisMetaStoreManager metaStoreManager, int maxTokenGenerationInSeconds) { - super(metaStoreManager, maxTokenGenerationInSeconds); - } - - KeyProvider getKeyProvider() { - return new LocalRSAKeyProvider(); - } - - @Override - Algorithm getAlgorithm() { - KeyProvider keyProvider = getKeyProvider(); - return Algorithm.RSA256( - (RSAPublicKey) keyProvider.getPublicKey(), (RSAPrivateKey) keyProvider.getPrivateKey()); - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/JWTRSAKeyPairFactory.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/JWTRSAKeyPairFactory.java deleted file mode 100644 index 96df25313..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/JWTRSAKeyPairFactory.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.auth; - -import io.quarkus.arc.lookup.LookupIfProperty; -import jakarta.enterprise.context.RequestScoped; -import java.time.Duration; -import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.service.config.RuntimeCandidate; -import org.eclipse.microprofile.config.inject.ConfigProperty; - -@RequestScoped -@RuntimeCandidate -@LookupIfProperty( - name = "polaris.authentication.token-broker-factory.type", - stringValue = "rsa-key-pair") -public class JWTRSAKeyPairFactory implements TokenBrokerFactory { - - private final MetaStoreManagerFactory metaStoreManagerFactory; - private final Duration maxTokenGenerationInSeconds; - - public JWTRSAKeyPairFactory( - MetaStoreManagerFactory metaStoreManagerFactory, - @ConfigProperty(name = "polaris.authentication.token-broker-factory.max-token-generation") - Duration maxTokenGenerationInSeconds) { - this.metaStoreManagerFactory = metaStoreManagerFactory; - this.maxTokenGenerationInSeconds = maxTokenGenerationInSeconds; - } - - @Override - public TokenBroker apply(RealmContext realmContext) { - return new JWTRSAKeyPair( - metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext), - (int) maxTokenGenerationInSeconds.toSeconds()); - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/JWTSymmetricKeyBroker.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/JWTSymmetricKeyBroker.java deleted file mode 100644 index 9189b8b7e..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/JWTSymmetricKeyBroker.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.auth; - -import com.auth0.jwt.algorithms.Algorithm; -import java.util.function.Supplier; -import org.apache.polaris.core.persistence.PolarisMetaStoreManager; - -/** Generates a JWT using a Symmetric Key. */ -public class JWTSymmetricKeyBroker extends JWTBroker { - private final Supplier secretSupplier; - - JWTSymmetricKeyBroker( - PolarisMetaStoreManager metaStoreManager, - int maxTokenGenerationInSeconds, - Supplier secretSupplier) { - super(metaStoreManager, maxTokenGenerationInSeconds); - this.secretSupplier = secretSupplier; - } - - @Override - Algorithm getAlgorithm() { - return Algorithm.HMAC256(secretSupplier.get()); - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/JWTSymmetricKeyFactory.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/JWTSymmetricKeyFactory.java deleted file mode 100644 index 4d09e803f..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/JWTSymmetricKeyFactory.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.auth; - -import io.quarkus.arc.lookup.LookupIfProperty; -import jakarta.enterprise.context.RequestScoped; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.Duration; -import java.util.Optional; -import java.util.function.Supplier; -import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.service.config.RuntimeCandidate; -import org.eclipse.microprofile.config.inject.ConfigProperty; - -@RequestScoped -@RuntimeCandidate -@LookupIfProperty( - name = "polaris.authentication.token-broker-factory.type", - stringValue = "symmetric-key") -public class JWTSymmetricKeyFactory implements TokenBrokerFactory { - - private MetaStoreManagerFactory metaStoreManagerFactory; - private final Duration maxTokenGenerationInSeconds; - private final Path file; - private final String secret; - - public JWTSymmetricKeyFactory( - MetaStoreManagerFactory metaStoreManagerFactory, - @ConfigProperty(name = "polaris.authentication.token-broker-factory.max-token-generation") - Duration maxTokenGenerationInSeconds, - @ConfigProperty(name = "polaris.authentication.token-broker-factory.symmetric-key.secret") - Optional secret, - @ConfigProperty(name = "polaris.authentication.token-broker-factory.symmetric-key.file") - Optional file) { - this.metaStoreManagerFactory = metaStoreManagerFactory; - this.maxTokenGenerationInSeconds = maxTokenGenerationInSeconds; - this.secret = secret.orElse(null); - this.file = file.orElse(null); - if (this.file == null && this.secret == null) { - throw new IllegalStateException("Either file or secret must be set"); - } - } - - @Override - public TokenBroker apply(RealmContext realmContext) { - if (file == null && secret == null) { - throw new IllegalStateException("Either file or secret must be set"); - } - Supplier secretSupplier = secret != null ? () -> secret : readSecretFromDisk(); - return new JWTSymmetricKeyBroker( - metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext), - (int) maxTokenGenerationInSeconds.toSeconds(), - secretSupplier); - } - - private Supplier readSecretFromDisk() { - return () -> { - try { - return Files.readString(file); - } catch (IOException e) { - throw new RuntimeException("Failed to read secret from file: " + file, e); - } - }; - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/KeyProvider.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/KeyProvider.java deleted file mode 100644 index a28b9a059..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/KeyProvider.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.auth; - -import java.security.PrivateKey; -import java.security.PublicKey; - -public interface KeyProvider { - PublicKey getPublicKey(); - - PrivateKey getPrivateKey(); -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/LocalRSAKeyProvider.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/LocalRSAKeyProvider.java deleted file mode 100644 index e6d5acaf7..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/LocalRSAKeyProvider.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.auth; - -import io.quarkus.arc.lookup.LookupIfProperty; -import jakarta.enterprise.context.RequestScoped; -import java.io.IOException; -import java.security.PrivateKey; -import java.security.PublicKey; -import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.service.config.RuntimeCandidate; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Class that can load public / private keys stored on localhost. Meant to be a simple - * implementation for now where a PEM file is loaded off disk. - */ -@RequestScoped -@RuntimeCandidate -@LookupIfProperty(name = "polaris.authentication.key-provider.type", stringValue = "local-rsa") -public class LocalRSAKeyProvider implements KeyProvider { - - private static final String LOCAL_PRIVATE_KEY_LOCATION_KEY = "LOCAL_PRIVATE_KEY_LOCATION_KEY"; - private static final String LOCAL_PUBLIC_KEY_LOCATION_KEY = "LOCAL_PUBLIC_LOCATION_KEY"; - - private static final Logger LOGGER = LoggerFactory.getLogger(LocalRSAKeyProvider.class); - - private String getLocation(String configKey) { - CallContext callContext = CallContext.getCurrentContext(); - PolarisCallContext pCtx = callContext.getPolarisCallContext(); - String fileLocation = pCtx.getConfigurationStore().getConfiguration(pCtx, configKey); - if (fileLocation == null) { - throw new RuntimeException("Cannot find location for key " + configKey); - } - return fileLocation; - } - - /** - * Getter for the Public Key instance - * - * @return the Public Key instance - */ - @Override - public PublicKey getPublicKey() { - final String publicKeyFileLocation = getLocation(LOCAL_PUBLIC_KEY_LOCATION_KEY); - try { - return PemUtils.readPublicKeyFromFile(publicKeyFileLocation, "RSA"); - } catch (IOException e) { - LOGGER.error("Unable to read public key from file {}", publicKeyFileLocation, e); - throw new RuntimeException("Unable to read public key from file " + publicKeyFileLocation, e); - } - } - - /** - * Getter for the Private Key instance. Used to sign the content on the JWT signing stage. - * - * @return the Private Key instance - */ - @Override - public PrivateKey getPrivateKey() { - final String privateKeyFileLocation = getLocation(LOCAL_PRIVATE_KEY_LOCATION_KEY); - try { - return PemUtils.readPrivateKeyFromFile(privateKeyFileLocation, "RSA"); - } catch (IOException e) { - LOGGER.error("Unable to read private key from file {}", privateKeyFileLocation, e); - throw new RuntimeException( - "Unable to read private key from file " + privateKeyFileLocation, e); - } - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/OAuthTokenErrorResponse.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/OAuthTokenErrorResponse.java deleted file mode 100644 index bd8140767..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/OAuthTokenErrorResponse.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.auth; - -import com.fasterxml.jackson.annotation.JsonProperty; - -/** An OAuth Error Token Response as defined by the Iceberg REST API OpenAPI Spec. */ -public class OAuthTokenErrorResponse { - - public enum Error { - invalid_request("The request is invalid"), - invalid_client("The Client is invalid"), - invalid_grant("The grant is invalid"), - unauthorized_client("The client is not authorized"), - unsupported_grant_type("The grant type is invalid"), - invalid_scope("The scope is invalid"), - ; - - final String errorDescription; - - Error(String errorDescription) { - this.errorDescription = errorDescription; - } - - public String getErrorDescription() { - return errorDescription; - } - } - - private final String error; - private final String errorDescription; - private final String errorUri; - - /** Initlaizes a response from one of the supported errors */ - public OAuthTokenErrorResponse(Error error) { - this.error = error.name(); - this.errorDescription = error.getErrorDescription(); - this.errorUri = null; // Not yet used - } - - @JsonProperty("error") - public String getError() { - return error; - } - - @JsonProperty("error_description") - public String getErrorDescription() { - return errorDescription; - } - - @JsonProperty("error_uri") - public String getErrorUri() { - return errorUri; - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/OAuthUtils.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/OAuthUtils.java deleted file mode 100644 index 2b3e928e9..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/OAuthUtils.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.auth; - -import jakarta.ws.rs.core.Response; - -/** Simple utility class to assist with OAuth operations */ -public class OAuthUtils { - public static final String POLARIS_ROLE_PREFIX = "PRINCIPAL_ROLE:"; - - public static Response getResponseFromError(OAuthTokenErrorResponse.Error error) { - return switch (error) { - case unauthorized_client -> - Response.status(Response.Status.UNAUTHORIZED) - .entity( - new OAuthTokenErrorResponse(OAuthTokenErrorResponse.Error.unauthorized_client)) - .build(); - case invalid_client -> - Response.status(Response.Status.BAD_REQUEST) - .entity(new OAuthTokenErrorResponse(OAuthTokenErrorResponse.Error.invalid_client)) - .build(); - case invalid_grant -> - Response.status(Response.Status.BAD_REQUEST) - .entity(new OAuthTokenErrorResponse(OAuthTokenErrorResponse.Error.invalid_grant)) - .build(); - case unsupported_grant_type -> - Response.status(Response.Status.BAD_REQUEST) - .entity( - new OAuthTokenErrorResponse(OAuthTokenErrorResponse.Error.unsupported_grant_type)) - .build(); - case invalid_scope -> - Response.status(Response.Status.BAD_REQUEST) - .entity(new OAuthTokenErrorResponse(OAuthTokenErrorResponse.Error.invalid_scope)) - .build(); - default -> - Response.status(Response.Status.BAD_REQUEST) - .entity(new OAuthTokenErrorResponse(OAuthTokenErrorResponse.Error.invalid_request)) - .build(); - }; - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/PemUtils.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/PemUtils.java deleted file mode 100644 index 24c43a7f0..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/PemUtils.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.auth; - -import static java.nio.charset.StandardCharsets.UTF_8; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileReader; -import java.io.IOException; -import java.security.KeyFactory; -import java.security.NoSuchAlgorithmException; -import java.security.PrivateKey; -import java.security.PublicKey; -import java.security.spec.EncodedKeySpec; -import java.security.spec.InvalidKeySpecException; -import java.security.spec.PKCS8EncodedKeySpec; -import java.security.spec.X509EncodedKeySpec; -import org.bouncycastle.util.io.pem.PemObject; -import org.bouncycastle.util.io.pem.PemReader; - -public class PemUtils { - - private static byte[] parsePEMFile(File pemFile) throws IOException { - if (!pemFile.isFile() || !pemFile.exists()) { - throw new FileNotFoundException( - String.format("The file '%s' doesn't exist.", pemFile.getAbsolutePath())); - } - PemReader reader = new PemReader(new FileReader(pemFile, UTF_8)); - PemObject pemObject = reader.readPemObject(); - byte[] content = pemObject.getContent(); - reader.close(); - return content; - } - - private static PublicKey getPublicKey(byte[] keyBytes, String algorithm) { - PublicKey publicKey = null; - try { - KeyFactory kf = KeyFactory.getInstance(algorithm); - EncodedKeySpec keySpec = new X509EncodedKeySpec(keyBytes); - publicKey = kf.generatePublic(keySpec); - } catch (NoSuchAlgorithmException e) { - System.out.println( - "Could not reconstruct the public key, the given algorithm could not be found."); - } catch (InvalidKeySpecException e) { - System.out.println("Could not reconstruct the public key"); - } - - return publicKey; - } - - private static PrivateKey getPrivateKey(byte[] keyBytes, String algorithm) { - PrivateKey privateKey = null; - try { - KeyFactory kf = KeyFactory.getInstance(algorithm); - EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(keyBytes); - privateKey = kf.generatePrivate(keySpec); - } catch (NoSuchAlgorithmException e) { - System.out.println( - "Could not reconstruct the private key, the given algorithm could not be found."); - } catch (InvalidKeySpecException e) { - System.out.println("Could not reconstruct the private key"); - } - - return privateKey; - } - - public static PublicKey readPublicKeyFromFile(String filepath, String algorithm) - throws IOException { - byte[] bytes = PemUtils.parsePEMFile(new File(filepath)); - return PemUtils.getPublicKey(bytes, algorithm); - } - - public static PrivateKey readPrivateKeyFromFile(String filepath, String algorithm) - throws IOException { - byte[] bytes = PemUtils.parsePEMFile(new File(filepath)); - return PemUtils.getPrivateKey(bytes, algorithm); - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/TestInlineBearerTokenPolarisAuthenticator.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/TestInlineBearerTokenPolarisAuthenticator.java deleted file mode 100644 index 95b798cf1..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/TestInlineBearerTokenPolarisAuthenticator.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.auth; - -import com.google.common.base.Splitter; -import io.quarkus.arc.lookup.LookupIfProperty; -import io.quarkus.security.AuthenticationFailedException; -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.inject.Inject; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; -import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.entity.PolarisPrincipalSecrets; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.service.config.RuntimeCandidate; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Authenticator that parses a token as a sequence of key/value pairs. Specifically, we expect to - * find - * - *

    - *
  • principal - the clientId of the principal - *
  • realm - the current realm - *
- * - * This class does not expect a client to be either present or correct. Lookup is delegated to the - * {@link PolarisMetaStoreManager} for the current realm. - */ -@ApplicationScoped -@RuntimeCandidate -@LookupIfProperty(name = "polaris.authentication.type", stringValue = "test") -public class TestInlineBearerTokenPolarisAuthenticator extends BasePolarisAuthenticator { - private static final Logger LOGGER = - LoggerFactory.getLogger(TestInlineBearerTokenPolarisAuthenticator.class); - - // Required for CDI - public TestInlineBearerTokenPolarisAuthenticator() { - this(null); - } - - @Inject - public TestInlineBearerTokenPolarisAuthenticator( - MetaStoreManagerFactory metaStoreManagerFactory) { - super(metaStoreManagerFactory); - } - - @Override - public Optional authenticate(String credentials) - throws AuthenticationFailedException { - Map properties = extractPrincipal(credentials); - PolarisMetaStoreManager metaStoreManager = - metaStoreManagerFactory.getOrCreateMetaStoreManager( - CallContext.getCurrentContext().getRealmContext()); - PolarisCallContext callContext = CallContext.getCurrentContext().getPolarisCallContext(); - String principal = properties.get("principal"); - - LOGGER.info("Checking for existence of principal {} in map {}", principal, properties); - - TokenInfoExchangeResponse tokenInfo = new TokenInfoExchangeResponse(); - tokenInfo.setSub(principal); - if (properties.get("role") != null) { - tokenInfo.setScope( - Arrays.stream(properties.get("role").split(" ")) - .map(r -> PRINCIPAL_ROLE_PREFIX + r) - .collect(Collectors.joining(" "))); - } - - PolarisPrincipalSecrets secrets = - metaStoreManager.loadPrincipalSecrets(callContext, principal).getPrincipalSecrets(); - if (secrets == null) { - // For test scenarios, if we're allowing short-circuiting into the bearer flow, there may - // not be a clientId/clientSecret, and instead we'll let the BasePolarisAuthenticator - // resolve the principal by name from the persistence store. - LOGGER.warn("Failed to load secrets for principal {}", principal); - } else { - tokenInfo.setIntegrationId(secrets.getPrincipalId()); - } - - return getPrincipal(tokenInfo); - } - - private static Map extractPrincipal(String credentials) { - if (credentials.contains(";") || credentials.contains(":")) { - return new HashMap<>( - Splitter.on(';').trimResults().withKeyValueSeparator(':').split(credentials)); - } - return Map.of(); - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/TestOAuth2ApiService.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/TestOAuth2ApiService.java deleted file mode 100644 index 717e0043c..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/TestOAuth2ApiService.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.auth; - -import io.quarkus.arc.lookup.LookupIfProperty; -import jakarta.enterprise.context.RequestScoped; -import jakarta.inject.Inject; -import jakarta.ws.rs.core.Response; -import jakarta.ws.rs.core.SecurityContext; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; -import org.apache.iceberg.exceptions.NotAuthorizedException; -import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.auth.PolarisSecretsManager.PrincipalSecretsResult; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.entity.PolarisEntitySubType; -import org.apache.polaris.core.entity.PolarisEntityType; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.service.catalog.api.IcebergRestOAuth2ApiService; -import org.apache.polaris.service.config.RuntimeCandidate; -import org.apache.polaris.service.types.TokenType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@RequestScoped -@RuntimeCandidate -@LookupIfProperty(name = "polaris.authentication.oauth2-service.type", stringValue = "test") -public class TestOAuth2ApiService implements IcebergRestOAuth2ApiService { - private static final Logger LOGGER = LoggerFactory.getLogger(TestOAuth2ApiService.class); - - private final MetaStoreManagerFactory metaStoreManagerFactory; - - @Inject - public TestOAuth2ApiService(MetaStoreManagerFactory metaStoreManagerFactory) { - this.metaStoreManagerFactory = metaStoreManagerFactory; - } - - @Override - public Response getToken( - String authHeader, - String grantType, - String scope, - String clientId, - String clientSecret, - TokenType requestedTokenType, - String subjectToken, - TokenType subjectTokenType, - String actorToken, - TokenType actorTokenType, - SecurityContext securityContext) { - Map response = new HashMap<>(); - String principalName = getPrincipalName(clientId); - response.put( - "access_token", - "principal:" - + principalName - + ";password:" - + clientSecret - + ";realm:" - + CallContext.getCurrentContext().getRealmContext().getRealmIdentifier() - + ";role:" - + scope.replaceAll(BasePolarisAuthenticator.PRINCIPAL_ROLE_PREFIX, "")); - response.put("token_type", "bearer"); - response.put("expires_in", 3600); - response.put("scope", Objects.requireNonNullElse(scope, "catalog")); - return Response.ok(response).build(); - } - - private String getPrincipalName(String clientId) { - PolarisMetaStoreManager metaStoreManager = - metaStoreManagerFactory.getOrCreateMetaStoreManager( - CallContext.getCurrentContext().getRealmContext()); - PolarisCallContext polarisCallContext = CallContext.getCurrentContext().getPolarisCallContext(); - PrincipalSecretsResult secretsResult = - metaStoreManager.loadPrincipalSecrets(polarisCallContext, clientId); - if (secretsResult.isSuccess()) { - LOGGER.debug("Found principal secrets for client id {}", clientId); - PolarisMetaStoreManager.EntityResult principalResult = - metaStoreManager.loadEntity( - polarisCallContext, 0L, secretsResult.getPrincipalSecrets().getPrincipalId()); - if (!principalResult.isSuccess()) { - throw new NotAuthorizedException("Failed to load principal entity"); - } - return principalResult.getEntity().getName(); - } else { - LOGGER.debug( - "Unable to find principal secrets for client id {} - trying as principal name", clientId); - PolarisMetaStoreManager.EntityResult principalResult = - metaStoreManager.readEntityByName( - polarisCallContext, - null, - PolarisEntityType.PRINCIPAL, - PolarisEntitySubType.NULL_SUBTYPE, - clientId); - if (!principalResult.isSuccess()) { - throw new NotAuthorizedException("Failed to read principal entity"); - } - return principalResult.getEntity().getName(); - } - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/TokenBroker.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/TokenBroker.java deleted file mode 100644 index f5fea376b..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/TokenBroker.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.auth; - -import jakarta.annotation.Nonnull; -import java.util.Optional; -import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.entity.PolarisEntityType; -import org.apache.polaris.core.entity.PrincipalEntity; -import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.service.types.TokenType; - -/** Generic token class intended to be extended by different token types */ -public interface TokenBroker { - - boolean supportsGrantType(String grantType); - - boolean supportsRequestedTokenType(TokenType tokenType); - - TokenResponse generateFromClientSecrets( - final String clientId, final String clientSecret, final String grantType, final String scope); - - TokenResponse generateFromToken( - TokenType tokenType, String subjectToken, final String grantType, final String scope); - - DecodedToken verify(String token); - - static @Nonnull Optional findPrincipalEntity( - PolarisMetaStoreManager metaStoreManager, String clientId, String clientSecret) { - // Validate the principal is present and secrets match - PolarisCallContext polarisCallContext = CallContext.getCurrentContext().getPolarisCallContext(); - PolarisMetaStoreManager.PrincipalSecretsResult principalSecrets = - metaStoreManager.loadPrincipalSecrets(polarisCallContext, clientId); - if (!principalSecrets.isSuccess()) { - return Optional.empty(); - } - if (!principalSecrets.getPrincipalSecrets().getMainSecret().equals(clientSecret) - && !principalSecrets.getPrincipalSecrets().getSecondarySecret().equals(clientSecret)) { - return Optional.empty(); - } - PolarisMetaStoreManager.EntityResult result = - metaStoreManager.loadEntity( - polarisCallContext, 0L, principalSecrets.getPrincipalSecrets().getPrincipalId()); - if (!result.isSuccess() || result.getEntity().getType() != PolarisEntityType.PRINCIPAL) { - return Optional.empty(); - } - return Optional.of(PrincipalEntity.of(result.getEntity())); - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/TokenBrokerFactory.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/TokenBrokerFactory.java deleted file mode 100644 index 131f3ed64..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/TokenBrokerFactory.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.auth; - -import java.util.function.Function; -import org.apache.polaris.core.context.RealmContext; - -/** - * Factory that creates a {@link TokenBroker} for generating and parsing. The {@link TokenBroker} is - * created based on the realm context. - */ -public interface TokenBrokerFactory extends Function {} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/TokenInfoExchangeResponse.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/TokenInfoExchangeResponse.java deleted file mode 100644 index 8e580c656..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/TokenInfoExchangeResponse.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.auth; - -import com.fasterxml.jackson.annotation.JsonProperty; - -public class TokenInfoExchangeResponse implements DecodedToken { - - private boolean active; - - @JsonProperty("active") - public boolean isActive() { - return active; - } - - @JsonProperty("active") - public void setActive(boolean active) { - this.active = active; - } - - private String scope; - - @JsonProperty("scope") - @Override - public String getScope() { - return scope; - } - - @JsonProperty("scope") - public void setScope(String scope) { - this.scope = scope; - } - - private String clientId; - - @JsonProperty("client_id") - @Override - public String getClientId() { - return clientId; - } - - @JsonProperty("client_id") - public void setClientId(String clientId) { - this.clientId = clientId; - } - - private String tokenType; - - @JsonProperty("token_type") - public String getTokenType() { - return tokenType; - } - - @JsonProperty("token_type") - public void setTokenType(String tokenType) { - this.tokenType = tokenType; - } - - private Long exp; - - @JsonProperty("exp") - public Long getExp() { - return exp; - } - - @JsonProperty("exp") - public void setExp(Long exp) { - this.exp = exp; - } - - private String sub; - - @JsonProperty("sub") - @Override - public String getSub() { - return sub; - } - - @JsonProperty("sub") - public void setSub(String sub) { - this.sub = sub; - } - - private String aud; - - @JsonProperty("aud") - public String getAud() { - return aud; - } - - @JsonProperty("aud") - public void setAud(String aud) { - this.aud = aud; - } - - @JsonProperty("iss") - private String iss; - - @JsonProperty("iss") - public String getIss() { - return iss; - } - - @JsonProperty("iss") - public void setIss(String iss) { - this.iss = iss; - } - - private String token; - - @JsonProperty("token") - public String getToken() { - return token; - } - - @JsonProperty("token") - public void setToken(String token) { - this.token = token; - } - - private long integrationId; - - public long getIntegrationId() { - return integrationId; - } - - @JsonProperty("integration_id") - public void setIntegrationId(long integrationId) { - this.integrationId = integrationId; - } - - /* integration ID is effectively principal ID */ - @Override - public Long getPrincipalId() { - return integrationId; - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/TokenRequestValidator.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/TokenRequestValidator.java deleted file mode 100644 index b123620cb..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/TokenRequestValidator.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.auth; - -import java.util.Optional; -import java.util.Set; -import java.util.logging.Logger; - -public class TokenRequestValidator { - - static final Logger LOGGER = Logger.getLogger(TokenRequestValidator.class.getName()); - - public static final String TOKEN_EXCHANGE = "urn:ietf:params:oauth:grant-type:token-exchange"; - public static final String CLIENT_CREDENTIALS = "client_credentials"; - public static final Set ALLOWED_GRANT_TYPES = Set.of(CLIENT_CREDENTIALS, TOKEN_EXCHANGE); - - /** Default constructor */ - public TokenRequestValidator() {} - - /** - * Validates the incoming Client Credentials flow. - * - *
    - *
  • Non-null scope: while optional in the spec we make it required and expect it to conform - * to the format - *
- * - * @param scope while optional in the Iceberg REST API Spec we make it required and expect it to - * conform to the format "PRINCIPAL_ROLE:NAME PRINCIPAL_ROLE:NAME2 ..." - */ - public Optional validateForClientCredentialsFlow( - final String clientId, - final String clientSecret, - final String grantType, - final String scope) { - if (clientId == null || clientId.isEmpty() || clientSecret == null || clientSecret.isEmpty()) { - // TODO: Figure out how to get the authorization header from `securityContext` - LOGGER.info("Missing Client ID or Client Secret in Request Body"); - return Optional.of(OAuthTokenErrorResponse.Error.invalid_client); - } - if (grantType == null || grantType.isEmpty() || !ALLOWED_GRANT_TYPES.contains(grantType)) { - LOGGER.info("Invalid grant type: " + grantType); - return Optional.of(OAuthTokenErrorResponse.Error.invalid_grant); - } - if (scope == null || scope.isEmpty()) { - LOGGER.info("Missing scope in Request Body"); - return Optional.of(OAuthTokenErrorResponse.Error.invalid_scope); - } - String[] scopes = scope.split(" "); - for (String s : scopes) { - if (!s.startsWith(OAuthUtils.POLARIS_ROLE_PREFIX)) { - LOGGER.info("Invalid scope provided. scopes=" + s + "scopes=" + scope); - return Optional.of(OAuthTokenErrorResponse.Error.invalid_scope); - } - if (s.replaceFirst(OAuthUtils.POLARIS_ROLE_PREFIX, "").isEmpty()) { - LOGGER.info("Invalid scope provided. scopes=" + s + "scopes=" + scope); - return Optional.of(OAuthTokenErrorResponse.Error.invalid_scope); - } - } - return Optional.empty(); - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/TokenResponse.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/TokenResponse.java deleted file mode 100644 index 84d0310a2..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/TokenResponse.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.auth; - -import java.util.Optional; - -public class TokenResponse { - private final Optional error; - private String accessToken; - private String tokenType; - private Integer expiresIn; - - public TokenResponse(OAuthTokenErrorResponse.Error error) { - this.error = Optional.of(error); - } - - public TokenResponse(String accessToken, String tokenType, int expiresIn) { - this.accessToken = accessToken; - this.expiresIn = expiresIn; - this.tokenType = tokenType; - this.error = Optional.empty(); - } - - public boolean isValid() { - return error.isEmpty(); - } - - public OAuthTokenErrorResponse.Error getError() { - return error.get(); - } - - public String getAccessToken() { - return accessToken; - } - - public int getExpiresIn() { - return expiresIn; - } - - public String getTokenType() { - return tokenType; - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/AccessDelegationMode.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/AccessDelegationMode.java deleted file mode 100644 index 46c67622b..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/AccessDelegationMode.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.catalog; - -import com.google.common.base.Functions; -import java.util.Arrays; -import java.util.EnumSet; -import java.util.Locale; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * Represents access mechanisms defined in the Iceberg REST API specification (values for the {@code - * X-Iceberg-Access-Delegation} header). - */ -public enum AccessDelegationMode { - UNKNOWN("unknown"), - VENDED_CREDENTIALS("vended-credentials"), - REMOTE_SIGNING("remote-signing"), - ; - - AccessDelegationMode(String protocolValue) { - this.protocolValue = protocolValue; - } - - private final String protocolValue; - - public String protocolValue() { - return protocolValue; - } - - public static EnumSet fromProtocolValuesList(String protocolValues) { - if (protocolValues == null || protocolValues.isEmpty()) { - return EnumSet.noneOf(AccessDelegationMode.class); - } - - // Backward-compatibility case for old clients that still use the unofficial value of `true` to - // request credential vending. Note that if the client requests `true` among other values it - // will be parsed as `UNKNOWN` (by the code below this `if`) since the client submitting - // multiple access modes is expected to be aware of the Iceberg REST API spec. - if (protocolValues.trim().toLowerCase(Locale.ROOT).equals("true")) { - return EnumSet.of(VENDED_CREDENTIALS); - } - - EnumSet set = EnumSet.noneOf(AccessDelegationMode.class); - Arrays.stream(protocolValues.split(",")) // per Iceberg REST Catalog spec - .map(String::trim) - .map(n -> Mapper.byProtocolValue.getOrDefault(n, UNKNOWN)) - .forEach(set::add); - return set; - } - - private static class Mapper { - private static final Map byProtocolValue = - Arrays.stream(AccessDelegationMode.values()) - .collect(Collectors.toMap(AccessDelegationMode::protocolValue, Functions.identity())); - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java deleted file mode 100644 index 6b6677da0..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ /dev/null @@ -1,2113 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.catalog; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; -import com.google.common.base.Objects; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; -import jakarta.annotation.Nonnull; -import java.io.Closeable; -import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Predicate; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.apache.commons.lang3.exception.ExceptionUtils; -import org.apache.iceberg.BaseMetastoreTableOperations; -import org.apache.iceberg.BaseTable; -import org.apache.iceberg.CatalogProperties; -import org.apache.iceberg.Schema; -import org.apache.iceberg.Table; -import org.apache.iceberg.TableMetadata; -import org.apache.iceberg.TableMetadataParser; -import org.apache.iceberg.TableOperations; -import org.apache.iceberg.aws.s3.S3FileIOProperties; -import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.catalog.SupportsNamespaces; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.exceptions.AlreadyExistsException; -import org.apache.iceberg.exceptions.BadRequestException; -import org.apache.iceberg.exceptions.CommitFailedException; -import org.apache.iceberg.exceptions.ForbiddenException; -import org.apache.iceberg.exceptions.NamespaceNotEmptyException; -import org.apache.iceberg.exceptions.NoSuchNamespaceException; -import org.apache.iceberg.exceptions.NoSuchTableException; -import org.apache.iceberg.exceptions.NoSuchViewException; -import org.apache.iceberg.exceptions.NotFoundException; -import org.apache.iceberg.exceptions.UnprocessableEntityException; -import org.apache.iceberg.exceptions.ValidationException; -import org.apache.iceberg.io.CloseableGroup; -import org.apache.iceberg.io.FileIO; -import org.apache.iceberg.io.InputFile; -import org.apache.iceberg.util.PropertyUtil; -import org.apache.iceberg.view.BaseMetastoreViewCatalog; -import org.apache.iceberg.view.BaseViewOperations; -import org.apache.iceberg.view.ViewBuilder; -import org.apache.iceberg.view.ViewMetadata; -import org.apache.iceberg.view.ViewMetadataParser; -import org.apache.iceberg.view.ViewOperations; -import org.apache.iceberg.view.ViewUtil; -import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.PolarisConfiguration; -import org.apache.polaris.core.admin.model.StorageConfigInfo; -import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; -import org.apache.polaris.core.catalog.PolarisCatalogHelpers; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.entity.CatalogEntity; -import org.apache.polaris.core.entity.NamespaceEntity; -import org.apache.polaris.core.entity.PolarisEntity; -import org.apache.polaris.core.entity.PolarisEntityConstants; -import org.apache.polaris.core.entity.PolarisEntitySubType; -import org.apache.polaris.core.entity.PolarisEntityType; -import org.apache.polaris.core.entity.PolarisTaskConstants; -import org.apache.polaris.core.entity.TableLikeEntity; -import org.apache.polaris.core.persistence.BaseResult; -import org.apache.polaris.core.persistence.PolarisEntityManager; -import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; -import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest; -import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifestCatalogView; -import org.apache.polaris.core.persistence.resolver.ResolverPath; -import org.apache.polaris.core.persistence.resolver.ResolverStatus; -import org.apache.polaris.core.storage.*; -import org.apache.polaris.core.storage.aws.PolarisS3FileIOClientFactory; -import org.apache.polaris.service.catalog.io.FileIOFactory; -import org.apache.polaris.service.exception.IcebergExceptionMapper; -import org.apache.polaris.service.task.TaskExecutor; -import org.apache.polaris.service.types.NotificationRequest; -import org.apache.polaris.service.types.NotificationType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import software.amazon.awssdk.core.exception.SdkException; - -/** Defines the relationship between PolarisEntities and Iceberg's business logic. */ -public class BasePolarisCatalog extends BaseMetastoreViewCatalog - implements SupportsNamespaces, SupportsNotifications, Closeable, SupportsCredentialDelegation { - private static final Logger LOGGER = LoggerFactory.getLogger(BasePolarisCatalog.class); - - private static final Joiner SLASH = Joiner.on("/"); - - // Config key for whether to allow setting the FILE_IO_IMPL using catalog properties. Should - // only be allowed in dev/test environments. - static final String ALLOW_SPECIFYING_FILE_IO_IMPL = "ALLOW_SPECIFYING_FILE_IO_IMPL"; - static final boolean ALLOW_SPECIFYING_FILE_IO_IMPL_DEFAULT = false; - - // Config key for whether to skip credential-subscoping indirection entirely whenever trying - // to obtain storage credentials for instantiating a FileIO. If 'true', no attempt is made - // to use StorageConfigs to generate table-specific storage credentials, but instead the default - // fallthrough of table-level credential properties or else provider-specific APPLICATION_DEFAULT - // credential-loading will be used for the FileIO. - // Typically this setting is used in single-tenant server deployments that don't rely on - // "credential-vending" and can use server-default environment variables or credential config - // files for all storage access, or in test/dev scenarios. - static final String SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION = - "SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION"; - static final boolean SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION_DEFAULT = false; - - // Config key for initializing a default "catalogFileIO" that is available either via getIo() - // or for any TableOperations/ViewOperations instantiated, via ops.io() before entity-specific - // FileIO initialization is triggered for any such operations. - // Typically this should only be used in test scenarios where a BasePolarisCatalog instance - // is used for both the "client-side" and "server-side" logic instead of being access through - // a REST layer. - static final String INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST = - "INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST"; - static final boolean INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST_DEFAULT = false; - - private static final int MAX_RETRIES = 12; - - static final Predicate SHOULD_RETRY_REFRESH_PREDICATE = - ex -> { - // Default arguments from BaseMetastoreTableOperation only stop retries on - // NotFoundException. We should more carefully identify the set of retriable - // and non-retriable exceptions here. - return !(ex instanceof NotFoundException) - && !(ex instanceof IllegalArgumentException) - && !(ex instanceof AlreadyExistsException) - && !(ex instanceof ForbiddenException) - && !(ex instanceof UnprocessableEntityException) - && isStorageProviderRetryableException(ex); - }; - - private final PolarisEntityManager entityManager; - private final CallContext callContext; - private final PolarisResolutionManifestCatalogView resolvedEntityView; - private final CatalogEntity catalogEntity; - private final TaskExecutor taskExecutor; - private final AuthenticatedPolarisPrincipal authenticatedPrincipal; - private String ioImplClassName; - private FileIO catalogFileIO; - private final String catalogName; - private long catalogId = -1; - private String defaultBaseLocation; - private CloseableGroup closeableGroup; - private Map catalogProperties; - private Map tableDefaultProperties; - private FileIOFactory fileIOFactory; - private PolarisMetaStoreManager metaStoreManager; - - /** - * @param entityManager provides handle to underlying PolarisMetaStoreManager with which to - * perform mutations on entities. - * @param callContext the current CallContext - * @param resolvedEntityView accessor to resolved entity paths that have been pre-vetted to ensure - * this catalog instance only interacts with authorized resolved paths. - * @param taskExecutor Executor we use to register cleanup task handlers - */ - public BasePolarisCatalog( - PolarisEntityManager entityManager, - PolarisMetaStoreManager metaStoreManager, - CallContext callContext, - PolarisResolutionManifestCatalogView resolvedEntityView, - AuthenticatedPolarisPrincipal authenticatedPrincipal, - TaskExecutor taskExecutor, - FileIOFactory fileIOFactory) { - this.entityManager = entityManager; - this.callContext = callContext; - this.resolvedEntityView = resolvedEntityView; - this.catalogEntity = - CatalogEntity.of(resolvedEntityView.getResolvedReferenceCatalogEntity().getRawLeafEntity()); - this.authenticatedPrincipal = authenticatedPrincipal; - this.taskExecutor = taskExecutor; - this.catalogId = catalogEntity.getId(); - this.catalogName = catalogEntity.getName(); - this.fileIOFactory = fileIOFactory; - this.metaStoreManager = metaStoreManager; - } - - @Override - public String name() { - return catalogName; - } - - @VisibleForTesting - FileIO getIo() { - return catalogFileIO; - } - - @Override - public void initialize(String name, Map properties) { - Preconditions.checkState( - this.catalogName.equals(name), - "Tried to initialize catalog as name %s but already constructed with name %s", - name, - this.catalogName); - - // Base location from catalogEntity is primary source of truth, otherwise fall through - // to the same key from the properties map, and finally fall through to WAREHOUSE_LOCATION. - String baseLocation = - Optional.ofNullable(catalogEntity.getDefaultBaseLocation()) - .orElse( - properties.getOrDefault( - CatalogEntity.DEFAULT_BASE_LOCATION_KEY, - properties.getOrDefault(CatalogProperties.WAREHOUSE_LOCATION, ""))); - this.defaultBaseLocation = baseLocation.replaceAll("/*$", ""); - - Boolean allowSpecifyingFileIoImpl = - getBooleanContextConfiguration( - ALLOW_SPECIFYING_FILE_IO_IMPL, ALLOW_SPECIFYING_FILE_IO_IMPL_DEFAULT); - - PolarisStorageConfigurationInfo storageConfigurationInfo = - catalogEntity.getStorageConfigurationInfo(); - if (properties.containsKey(CatalogProperties.FILE_IO_IMPL)) { - ioImplClassName = properties.get(CatalogProperties.FILE_IO_IMPL); - - if (!Boolean.TRUE.equals(allowSpecifyingFileIoImpl)) { - throw new ValidationException( - "Cannot set property '%s' to '%s' for this catalog.", - CatalogProperties.FILE_IO_IMPL, ioImplClassName); - } - LOGGER.debug( - "Allowing overriding ioImplClassName to {} for storageConfiguration {}", - ioImplClassName, - storageConfigurationInfo); - } else { - if (storageConfigurationInfo != null) { - ioImplClassName = storageConfigurationInfo.getFileIoImplClassName(); - LOGGER.debug( - "Resolved ioImplClassName {} from storageConfiguration {}", - ioImplClassName, - storageConfigurationInfo); - } else { - LOGGER.warn( - "Cannot resolve property '{}' for null storageConfiguration.", - CatalogProperties.FILE_IO_IMPL); - } - } - this.closeableGroup = CallContext.getCurrentContext().closeables(); - closeableGroup.addCloseable(metricsReporter()); - closeableGroup.setSuppressCloseFailure(true); - - catalogProperties = properties; - tableDefaultProperties = - PropertyUtil.propertiesWithPrefix(properties, CatalogProperties.TABLE_DEFAULT_PREFIX); - - Boolean initializeDefaultCatalogFileioForTest = - getBooleanContextConfiguration( - INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST, - INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST_DEFAULT); - if (Boolean.TRUE.equals(initializeDefaultCatalogFileioForTest)) { - LOGGER.debug( - "Initializing a default catalogFileIO with properties {}", tableDefaultProperties); - this.catalogFileIO = loadFileIO(ioImplClassName, tableDefaultProperties); - closeableGroup.addCloseable(this.catalogFileIO); - } else { - LOGGER.debug("Not initializing default catalogFileIO"); - this.catalogFileIO = null; - } - } - - public void setMetaStoreManager(PolarisMetaStoreManager newMetaStoreManager) { - this.metaStoreManager = newMetaStoreManager; - } - - @Override - protected Map properties() { - return catalogProperties == null ? ImmutableMap.of() : catalogProperties; - } - - @Override - public Table registerTable(TableIdentifier identifier, String metadataFileLocation) { - Preconditions.checkArgument( - identifier != null && isValidIdentifier(identifier), "Invalid identifier: %s", identifier); - Preconditions.checkArgument( - metadataFileLocation != null && !metadataFileLocation.isEmpty(), - "Cannot register an empty metadata file location as a table"); - - // Throw an exception if this table already exists in the catalog. - if (tableExists(identifier)) { - throw new AlreadyExistsException("Table already exists: %s", identifier); - } - - String locationDir = metadataFileLocation.substring(0, metadataFileLocation.lastIndexOf("/")); - - TableOperations ops = newTableOps(identifier); - - PolarisResolvedPathWrapper resolvedParent = - resolvedEntityView.getResolvedPath(identifier.namespace()); - if (resolvedParent == null) { - // Illegal state because the namespace should've already been in the static resolution set. - throw new IllegalStateException( - String.format("Failed to fetch resolved parent for TableIdentifier '%s'", identifier)); - } - FileIO fileIO = - refreshIOWithCredentials( - identifier, - Set.of(locationDir), - resolvedParent, - new HashMap<>(tableDefaultProperties), - Set.of(PolarisStorageActions.READ)); - - InputFile metadataFile = fileIO.newInputFile(metadataFileLocation); - TableMetadata metadata = TableMetadataParser.read(fileIO, metadataFile); - ops.commit(null, metadata); - - return new BaseTable(ops, fullTableName(name(), identifier), metricsReporter()); - } - - @Override - public TableBuilder buildTable(TableIdentifier identifier, Schema schema) { - return new BasePolarisCatalogTableBuilder(identifier, schema); - } - - @Override - public ViewBuilder buildView(TableIdentifier identifier) { - return new BasePolarisCatalogViewBuilder(identifier); - } - - @Override - protected TableOperations newTableOps(TableIdentifier tableIdentifier) { - return new BasePolarisTableOperations(catalogFileIO, tableIdentifier); - } - - @Override - protected String defaultWarehouseLocation(TableIdentifier tableIdentifier) { - if (tableIdentifier.namespace().isEmpty()) { - return SLASH.join( - defaultNamespaceLocation(tableIdentifier.namespace()), tableIdentifier.name()); - } else { - PolarisResolvedPathWrapper resolvedNamespace = - resolvedEntityView.getResolvedPath(tableIdentifier.namespace()); - if (resolvedNamespace == null) { - throw new NoSuchNamespaceException( - "Namespace does not exist: %s", tableIdentifier.namespace()); - } - List namespacePath = resolvedNamespace.getRawFullPath(); - String namespaceLocation = resolveLocationForPath(namespacePath); - return SLASH.join(namespaceLocation, tableIdentifier.name()); - } - } - - private String defaultNamespaceLocation(Namespace namespace) { - if (namespace.isEmpty()) { - return defaultBaseLocation; - } else { - return SLASH.join(defaultBaseLocation, SLASH.join(namespace.levels())); - } - } - - private Set getLocationsAllowedToBeAccessed(TableMetadata tableMetadata) { - Set locations = new HashSet<>(); - locations.add(tableMetadata.location()); - if (tableMetadata - .properties() - .containsKey(TableLikeEntity.USER_SPECIFIED_WRITE_DATA_LOCATION_KEY)) { - locations.add( - tableMetadata.properties().get(TableLikeEntity.USER_SPECIFIED_WRITE_DATA_LOCATION_KEY)); - } - if (tableMetadata - .properties() - .containsKey(TableLikeEntity.USER_SPECIFIED_WRITE_METADATA_LOCATION_KEY)) { - locations.add( - tableMetadata - .properties() - .get(TableLikeEntity.USER_SPECIFIED_WRITE_METADATA_LOCATION_KEY)); - } - return locations; - } - - private Set getLocationsAllowedToBeAccessed(ViewMetadata viewMetadata) { - return Set.of(viewMetadata.location()); - } - - @Override - public boolean dropTable(TableIdentifier tableIdentifier, boolean purge) { - TableOperations ops = newTableOps(tableIdentifier); - TableMetadata lastMetadata; - if (purge && ops.current() != null) { - lastMetadata = ops.current(); - } else { - lastMetadata = null; - } - - Optional storageInfoEntity = findStorageInfo(tableIdentifier); - - // The storageProperties we stash away in the Task should be the superset of the - // internalProperties of the StorageInfoEntity to be able to use its StorageIntegration - // combined with other miscellaneous FileIO-related initialization properties defined - // by the Table. - Map storageProperties = - storageInfoEntity - .map(PolarisEntity::getInternalPropertiesAsMap) - .map( - properties -> { - if (lastMetadata == null) { - return Map.of(); - } - Map clone = new HashMap<>(); - - // The user-configurable table properties are the baseline, but then override - // with our restricted properties so that table properties can't clobber the - // more restricted ones. - clone.putAll(lastMetadata.properties()); - clone.put(CatalogProperties.FILE_IO_IMPL, ioImplClassName); - clone.putAll(properties); - clone.put(PolarisTaskConstants.STORAGE_LOCATION, lastMetadata.location()); - return clone; - }) - .orElse(Map.of()); - PolarisMetaStoreManager.DropEntityResult dropEntityResult = - dropTableLike(PolarisEntitySubType.TABLE, tableIdentifier, storageProperties, purge); - if (!dropEntityResult.isSuccess()) { - return false; - } - - if (purge && lastMetadata != null && dropEntityResult.getCleanupTaskId() != null) { - LOGGER.info( - "Scheduled cleanup task {} for table {}", - dropEntityResult.getCleanupTaskId(), - tableIdentifier); - taskExecutor.addTaskHandlerContext( - dropEntityResult.getCleanupTaskId(), CallContext.getCurrentContext()); - } - - return true; - } - - @Override - public List listTables(Namespace namespace) { - if (!namespaceExists(namespace) && !namespace.isEmpty()) { - throw new NoSuchNamespaceException( - "Cannot list tables for namespace. Namespace does not exist: %s", namespace); - } - - return listTableLike(PolarisEntitySubType.TABLE, namespace); - } - - @Override - public void renameTable(TableIdentifier from, TableIdentifier to) { - if (from.equals(to)) { - return; - } - - renameTableLike(PolarisEntitySubType.TABLE, from, to); - } - - @Override - public void createNamespace(Namespace namespace) { - createNamespace(namespace, Collections.emptyMap()); - } - - @Override - public void createNamespace(Namespace namespace, Map metadata) { - LOGGER.debug("Creating namespace {} with metadata {}", namespace, metadata); - if (namespace.isEmpty()) { - throw new AlreadyExistsException( - "Cannot create root namespace, as it already exists implicitly."); - } - - // TODO: These should really be helpers in core Iceberg Namespace. - Namespace parentNamespace = PolarisCatalogHelpers.getParentNamespace(namespace); - - PolarisResolvedPathWrapper resolvedParent = resolvedEntityView.getResolvedPath(parentNamespace); - if (resolvedParent == null) { - throw new NoSuchNamespaceException( - "Cannot create namespace %s. Parent namespace does not exist.", namespace); - } - createNamespaceInternal(namespace, metadata, resolvedParent); - } - - private void createNamespaceInternal( - Namespace namespace, - Map metadata, - PolarisResolvedPathWrapper resolvedParent) { - String baseLocation = resolveNamespaceLocation(namespace, metadata); - NamespaceEntity entity = - new NamespaceEntity.Builder(namespace) - .setCatalogId(getCatalogId()) - .setId(getMetaStoreManager().generateNewEntityId(getCurrentPolarisContext()).getId()) - .setParentId(resolvedParent.getRawLeafEntity().getId()) - .setProperties(metadata) - .setCreateTimestamp(System.currentTimeMillis()) - .setBaseLocation(baseLocation) - .build(); - if (!callContext - .getPolarisCallContext() - .getConfigurationStore() - .getConfiguration( - callContext.getPolarisCallContext(), - PolarisConfiguration.ALLOW_NAMESPACE_LOCATION_OVERLAP)) { - LOGGER.debug("Validating no overlap for {} with sibling tables or namespaces", namespace); - validateNoLocationOverlap( - entity.getBaseLocation(), resolvedParent.getRawFullPath(), entity.getName()); - } else { - LOGGER.debug("Skipping location overlap validation for namespace '{}'", namespace); - } - PolarisEntity returnedEntity = - PolarisEntity.of( - getMetaStoreManager() - .createEntityIfNotExists( - getCurrentPolarisContext(), - PolarisEntity.toCoreList(resolvedParent.getRawFullPath()), - entity)); - if (returnedEntity == null) { - throw new AlreadyExistsException( - "Cannot create namespace %s. Namespace already exists", namespace); - } - } - - private String resolveNamespaceLocation(Namespace namespace, Map properties) { - if (properties.containsKey(PolarisEntityConstants.ENTITY_BASE_LOCATION)) { - return properties.get(PolarisEntityConstants.ENTITY_BASE_LOCATION); - } else { - List parentPath = - namespace.length() > 1 - ? getResolvedParentNamespace(namespace).getRawFullPath() - : List.of(resolvedEntityView.getResolvedReferenceCatalogEntity().getRawLeafEntity()); - - String parentLocation = resolveLocationForPath(parentPath); - - return parentLocation + "/" + namespace.level(namespace.length() - 1); - } - } - - private static @Nonnull String resolveLocationForPath(List parentPath) { - // always take the first object. If it has the base-location, stop there - AtomicBoolean foundBaseLocation = new AtomicBoolean(false); - return parentPath.reversed().stream() - .takeWhile( - entity -> - !foundBaseLocation.getAndSet( - entity - .getPropertiesAsMap() - .containsKey(PolarisEntityConstants.ENTITY_BASE_LOCATION))) - .toList() - .reversed() - .stream() - .map( - entity -> { - if (entity.getType().equals(PolarisEntityType.CATALOG)) { - return CatalogEntity.of(entity).getDefaultBaseLocation(); - } else { - String baseLocation = - entity.getPropertiesAsMap().get(PolarisEntityConstants.ENTITY_BASE_LOCATION); - if (baseLocation != null) { - return baseLocation; - } else { - return entity.getName(); - } - } - }) - .map(BasePolarisCatalog::stripLeadingTrailingSlash) - .collect(Collectors.joining("/")); - } - - private static String stripLeadingTrailingSlash(String location) { - if (location.startsWith("/")) { - return stripLeadingTrailingSlash(location.substring(1)); - } - if (location.endsWith("/")) { - return location.substring(0, location.length() - 1); - } else { - return location; - } - } - - private PolarisResolvedPathWrapper getResolvedParentNamespace(Namespace namespace) { - Namespace parentNamespace = - Namespace.of(Arrays.copyOf(namespace.levels(), namespace.length() - 1)); - PolarisResolvedPathWrapper resolvedParent = resolvedEntityView.getResolvedPath(parentNamespace); - if (resolvedParent == null) { - return resolvedEntityView.getPassthroughResolvedPath(parentNamespace); - } - return resolvedParent; - } - - @Override - public boolean namespaceExists(Namespace namespace) { - return resolvedEntityView.getResolvedPath(namespace) != null; - } - - @Override - public boolean dropNamespace(Namespace namespace) throws NamespaceNotEmptyException { - PolarisResolvedPathWrapper resolvedEntities = resolvedEntityView.getResolvedPath(namespace); - if (resolvedEntities == null) { - return false; - } - - List catalogPath = resolvedEntities.getRawParentPath(); - PolarisEntity leafEntity = resolvedEntities.getRawLeafEntity(); - - // drop if exists and is empty - PolarisCallContext polarisCallContext = callContext.getPolarisCallContext(); - PolarisMetaStoreManager.DropEntityResult dropEntityResult = - getMetaStoreManager() - .dropEntityIfExists( - getCurrentPolarisContext(), - PolarisEntity.toCoreList(catalogPath), - leafEntity, - Map.of(), - polarisCallContext - .getConfigurationStore() - .getConfiguration( - polarisCallContext, PolarisConfiguration.CLEANUP_ON_NAMESPACE_DROP)); - - if (!dropEntityResult.isSuccess() && dropEntityResult.failedBecauseNotEmpty()) { - throw new NamespaceNotEmptyException("Namespace %s is not empty", namespace); - } - - // return status of drop operation - return dropEntityResult.isSuccess(); - } - - @Override - public boolean setProperties(Namespace namespace, Map properties) - throws NoSuchNamespaceException { - PolarisResolvedPathWrapper resolvedEntities = resolvedEntityView.getResolvedPath(namespace); - if (resolvedEntities == null) { - throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace); - } - PolarisEntity entity = resolvedEntities.getRawLeafEntity(); - Map newProperties = new HashMap<>(entity.getPropertiesAsMap()); - - // Merge new properties into existing map. - newProperties.putAll(properties); - PolarisEntity updatedEntity = - new PolarisEntity.Builder(entity).setProperties(newProperties).build(); - - if (!callContext - .getPolarisCallContext() - .getConfigurationStore() - .getConfiguration( - callContext.getPolarisCallContext(), - PolarisConfiguration.ALLOW_NAMESPACE_LOCATION_OVERLAP)) { - LOGGER.debug("Validating no overlap with sibling tables or namespaces"); - validateNoLocationOverlap( - NamespaceEntity.of(updatedEntity).getBaseLocation(), - resolvedEntities.getRawParentPath(), - updatedEntity.getName()); - } else { - LOGGER.debug("Skipping location overlap validation for namespace '{}'", namespace); - } - - List parentPath = resolvedEntities.getRawFullPath(); - PolarisEntity returnedEntity = - Optional.ofNullable( - getMetaStoreManager() - .updateEntityPropertiesIfNotChanged( - getCurrentPolarisContext(), - PolarisEntity.toCoreList(parentPath), - updatedEntity) - .getEntity()) - .map(PolarisEntity::new) - .orElse(null); - if (returnedEntity == null) { - throw new RuntimeException("Concurrent modification of namespace: " + namespace); - } - return true; - } - - @Override - public boolean removeProperties(Namespace namespace, Set properties) - throws NoSuchNamespaceException { - PolarisResolvedPathWrapper resolvedEntities = resolvedEntityView.getResolvedPath(namespace); - if (resolvedEntities == null) { - throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace); - } - PolarisEntity entity = resolvedEntities.getRawLeafEntity(); - - Map updatedProperties = new HashMap<>(entity.getPropertiesAsMap()); - properties.forEach(updatedProperties::remove); - - PolarisEntity updatedEntity = - new PolarisEntity.Builder(entity).setProperties(updatedProperties).build(); - - List parentPath = resolvedEntities.getRawFullPath(); - PolarisEntity returnedEntity = - Optional.ofNullable( - getMetaStoreManager() - .updateEntityPropertiesIfNotChanged( - getCurrentPolarisContext(), - PolarisEntity.toCoreList(parentPath), - updatedEntity) - .getEntity()) - .map(PolarisEntity::new) - .orElse(null); - if (returnedEntity == null) { - throw new RuntimeException("Concurrent modification of namespace: " + namespace); - } - return true; - } - - @Override - public Map loadNamespaceMetadata(Namespace namespace) - throws NoSuchNamespaceException { - PolarisResolvedPathWrapper resolvedEntities = resolvedEntityView.getResolvedPath(namespace); - if (resolvedEntities == null) { - throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace); - } - NamespaceEntity entity = NamespaceEntity.of(resolvedEntities.getRawLeafEntity()); - Preconditions.checkState( - entity.getParentNamespace().equals(PolarisCatalogHelpers.getParentNamespace(namespace)), - "Mismatched stored parentNamespace '%s' vs looked up parentNamespace '%s", - entity.getParentNamespace(), - PolarisCatalogHelpers.getParentNamespace(namespace)); - - return entity.getPropertiesAsMap(); - } - - @Override - public List listNamespaces() { - return listNamespaces(Namespace.empty()); - } - - @Override - public List listNamespaces(Namespace namespace) throws NoSuchNamespaceException { - PolarisResolvedPathWrapper resolvedEntities = resolvedEntityView.getResolvedPath(namespace); - if (resolvedEntities == null) { - throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace); - } - - List catalogPath = resolvedEntities.getRawFullPath(); - List entities = - PolarisEntity.toNameAndIdList( - getMetaStoreManager() - .listEntities( - getCurrentPolarisContext(), - PolarisEntity.toCoreList(catalogPath), - PolarisEntityType.NAMESPACE, - PolarisEntitySubType.NULL_SUBTYPE) - .getEntities()); - return PolarisCatalogHelpers.nameAndIdToNamespaces(catalogPath, entities); - } - - @Override - public void close() throws IOException {} - - @Override - public List listViews(Namespace namespace) { - if (!namespaceExists(namespace) && !namespace.isEmpty()) { - throw new NoSuchNamespaceException( - "Cannot list views for namespace. Namespace does not exist: %s", namespace); - } - - return listTableLike(PolarisEntitySubType.VIEW, namespace); - } - - @Override - protected ViewOperations newViewOps(TableIdentifier identifier) { - return new BasePolarisViewOperations(catalogFileIO, identifier); - } - - @Override - public boolean dropView(TableIdentifier identifier) { - return dropTableLike(PolarisEntitySubType.VIEW, identifier, Map.of(), true).isSuccess(); - } - - @Override - public void renameView(TableIdentifier from, TableIdentifier to) { - if (from.equals(to)) { - return; - } - - renameTableLike(PolarisEntitySubType.VIEW, from, to); - } - - @Override - public boolean sendNotification( - TableIdentifier identifier, NotificationRequest notificationRequest) { - return sendNotificationForTableLike( - PolarisEntitySubType.TABLE, identifier, notificationRequest); - } - - @Override - public Map getCredentialConfig( - TableIdentifier tableIdentifier, - TableMetadata tableMetadata, - Set storageActions) { - Optional storageInfo = findStorageInfo(tableIdentifier); - if (storageInfo.isEmpty()) { - LOGGER - .atWarn() - .addKeyValue("tableIdentifier", tableIdentifier) - .log("Table entity has no storage configuration in its hierarchy"); - return Map.of(); - } - return refreshCredentials( - tableIdentifier, - storageActions, - getLocationsAllowedToBeAccessed(tableMetadata), - storageInfo.get()); - } - - /** - * Based on configuration settings, for callsites that need to handle potentially setting a new - * base location for a TableLike entity, produces the transformed location if applicable, or else - * the unaltered specified location. - */ - public String transformTableLikeLocation(String specifiedTableLikeLocation) { - String replaceNewLocationPrefix = catalogEntity.getReplaceNewLocationPrefixWithCatalogDefault(); - if (specifiedTableLikeLocation != null - && replaceNewLocationPrefix != null - && specifiedTableLikeLocation.startsWith(replaceNewLocationPrefix)) { - String modifiedLocation = - defaultBaseLocation - + specifiedTableLikeLocation.substring(replaceNewLocationPrefix.length()); - LOGGER - .atDebug() - .addKeyValue("specifiedTableLikeLocation", specifiedTableLikeLocation) - .addKeyValue("modifiedLocation", modifiedLocation) - .log("Translating specifiedTableLikeLocation based on config"); - return modifiedLocation; - } - return specifiedTableLikeLocation; - } - - private @Nonnull Optional findStorageInfo(TableIdentifier tableIdentifier) { - PolarisResolvedPathWrapper resolvedTableEntities = - resolvedEntityView.getResolvedPath(tableIdentifier, PolarisEntitySubType.TABLE); - - PolarisResolvedPathWrapper resolvedStorageEntity = - resolvedTableEntities == null - ? resolvedEntityView.getResolvedPath(tableIdentifier.namespace()) - : resolvedTableEntities; - - return findStorageInfoFromHierarchy(resolvedStorageEntity); - } - - private Map refreshCredentials( - TableIdentifier tableIdentifier, - Set storageActions, - String tableLocation, - PolarisEntity entity) { - return refreshCredentials(tableIdentifier, storageActions, Set.of(tableLocation), entity); - } - - private Map refreshCredentials( - TableIdentifier tableIdentifier, - Set storageActions, - Set tableLocations, - PolarisEntity entity) { - Boolean skipCredentialSubscopingIndirection = - getBooleanContextConfiguration( - SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION, SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION_DEFAULT); - if (Boolean.TRUE.equals(skipCredentialSubscopingIndirection)) { - LOGGER - .atInfo() - .addKeyValue("tableIdentifier", tableIdentifier) - .log("Skipping generation of subscoped creds for table"); - return Map.of(); - } - - boolean allowList = - storageActions.contains(PolarisStorageActions.LIST) - || storageActions.contains(PolarisStorageActions.ALL); - Set writeLocations = - storageActions.contains(PolarisStorageActions.WRITE) - || storageActions.contains(PolarisStorageActions.DELETE) - || storageActions.contains(PolarisStorageActions.ALL) - ? tableLocations - : Set.of(); - Map credentialsMap = - entityManager - .getCredentialCache() - .getOrGenerateSubScopeCreds( - getCredentialVendor(), - callContext.getPolarisCallContext(), - entity, - allowList, - tableLocations, - writeLocations); - LOGGER - .atDebug() - .addKeyValue("tableIdentifier", tableIdentifier) - .addKeyValue("credentialKeys", credentialsMap.keySet()) - .log("Loaded scoped credentials for table"); - if (credentialsMap.isEmpty()) { - LOGGER.debug("No credentials found for table"); - } - return credentialsMap; - } - - /** - * Validates that the specified {@code location} is valid for whatever storage config is found for - * this TableLike's parent hierarchy. - */ - private void validateLocationForTableLike(TableIdentifier identifier, String location) { - PolarisResolvedPathWrapper resolvedStorageEntity = - resolvedEntityView.getResolvedPath(identifier, PolarisEntitySubType.ANY_SUBTYPE); - if (resolvedStorageEntity == null) { - resolvedStorageEntity = resolvedEntityView.getResolvedPath(identifier.namespace()); - } - if (resolvedStorageEntity == null) { - resolvedStorageEntity = resolvedEntityView.getPassthroughResolvedPath(identifier.namespace()); - } - - validateLocationForTableLike(identifier, location, resolvedStorageEntity); - } - - /** - * Validates that the specified {@code location} is valid for whatever storage config is found for - * this TableLike's parent hierarchy. - */ - private void validateLocationForTableLike( - TableIdentifier identifier, - String location, - PolarisResolvedPathWrapper resolvedStorageEntity) { - validateLocationsForTableLike(identifier, Set.of(location), resolvedStorageEntity); - } - - /** - * Validates that the specified {@code locations} are valid for whatever storage config is found - * for this TableLike's parent hierarchy. - */ - private void validateLocationsForTableLike( - TableIdentifier identifier, - Set locations, - PolarisResolvedPathWrapper resolvedStorageEntity) { - Optional optStorageConfiguration = - PolarisStorageConfigurationInfo.forEntityPath( - callContext.getPolarisCallContext().getDiagServices(), - resolvedStorageEntity.getRawFullPath()); - - optStorageConfiguration.ifPresentOrElse( - storageConfigInfo -> { - Map> - validationResults = - InMemoryStorageIntegration.validateSubpathsOfAllowedLocations( - storageConfigInfo, Set.of(PolarisStorageActions.ALL), locations); - validationResults - .values() - .forEach( - actionResult -> - actionResult - .values() - .forEach( - result -> { - if (!result.isSuccess()) { - throw new ForbiddenException( - "Invalid locations '%s' for identifier '%s': %s", - locations, identifier, result.getMessage()); - } else { - LOGGER.debug( - "Validated locations '{}' for identifier '{}'", - locations, - identifier); - } - })); - - // TODO: Consider exposing a property to control whether to use the explicit default - // in-memory PolarisStorageIntegration implementation to perform validation or - // whether to delegate to PolarisMetaStoreManager::validateAccessToLocations. - // Usually the validation is better to perform with local business logic, but if - // there are additional rules to be evaluated by a custom PolarisMetaStoreManager - // implementation, then the validation should go through that API instead as follows: - // - // PolarisMetaStoreManager.ValidateAccessResult validateResult = - // getMetaStoreManager().validateAccessToLocations( - // getCurrentPolarisContext(), - // storageInfoHolderEntity.getCatalogId(), - // storageInfoHolderEntity.getId(), - // Set.of(PolarisStorageActions.ALL), - // Set.of(location)); - // if (!validateResult.isSuccess()) { - // throw new ForbiddenException("Invalid location '%s' for identifier '%s': %s", - // location, identifier, validateResult.getExtraInformation()); - // } - }, - () -> { - List allowedStorageTypes = - callContext - .getPolarisCallContext() - .getConfigurationStore() - .getConfiguration( - callContext.getPolarisCallContext(), - PolarisConfiguration.SUPPORTED_CATALOG_STORAGE_TYPES); - if (!allowedStorageTypes.contains(StorageConfigInfo.StorageTypeEnum.FILE.name())) { - List invalidLocations = - locations.stream() - .filter(location -> location.startsWith("file:") || location.startsWith("http")) - .collect(Collectors.toList()); - if (!invalidLocations.isEmpty()) { - throw new ForbiddenException( - "Invalid locations '%s' for identifier '%s': File locations are not allowed", - invalidLocations, identifier); - } - } - }); - } - - /** - * Validates the table location has no overlap with other entities after checking the - * configuration of the service - */ - private void validateNoLocationOverlap( - CatalogEntity catalog, - TableIdentifier identifier, - List resolvedNamespace, - String location) { - if (callContext - .getPolarisCallContext() - .getConfigurationStore() - .getConfiguration( - callContext.getPolarisCallContext(), - catalog, - PolarisConfiguration.ALLOW_TABLE_LOCATION_OVERLAP)) { - LOGGER.debug("Skipping location overlap validation for identifier '{}'", identifier); - } else { // if (entity.getSubType().equals(PolarisEntitySubType.TABLE)) { - // TODO - is this necessary for views? overlapping views do not expose subdirectories via the - // credential vending so this feels like an unnecessary restriction - LOGGER.debug("Validating no overlap with sibling tables or namespaces"); - validateNoLocationOverlap(location, resolvedNamespace, identifier.name()); - } - } - - /** - * Validate no location overlap exists between the entity path and its sibling entities. This - * resolves all siblings at the same level as the target entity (namespaces if the target entity - * is a namespace whose parent is the catalog, namespaces and tables otherwise) and checks the - * base-location property of each. The target entity's base location may not be a prefix or a - * suffix of any sibling entity's base location. - */ - private void validateNoLocationOverlap( - String location, List parentPath, String name) { - PolarisMetaStoreManager.ListEntitiesResult siblingNamespacesResult = - getMetaStoreManager() - .listEntities( - callContext.getPolarisCallContext(), - parentPath.stream().map(PolarisEntity::toCore).collect(Collectors.toList()), - PolarisEntityType.NAMESPACE, - PolarisEntitySubType.ANY_SUBTYPE); - if (!siblingNamespacesResult.isSuccess()) { - throw new IllegalStateException( - "Unable to resolve siblings entities to validate location - could not list namespaces"); - } - - // if the entity path has more than just the catalog, check for tables as well as other - // namespaces - Optional parentNamespace = - parentPath.size() > 1 - ? Optional.of(NamespaceEntity.of(parentPath.getLast())) - : Optional.empty(); - - List siblingTables = - parentNamespace - .map( - ns -> { - PolarisMetaStoreManager.ListEntitiesResult siblingTablesResult = - getMetaStoreManager() - .listEntities( - callContext.getPolarisCallContext(), - parentPath.stream() - .map(PolarisEntity::toCore) - .collect(Collectors.toList()), - PolarisEntityType.TABLE_LIKE, - PolarisEntitySubType.ANY_SUBTYPE); - if (!siblingTablesResult.isSuccess()) { - throw new IllegalStateException( - "Unable to resolve siblings entities to validate location - could not list tables"); - } - return siblingTablesResult.getEntities().stream() - .map(tbl -> TableIdentifier.of(ns.asNamespace(), tbl.getName())) - .collect(Collectors.toList()); - }) - .orElse(List.of()); - - List siblingNamespaces = - siblingNamespacesResult.getEntities().stream() - .map( - ns -> { - String[] nsLevels = - parentNamespace - .map(parent -> parent.asNamespace().levels()) - .orElse(new String[0]); - String[] newLevels = Arrays.copyOf(nsLevels, nsLevels.length + 1); - newLevels[nsLevels.length] = ns.getName(); - return Namespace.of(newLevels); - }) - .toList(); - LOGGER.debug( - "Resolving {} sibling entities to validate location", - siblingTables.size() + siblingNamespaces.size()); - PolarisResolutionManifest resolutionManifest = - new PolarisResolutionManifest( - callContext, entityManager, authenticatedPrincipal, parentPath.getFirst().getName()); - siblingTables.forEach( - tbl -> - resolutionManifest.addPath( - new ResolverPath( - PolarisCatalogHelpers.tableIdentifierToList(tbl), PolarisEntityType.TABLE_LIKE), - tbl)); - siblingNamespaces.forEach( - ns -> - resolutionManifest.addPath( - new ResolverPath(Arrays.asList(ns.levels()), PolarisEntityType.NAMESPACE), ns)); - ResolverStatus status = resolutionManifest.resolveAll(); - if (!status.getStatus().equals(ResolverStatus.StatusEnum.SUCCESS)) { - throw new IllegalStateException( - "Unable to resolve sibling entities to validate location - could not resolve" - + status.getFailedToResolvedEntityName()); - } - - StorageLocation targetLocation = StorageLocation.of(location); - Stream.concat( - siblingTables.stream() - .filter(tbl -> !tbl.name().equals(name)) - .map( - tbl -> { - PolarisResolvedPathWrapper resolveTablePath = - resolutionManifest.getResolvedPath(tbl); - return TableLikeEntity.of(resolveTablePath.getRawLeafEntity()) - .getBaseLocation(); - }), - siblingNamespaces.stream() - .filter(ns -> !ns.level(ns.length() - 1).equals(name)) - .map( - ns -> { - PolarisResolvedPathWrapper resolveNamespacePath = - resolutionManifest.getResolvedPath(ns); - return NamespaceEntity.of(resolveNamespacePath.getRawLeafEntity()) - .getBaseLocation(); - })) - .filter(java.util.Objects::nonNull) - .map(StorageLocation::of) - .forEach( - siblingLocation -> { - if (targetLocation.isChildOf(siblingLocation) - || siblingLocation.isChildOf(targetLocation)) { - throw new org.apache.iceberg.exceptions.ForbiddenException( - "Unable to create table at location '%s' because it conflicts with existing table or namespace at location '%s'", - targetLocation, siblingLocation); - } - }); - } - - private class BasePolarisCatalogTableBuilder - extends BaseMetastoreViewCatalog.BaseMetastoreViewCatalogTableBuilder { - - public BasePolarisCatalogTableBuilder(TableIdentifier identifier, Schema schema) { - super(identifier, schema); - } - - @Override - public TableBuilder withLocation(String newLocation) { - return super.withLocation(transformTableLikeLocation(newLocation)); - } - } - - private class BasePolarisCatalogViewBuilder extends BaseMetastoreViewCatalog.BaseViewBuilder { - - public BasePolarisCatalogViewBuilder(TableIdentifier identifier) { - super(identifier); - withProperties( - PropertyUtil.propertiesWithPrefix( - BasePolarisCatalog.this.properties(), "table-default.")); - } - - @Override - public ViewBuilder withLocation(String newLocation) { - return super.withLocation(transformTableLikeLocation(newLocation)); - } - } - - private class BasePolarisTableOperations extends BaseMetastoreTableOperations { - private final TableIdentifier tableIdentifier; - private final String fullTableName; - private FileIO tableFileIO; - - BasePolarisTableOperations(FileIO defaultFileIO, TableIdentifier tableIdentifier) { - LOGGER.debug("new BasePolarisTableOperations for {}", tableIdentifier); - this.tableIdentifier = tableIdentifier; - this.fullTableName = fullTableName(catalogName, tableIdentifier); - this.tableFileIO = defaultFileIO; - } - - @Override - public void doRefresh() { - LOGGER.debug("doRefresh for tableIdentifier {}", tableIdentifier); - // While doing refresh/commit protocols, we must fetch the fresh "passthrough" resolved - // table entity instead of the statically-resolved authz resolution set. - PolarisResolvedPathWrapper resolvedEntities = - resolvedEntityView.getPassthroughResolvedPath( - tableIdentifier, PolarisEntitySubType.TABLE); - TableLikeEntity entity = null; - - if (resolvedEntities != null) { - entity = TableLikeEntity.of(resolvedEntities.getRawLeafEntity()); - if (!tableIdentifier.equals(entity.getTableIdentifier())) { - LOGGER - .atError() - .addKeyValue("entity.getTableIdentifier()", entity.getTableIdentifier()) - .addKeyValue("tableIdentifier", tableIdentifier) - .log("Stored table identifier mismatches requested identifier"); - } - } - - String latestLocation = entity != null ? entity.getMetadataLocation() : null; - LOGGER.debug("Refreshing latestLocation: {}", latestLocation); - if (latestLocation == null) { - disableRefresh(); - } else { - refreshFromMetadataLocation( - latestLocation, - SHOULD_RETRY_REFRESH_PREDICATE, - MAX_RETRIES, - metadataLocation -> { - String latestLocationDir = - latestLocation.substring(0, latestLocation.lastIndexOf('/')); - // TODO: Once we have the "current" table properties pulled into the resolvedEntity - // then we should use the actual current table properties for IO refresh here - // instead of the general tableDefaultProperties. - FileIO fileIO = - refreshIOWithCredentials( - tableIdentifier, - Set.of(latestLocationDir), - resolvedEntities, - new HashMap<>(tableDefaultProperties), - Set.of(PolarisStorageActions.READ)); - return TableMetadataParser.read(fileIO, metadataLocation); - }); - } - } - - @Override - public void doCommit(TableMetadata base, TableMetadata metadata) { - LOGGER.debug( - "doCommit for table {} with base {}, metadata {}", tableIdentifier, base, metadata); - // TODO: Maybe avoid writing metadata if there's definitely a transaction conflict - if (null == base && !namespaceExists(tableIdentifier.namespace())) { - throw new NoSuchNamespaceException( - "Cannot create table %s. Namespace does not exist: %s", - tableIdentifier, tableIdentifier.namespace()); - } - - PolarisResolvedPathWrapper resolvedTableEntities = - resolvedEntityView.getPassthroughResolvedPath( - tableIdentifier, PolarisEntitySubType.TABLE); - - // Fetch credentials for the resolved entity. The entity could be the table itself (if it has - // already been stored and credentials have been configured directly) or it could be the - // table's namespace or catalog. - PolarisResolvedPathWrapper resolvedStorageEntity = - resolvedTableEntities == null - ? resolvedEntityView.getResolvedPath(tableIdentifier.namespace()) - : resolvedTableEntities; - - // refresh credentials because we need to read the metadata file to validate its location - tableFileIO = - refreshIOWithCredentials( - tableIdentifier, - getLocationsAllowedToBeAccessed(metadata), - resolvedStorageEntity, - new HashMap<>(metadata.properties()), - Set.of(PolarisStorageActions.READ, PolarisStorageActions.WRITE)); - - List resolvedNamespace = - resolvedTableEntities == null - ? resolvedEntityView.getResolvedPath(tableIdentifier.namespace()).getRawFullPath() - : resolvedTableEntities.getRawParentPath(); - CatalogEntity catalog = CatalogEntity.of(resolvedNamespace.getFirst()); - - if (base == null - || !metadata.location().equals(base.location()) - || !Objects.equal( - base.properties().get(TableLikeEntity.USER_SPECIFIED_WRITE_DATA_LOCATION_KEY), - metadata.properties().get(TableLikeEntity.USER_SPECIFIED_WRITE_DATA_LOCATION_KEY))) { - // If location is changing then we must validate that the requested location is valid - // for the storage configuration inherited under this entity's path. - Set dataLocations = new HashSet<>(); - dataLocations.add(metadata.location()); - if (metadata.properties().get(TableLikeEntity.USER_SPECIFIED_WRITE_DATA_LOCATION_KEY) - != null) { - dataLocations.add( - metadata.properties().get(TableLikeEntity.USER_SPECIFIED_WRITE_DATA_LOCATION_KEY)); - } - if (metadata.properties().get(TableLikeEntity.USER_SPECIFIED_WRITE_METADATA_LOCATION_KEY) - != null) { - dataLocations.add( - metadata - .properties() - .get(TableLikeEntity.USER_SPECIFIED_WRITE_METADATA_LOCATION_KEY)); - } - validateLocationsForTableLike(tableIdentifier, dataLocations, resolvedStorageEntity); - // also validate that the table location doesn't overlap an existing table - dataLocations.forEach( - location -> - validateNoLocationOverlap( - catalogEntity, tableIdentifier, resolvedNamespace, location)); - // and that the metadata file points to a location within the table's directory structure - if (metadata.metadataFileLocation() != null) { - validateMetadataFileInTableDir(tableIdentifier, metadata, catalog); - } - } - - String newLocation = writeNewMetadataIfRequired(base == null, metadata); - String oldLocation = base == null ? null : base.metadataFileLocation(); - - PolarisResolvedPathWrapper resolvedView = - resolvedEntityView.getPassthroughResolvedPath(tableIdentifier, PolarisEntitySubType.VIEW); - if (resolvedView != null) { - throw new AlreadyExistsException("View with same name already exists: %s", tableIdentifier); - } - - // TODO: Consider using the entity from doRefresh() directly to do the conflict detection - // instead of a two-layer CAS (checking metadataLocation to detect concurrent modification - // between doRefresh() and doCommit(), and then updateEntityPropertiesIfNotChanged to detect - // concurrent - // modification between our checking of unchanged metadataLocation here and actual - // persistence-layer commit). - PolarisResolvedPathWrapper resolvedEntities = - resolvedEntityView.getPassthroughResolvedPath( - tableIdentifier, PolarisEntitySubType.TABLE); - TableLikeEntity entity = - TableLikeEntity.of(resolvedEntities == null ? null : resolvedEntities.getRawLeafEntity()); - String existingLocation; - if (null == entity) { - existingLocation = null; - entity = - new TableLikeEntity.Builder(tableIdentifier, newLocation) - .setCatalogId(getCatalogId()) - .setSubType(PolarisEntitySubType.TABLE) - .setBaseLocation(metadata.location()) - .setId( - getMetaStoreManager().generateNewEntityId(getCurrentPolarisContext()).getId()) - .build(); - } else { - existingLocation = entity.getMetadataLocation(); - entity = - new TableLikeEntity.Builder(entity) - .setBaseLocation(metadata.location()) - .setMetadataLocation(newLocation) - .build(); - } - if (!Objects.equal(existingLocation, oldLocation)) { - if (null == base) { - throw new AlreadyExistsException("Table already exists: %s", tableName()); - } - - if (null == existingLocation) { - throw new NoSuchTableException("Table does not exist: %s", tableName()); - } - - throw new CommitFailedException( - "Cannot commit to table %s metadata location from %s to %s " - + "because it has been concurrently modified to %s", - tableIdentifier, oldLocation, newLocation, existingLocation); - } - if (null == existingLocation) { - createTableLike(tableIdentifier, entity); - } else { - updateTableLike(tableIdentifier, entity); - } - } - - @Override - public FileIO io() { - return tableFileIO; - } - - @Override - protected String tableName() { - return fullTableName; - } - } - - private void validateMetadataFileInTableDir( - TableIdentifier identifier, TableMetadata metadata, CatalogEntity catalog) { - PolarisCallContext polarisCallContext = callContext.getPolarisCallContext(); - boolean allowEscape = - polarisCallContext - .getConfigurationStore() - .getConfiguration( - polarisCallContext, PolarisConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION); - if (!allowEscape - && !polarisCallContext - .getConfigurationStore() - .getConfiguration( - polarisCallContext, PolarisConfiguration.ALLOW_EXTERNAL_METADATA_FILE_LOCATION)) { - LOGGER.debug( - "Validating base location {} for table {} in metadata file {}", - metadata.location(), - identifier, - metadata.metadataFileLocation()); - StorageLocation metadataFileLocation = StorageLocation.of(metadata.metadataFileLocation()); - StorageLocation baseLocation = StorageLocation.of(metadata.location()); - if (!metadataFileLocation.isChildOf(baseLocation)) { - throw new BadRequestException( - "Metadata location %s is not allowed outside of table location %s", - metadata.metadataFileLocation(), metadata.location()); - } - } - } - - private static @Nonnull Optional findStorageInfoFromHierarchy( - PolarisResolvedPathWrapper resolvedStorageEntity) { - Optional storageInfoEntity = - resolvedStorageEntity.getRawFullPath().reversed().stream() - .filter( - e -> - e.getInternalPropertiesAsMap() - .containsKey(PolarisEntityConstants.getStorageConfigInfoPropertyName())) - .findFirst(); - return storageInfoEntity; - } - - private class BasePolarisViewOperations extends BaseViewOperations { - private final TableIdentifier identifier; - private final String fullViewName; - private FileIO viewFileIO; - - BasePolarisViewOperations(FileIO defaultFileIO, TableIdentifier identifier) { - this.viewFileIO = defaultFileIO; - this.identifier = identifier; - this.fullViewName = ViewUtil.fullViewName(catalogName, identifier); - } - - @Override - public void doRefresh() { - PolarisResolvedPathWrapper resolvedEntities = - resolvedEntityView.getPassthroughResolvedPath(identifier, PolarisEntitySubType.VIEW); - TableLikeEntity entity = null; - - if (resolvedEntities != null) { - entity = TableLikeEntity.of(resolvedEntities.getRawLeafEntity()); - if (!identifier.equals(entity.getTableIdentifier())) { - LOGGER - .atError() - .addKeyValue("entity.getTableIdentifier()", entity.getTableIdentifier()) - .addKeyValue("identifier", identifier) - .log("Stored view identifier mismatches requested identifier"); - } - } - - String latestLocation = entity != null ? entity.getMetadataLocation() : null; - LOGGER.debug("Refreshing view latestLocation: {}", latestLocation); - if (latestLocation == null) { - disableRefresh(); - } else { - refreshFromMetadataLocation( - latestLocation, - SHOULD_RETRY_REFRESH_PREDICATE, - MAX_RETRIES, - metadataLocation -> { - String latestLocationDir = - latestLocation.substring(0, latestLocation.lastIndexOf('/')); - - // TODO: Once we have the "current" table properties pulled into the resolvedEntity - // then we should use the actual current table properties for IO refresh here - // instead of the general tableDefaultProperties. - FileIO fileIO = - refreshIOWithCredentials( - identifier, - Set.of(latestLocationDir), - resolvedEntities, - new HashMap<>(tableDefaultProperties), - Set.of(PolarisStorageActions.READ)); - - return ViewMetadataParser.read(fileIO.newInputFile(metadataLocation)); - }); - } - } - - @Override - public void doCommit(ViewMetadata base, ViewMetadata metadata) { - // TODO: Maybe avoid writing metadata if there's definitely a transaction conflict - LOGGER.debug("doCommit for view {} with base {}, metadata {}", identifier, base, metadata); - if (null == base && !namespaceExists(identifier.namespace())) { - throw new NoSuchNamespaceException( - "Cannot create view %s. Namespace does not exist: %s", - identifier, identifier.namespace()); - } - - PolarisResolvedPathWrapper resolvedTable = - resolvedEntityView.getPassthroughResolvedPath(identifier, PolarisEntitySubType.TABLE); - if (resolvedTable != null) { - throw new AlreadyExistsException("Table with same name already exists: %s", identifier); - } - - PolarisResolvedPathWrapper resolvedEntities = - resolvedEntityView.getPassthroughResolvedPath(identifier, PolarisEntitySubType.VIEW); - - // Fetch credentials for the resolved entity. The entity could be the view itself (if it has - // already been stored and credentials have been configured directly) or it could be the - // table's namespace or catalog. - PolarisResolvedPathWrapper resolvedStorageEntity = - resolvedEntities == null - ? resolvedEntityView.getResolvedPath(identifier.namespace()) - : resolvedEntities; - - List resolvedNamespace = - resolvedEntities == null - ? resolvedEntityView.getResolvedPath(identifier.namespace()).getRawFullPath() - : resolvedEntities.getRawParentPath(); - if (base == null || !metadata.location().equals(base.location())) { - // If location is changing then we must validate that the requested location is valid - // for the storage configuration inherited under this entity's path. - validateLocationForTableLike(identifier, metadata.location(), resolvedStorageEntity); - validateNoLocationOverlap( - catalogEntity, identifier, resolvedNamespace, metadata.location()); - } - - Map tableProperties = new HashMap<>(metadata.properties()); - - viewFileIO = - refreshIOWithCredentials( - identifier, - getLocationsAllowedToBeAccessed(metadata), - resolvedStorageEntity, - tableProperties, - Set.of(PolarisStorageActions.READ, PolarisStorageActions.WRITE)); - - String newLocation = writeNewMetadataIfRequired(metadata); - String oldLocation = base == null ? null : currentMetadataLocation(); - - TableLikeEntity entity = - TableLikeEntity.of(resolvedEntities == null ? null : resolvedEntities.getRawLeafEntity()); - String existingLocation; - if (null == entity) { - existingLocation = null; - entity = - new TableLikeEntity.Builder(identifier, newLocation) - .setCatalogId(getCatalogId()) - .setSubType(PolarisEntitySubType.VIEW) - .setId( - getMetaStoreManager().generateNewEntityId(getCurrentPolarisContext()).getId()) - .build(); - } else { - existingLocation = entity.getMetadataLocation(); - entity = new TableLikeEntity.Builder(entity).setMetadataLocation(newLocation).build(); - } - if (!Objects.equal(existingLocation, oldLocation)) { - if (null == base) { - throw new AlreadyExistsException("View already exists: %s", identifier); - } - - if (null == existingLocation) { - throw new NoSuchViewException("View does not exist: %s", identifier); - } - - throw new CommitFailedException( - "Cannot commit to view %s metadata location from %s to %s " - + "because it has been concurrently modified to %s", - identifier, oldLocation, newLocation, existingLocation); - } - if (null == existingLocation) { - createTableLike(identifier, entity); - } else { - updateTableLike(identifier, entity); - } - } - - @Override - public FileIO io() { - return viewFileIO; - } - - @Override - protected String viewName() { - return fullViewName; - } - } - - private FileIO refreshIOWithCredentials( - TableIdentifier identifier, - Set readLocations, - PolarisResolvedPathWrapper resolvedStorageEntity, - Map tableProperties, - Set storageActions) { - Optional storageInfoEntity = findStorageInfoFromHierarchy(resolvedStorageEntity); - Map credentialsMap = - storageInfoEntity - .map( - storageInfo -> - refreshCredentials(identifier, storageActions, readLocations, storageInfo)) - .orElse(Map.of()); - - // Update the FileIO before we write the new metadata file - // update with table properties in case there are table-level overrides - // the credentials should always override table-level properties, since - // storage configuration will be found at whatever entity defines it - tableProperties.putAll(credentialsMap); - FileIO fileIO = null; - fileIO = loadFileIO(ioImplClassName, tableProperties); - // ensure the new fileIO is closed when the catalog is closed - closeableGroup.addCloseable(fileIO); - return fileIO; - } - - private PolarisCallContext getCurrentPolarisContext() { - return callContext.getPolarisCallContext(); - } - - private PolarisMetaStoreManager getMetaStoreManager() { - return metaStoreManager; - } - - private PolarisCredentialVendor getCredentialVendor() { - return metaStoreManager; - } - - @VisibleForTesting - void setFileIOFactory(FileIOFactory newFactory) { - this.fileIOFactory = newFactory; - } - - @VisibleForTesting - long getCatalogId() { - // TODO: Properly handle initialization - if (catalogId <= 0) { - throw new RuntimeException( - "Failed to initialize catalogId before using catalog with name: " + catalogName); - } - return catalogId; - } - - private void renameTableLike( - PolarisEntitySubType subType, TableIdentifier from, TableIdentifier to) { - LOGGER.debug("Renaming tableLike from {} to {}", from, to); - PolarisResolvedPathWrapper resolvedEntities = resolvedEntityView.getResolvedPath(from, subType); - if (resolvedEntities == null) { - if (subType == PolarisEntitySubType.VIEW) { - throw new NoSuchViewException("Cannot rename %s to %s. View does not exist", from, to); - } else { - throw new NoSuchTableException("Cannot rename %s to %s. Table does not exist", from, to); - } - } - List catalogPath = resolvedEntities.getRawParentPath(); - PolarisEntity leafEntity = resolvedEntities.getRawLeafEntity(); - final TableLikeEntity toEntity; - List newCatalogPath = null; - if (!from.namespace().equals(to.namespace())) { - PolarisResolvedPathWrapper resolvedNewParentEntities = - resolvedEntityView.getResolvedPath(to.namespace()); - if (resolvedNewParentEntities == null) { - throw new NoSuchNamespaceException( - "Cannot rename %s to %s. Namespace does not exist: %s", from, to, to.namespace()); - } - newCatalogPath = resolvedNewParentEntities.getRawFullPath(); - - // the "to" table has a new parent and a new name / namespace path - toEntity = - new TableLikeEntity.Builder(TableLikeEntity.of(leafEntity)) - .setTableIdentifier(to) - .setParentId(resolvedNewParentEntities.getResolvedLeafEntity().getEntity().getId()) - .build(); - } else { - // only the name of the entity is changed - toEntity = - new TableLikeEntity.Builder(TableLikeEntity.of(leafEntity)) - .setTableIdentifier(to) - .build(); - } - - // rename the entity now - PolarisMetaStoreManager.EntityResult returnedEntityResult = - getMetaStoreManager() - .renameEntity( - getCurrentPolarisContext(), - PolarisEntity.toCoreList(catalogPath), - leafEntity, - PolarisEntity.toCoreList(newCatalogPath), - toEntity); - - // handle error - if (!returnedEntityResult.isSuccess()) { - LOGGER.debug( - "Rename error {} trying to rename {} to {}. Checking existing object.", - returnedEntityResult.getReturnStatus(), - from, - to); - switch (returnedEntityResult.getReturnStatus()) { - case BaseResult.ReturnStatus.ENTITY_ALREADY_EXISTS: - { - PolarisEntitySubType existingEntitySubType = - returnedEntityResult.getAlreadyExistsEntitySubType(); - if (existingEntitySubType == null) { - // this code path is unexpected - throw new AlreadyExistsException( - "Cannot rename %s to %s. Object already exists", from, to); - } else if (existingEntitySubType == PolarisEntitySubType.TABLE) { - throw new AlreadyExistsException( - "Cannot rename %s to %s. Table already exists", from, to); - } else if (existingEntitySubType == PolarisEntitySubType.VIEW) { - throw new AlreadyExistsException( - "Cannot rename %s to %s. View already exists", from, to); - } - throw new IllegalStateException( - String.format("Unexpected entity type '%s'", existingEntitySubType)); - } - - case BaseResult.ReturnStatus.ENTITY_NOT_FOUND: - throw new NotFoundException("Cannot rename %s to %s. %s does not exist", from, to, from); - - // this is temporary. Should throw a special error that will be caught and retried - case BaseResult.ReturnStatus.TARGET_ENTITY_CONCURRENTLY_MODIFIED: - case BaseResult.ReturnStatus.ENTITY_CANNOT_BE_RESOLVED: - throw new RuntimeException("concurrent update detected, please retry"); - - // some entities cannot be renamed - case BaseResult.ReturnStatus.ENTITY_CANNOT_BE_RENAMED: - throw new BadRequestException("Cannot rename built-in object %s", leafEntity.getName()); - - // some entities cannot be renamed - default: - throw new IllegalStateException( - "Unknown error status " + returnedEntityResult.getReturnStatus()); - } - } else { - TableLikeEntity returnedEntity = TableLikeEntity.of(returnedEntityResult.getEntity()); - if (!toEntity.getTableIdentifier().equals(returnedEntity.getTableIdentifier())) { - // As long as there are older deployments which don't support the atomic update of the - // internalProperties during rename, we can log and then patch it up explicitly - // in a best-effort way. - LOGGER - .atError() - .addKeyValue("toEntity.getTableIdentifier()", toEntity.getTableIdentifier()) - .addKeyValue("returnedEntity.getTableIdentifier()", returnedEntity.getTableIdentifier()) - .log("Returned entity identifier doesn't match toEntity identifier"); - getMetaStoreManager() - .updateEntityPropertiesIfNotChanged( - getCurrentPolarisContext(), - PolarisEntity.toCoreList(newCatalogPath), - new TableLikeEntity.Builder(returnedEntity).setTableIdentifier(to).build()); - } - } - } - - /** - * Caller must fill in all entity fields except parentId, since the caller may not want to - * duplicate the logic to try to resolve parentIds before constructing the proposed entity. This - * method will fill in the parentId if needed upon resolution. - */ - private void createTableLike(TableIdentifier identifier, PolarisEntity entity) { - PolarisResolvedPathWrapper resolvedParent = - resolvedEntityView.getResolvedPath(identifier.namespace()); - if (resolvedParent == null) { - // Illegal state because the namespace should've already been in the static resolution set. - throw new IllegalStateException( - String.format("Failed to fetch resolved parent for TableIdentifier '%s'", identifier)); - } - - createTableLike(identifier, entity, resolvedParent); - } - - private void createTableLike( - TableIdentifier identifier, PolarisEntity entity, PolarisResolvedPathWrapper resolvedParent) { - // Make sure the metadata file is valid for our allowed locations. - String metadataLocation = TableLikeEntity.of(entity).getMetadataLocation(); - validateLocationForTableLike(identifier, metadataLocation, resolvedParent); - - List catalogPath = resolvedParent.getRawFullPath(); - - if (entity.getParentId() <= 0) { - // TODO: Validate catalogPath size is at least 1 for catalog entity? - entity = - new PolarisEntity.Builder(entity) - .setParentId(resolvedParent.getRawLeafEntity().getId()) - .build(); - } - entity = - new PolarisEntity.Builder(entity).setCreateTimestamp(System.currentTimeMillis()).build(); - - PolarisEntity returnedEntity = - PolarisEntity.of( - getMetaStoreManager() - .createEntityIfNotExists( - getCurrentPolarisContext(), PolarisEntity.toCoreList(catalogPath), entity)); - LOGGER.debug("Created TableLike entity {} with TableIdentifier {}", entity, identifier); - if (returnedEntity == null) { - // TODO: Error or retry? - } - } - - private void updateTableLike(TableIdentifier identifier, PolarisEntity entity) { - PolarisResolvedPathWrapper resolvedEntities = - resolvedEntityView.getResolvedPath(identifier, entity.getSubType()); - if (resolvedEntities == null) { - // Illegal state because the identifier should've already been in the static resolution set. - throw new IllegalStateException( - String.format("Failed to fetch resolved TableIdentifier '%s'", identifier)); - } - - // Make sure the metadata file is valid for our allowed locations. - String metadataLocation = TableLikeEntity.of(entity).getMetadataLocation(); - validateLocationForTableLike(identifier, metadataLocation, resolvedEntities); - - List catalogPath = resolvedEntities.getRawParentPath(); - PolarisEntity returnedEntity = - Optional.ofNullable( - getMetaStoreManager() - .updateEntityPropertiesIfNotChanged( - getCurrentPolarisContext(), PolarisEntity.toCoreList(catalogPath), entity) - .getEntity()) - .map(PolarisEntity::new) - .orElse(null); - if (returnedEntity == null) { - // TODO: Error or retry? - } - } - - @SuppressWarnings("FormatStringAnnotation") - private @Nonnull PolarisMetaStoreManager.DropEntityResult dropTableLike( - PolarisEntitySubType subType, - TableIdentifier identifier, - Map storageProperties, - boolean purge) { - PolarisResolvedPathWrapper resolvedEntities = - resolvedEntityView.getResolvedPath(identifier, subType); - if (resolvedEntities == null) { - // TODO: Error? - return new PolarisMetaStoreManager.DropEntityResult( - BaseResult.ReturnStatus.ENTITY_NOT_FOUND, null); - } - - List catalogPath = resolvedEntities.getRawParentPath(); - PolarisEntity leafEntity = resolvedEntities.getRawLeafEntity(); - - // Check that purge is enabled, if it is set: - if (catalogPath != null && !catalogPath.isEmpty() && purge) { - boolean dropWithPurgeEnabled = - callContext - .getPolarisCallContext() - .getConfigurationStore() - .getConfiguration( - callContext.getPolarisCallContext(), - catalogEntity, - PolarisConfiguration.DROP_WITH_PURGE_ENABLED); - if (!dropWithPurgeEnabled) { - throw new ForbiddenException( - String.format( - "Unable to purge entity: %s. To enable this feature, set the Polaris configuration %s " - + "or the catalog configuration %s", - identifier.name(), - PolarisConfiguration.DROP_WITH_PURGE_ENABLED.key, - PolarisConfiguration.DROP_WITH_PURGE_ENABLED.catalogConfig())); - } - } - - return getMetaStoreManager() - .dropEntityIfExists( - getCurrentPolarisContext(), - PolarisEntity.toCoreList(catalogPath), - leafEntity, - storageProperties, - purge); - } - - private boolean sendNotificationForTableLike( - PolarisEntitySubType subType, TableIdentifier tableIdentifier, NotificationRequest request) { - LOGGER.debug( - "Handling notification request {} for tableIdentifier {}", request, tableIdentifier); - PolarisResolvedPathWrapper resolvedEntities = - resolvedEntityView.getPassthroughResolvedPath(tableIdentifier, subType); - - NotificationType notificationType = request.getNotificationType(); - - Preconditions.checkNotNull(notificationType, "Expected a valid notification type."); - - if (notificationType == NotificationType.DROP) { - return dropTableLike(PolarisEntitySubType.TABLE, tableIdentifier, Map.of(), false /* purge */) - .isSuccess(); - } else if (notificationType == NotificationType.VALIDATE) { - // In this mode we don't want to make any mutations, so we won't auto-create non-existing - // parent namespaces. This means when we want to validate allowedLocations for the proposed - // table metadata location, we must independently find the deepest non-null parent namespace - // of the TableIdentifier, which may even be the base CatalogEntity if no parent namespaces - // actually exist yet. We can then extract the right StorageInfo entity via a normal call - // to findStorageInfoFromHierarchy. - PolarisResolvedPathWrapper resolvedStorageEntity = null; - Optional storageInfoEntity = Optional.empty(); - for (int i = tableIdentifier.namespace().length(); i >= 0; i--) { - Namespace nsLevel = - Namespace.of( - Arrays.stream(tableIdentifier.namespace().levels()) - .limit(i) - .toArray(String[]::new)); - resolvedStorageEntity = resolvedEntityView.getResolvedPath(nsLevel); - if (resolvedStorageEntity != null) { - storageInfoEntity = findStorageInfoFromHierarchy(resolvedStorageEntity); - break; - } - } - - if (resolvedStorageEntity == null || storageInfoEntity.isEmpty()) { - throw new BadRequestException( - "Failed to find StorageInfo entity for TableIdentifier %s", tableIdentifier); - } - - // Validate location against the resolvedStorageEntity - String metadataLocation = - transformTableLikeLocation(request.getPayload().getMetadataLocation()); - validateLocationForTableLike(tableIdentifier, metadataLocation, resolvedStorageEntity); - - // Validate that we can construct a FileIO - String locationDir = metadataLocation.substring(0, metadataLocation.lastIndexOf("/")); - refreshIOWithCredentials( - tableIdentifier, - Set.of(locationDir), - resolvedStorageEntity, - new HashMap<>(tableDefaultProperties), - Set.of(PolarisStorageActions.READ)); - - LOGGER.debug( - "Successful VALIDATE notification for tableIdentifier {}, metadataLocation {}", - tableIdentifier, - metadataLocation); - } else if (notificationType == NotificationType.CREATE - || notificationType == NotificationType.UPDATE) { - - Namespace ns = tableIdentifier.namespace(); - createNonExistingNamespaces(ns); - - PolarisResolvedPathWrapper resolvedParent = resolvedEntityView.getPassthroughResolvedPath(ns); - - TableLikeEntity entity = - TableLikeEntity.of(resolvedEntities == null ? null : resolvedEntities.getRawLeafEntity()); - - String existingLocation; - String newLocation = transformTableLikeLocation(request.getPayload().getMetadataLocation()); - if (null == entity) { - existingLocation = null; - entity = - new TableLikeEntity.Builder(tableIdentifier, newLocation) - .setCatalogId(getCatalogId()) - .setSubType(PolarisEntitySubType.TABLE) - .setId( - getMetaStoreManager().generateNewEntityId(getCurrentPolarisContext()).getId()) - .setLastNotificationTimestamp(request.getPayload().getTimestamp()) - .build(); - } else { - // If the notification timestamp is out-of-order, we should not update the table - if (entity.getLastAdmittedNotificationTimestamp().isPresent() - && request.getPayload().getTimestamp() - <= entity.getLastAdmittedNotificationTimestamp().get()) { - throw new AlreadyExistsException( - "A notification with a newer timestamp has been processed for table %s", - tableIdentifier); - } - existingLocation = entity.getMetadataLocation(); - entity = - new TableLikeEntity.Builder(entity) - .setMetadataLocation(newLocation) - .setLastNotificationTimestamp(request.getPayload().getTimestamp()) - .build(); - } - // first validate we can read the metadata file - validateLocationForTableLike(tableIdentifier, newLocation); - - String locationDir = newLocation.substring(0, newLocation.lastIndexOf("/")); - - FileIO fileIO = - refreshIOWithCredentials( - tableIdentifier, - Set.of(locationDir), - resolvedParent, - new HashMap<>(tableDefaultProperties), - Set.of(PolarisStorageActions.READ, PolarisStorageActions.WRITE)); - TableMetadata tableMetadata = TableMetadataParser.read(fileIO, newLocation); - - // then validate that it points to a valid location for this table - validateLocationForTableLike(tableIdentifier, tableMetadata.location()); - - // finally, validate that the metadata file is within the table directory - validateMetadataFileInTableDir( - tableIdentifier, - tableMetadata, - CatalogEntity.of(resolvedParent.getRawFullPath().getFirst())); - - // TODO: These might fail due to concurrent update; we need to do a retry in those cases. - if (null == existingLocation) { - LOGGER.debug( - "Creating table {} for notification with metadataLocation {}", - tableIdentifier, - newLocation); - createTableLike(tableIdentifier, entity, resolvedParent); - } else { - LOGGER.debug( - "Updating table {} for notification with metadataLocation {}", - tableIdentifier, - newLocation); - - updateTableLike(tableIdentifier, entity); - } - } - return true; - } - - private void createNonExistingNamespaces(Namespace namespace) { - // Pre-create namespaces if they don't exist - for (int i = 1; i <= namespace.length(); i++) { - Namespace nsLevel = - Namespace.of(Arrays.stream(namespace.levels()).limit(i).toArray(String[]::new)); - if (resolvedEntityView.getPassthroughResolvedPath(nsLevel) == null) { - Namespace parentNamespace = PolarisCatalogHelpers.getParentNamespace(nsLevel); - PolarisResolvedPathWrapper resolvedParent = - resolvedEntityView.getPassthroughResolvedPath(parentNamespace); - createNamespaceInternal(nsLevel, Collections.emptyMap(), resolvedParent); - } - } - } - - private List listTableLike(PolarisEntitySubType subType, Namespace namespace) { - PolarisResolvedPathWrapper resolvedEntities = resolvedEntityView.getResolvedPath(namespace); - if (resolvedEntities == null) { - // Illegal state because the namespace should've already been in the static resolution set. - throw new IllegalStateException( - String.format("Failed to fetch resolved namespace '%s'", namespace)); - } - - List catalogPath = resolvedEntities.getRawFullPath(); - List entities = - PolarisEntity.toNameAndIdList( - getMetaStoreManager() - .listEntities( - getCurrentPolarisContext(), - PolarisEntity.toCoreList(catalogPath), - PolarisEntityType.TABLE_LIKE, - subType) - .getEntities()); - return PolarisCatalogHelpers.nameAndIdToTableIdentifiers(catalogPath, entities); - } - - /** - * Load FileIO with provided impl and properties - * - * @param ioImpl full class name of a custom FileIO implementation - * @param properties used to initialize the FileIO implementation - * @return FileIO object - */ - private FileIO loadFileIO(String ioImpl, Map properties) { - Map propertiesWithS3CustomizedClientFactory = new HashMap<>(properties); - propertiesWithS3CustomizedClientFactory.put( - S3FileIOProperties.CLIENT_FACTORY, PolarisS3FileIOClientFactory.class.getName()); - return fileIOFactory.loadFileIO(ioImpl, propertiesWithS3CustomizedClientFactory); - } - - private void blockedUserSpecifiedWriteLocation(Map properties) { - if (properties != null - && (properties.containsKey(TableLikeEntity.USER_SPECIFIED_WRITE_DATA_LOCATION_KEY) - || properties.containsKey( - TableLikeEntity.USER_SPECIFIED_WRITE_METADATA_LOCATION_KEY))) { - throw new ForbiddenException( - "Delegate access to table with user-specified write location is temporarily not supported."); - } - } - - /** Helper to retrieve dynamic context-based configuration that has a boolean value. */ - private Boolean getBooleanContextConfiguration(String configKey, boolean defaultValue) { - return callContext - .getPolarisCallContext() - .getConfigurationStore() - .getConfiguration(callContext.getPolarisCallContext(), configKey, defaultValue); - } - - /** - * Check if the exception is retryable for the storage provider - * - * @param ex exception - * @return true if the exception is retryable - */ - private static boolean isStorageProviderRetryableException(Exception ex) { - // For S3/Azure, the exception is not wrapped, while for GCP the exception is wrapped as a - // RuntimeException - Throwable rootCause = ExceptionUtils.getRootCause(ex); - if (rootCause == null) { - // no root cause, let it retry - return true; - } - // only S3 SdkException has this retryable property - if (rootCause instanceof SdkException && ((SdkException) rootCause).retryable()) { - return true; - } - // add more cases here if needed - // AccessDenied is not retryable - return !isAccessDenied(rootCause.getMessage()); - } - - private static boolean isAccessDenied(String errorMsg) { - // Corresponding error messages for storage providers Aws/Azure/Gcp - boolean isAccessDenied = - errorMsg != null && IcebergExceptionMapper.containsAnyAccessDeniedHint(errorMsg); - if (isAccessDenied) { - LOGGER.debug("Access Denied or Forbidden error: {}", errorMsg); - return true; - } - return false; - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java deleted file mode 100644 index 0b84fd268..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java +++ /dev/null @@ -1,476 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.catalog; - -import static org.apache.polaris.service.catalog.AccessDelegationMode.VENDED_CREDENTIALS; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; -import jakarta.enterprise.context.RequestScoped; -import jakarta.inject.Inject; -import jakarta.ws.rs.core.Response; -import jakarta.ws.rs.core.SecurityContext; -import java.net.URLEncoder; -import java.nio.charset.Charset; -import java.util.EnumSet; -import java.util.Map; -import java.util.Optional; -import org.apache.iceberg.catalog.Catalog; -import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.exceptions.BadRequestException; -import org.apache.iceberg.exceptions.NotAuthorizedException; -import org.apache.iceberg.exceptions.NotFoundException; -import org.apache.iceberg.rest.RESTUtil; -import org.apache.iceberg.rest.requests.CommitTransactionRequest; -import org.apache.iceberg.rest.requests.CreateNamespaceRequest; -import org.apache.iceberg.rest.requests.CreateTableRequest; -import org.apache.iceberg.rest.requests.CreateViewRequest; -import org.apache.iceberg.rest.requests.RegisterTableRequest; -import org.apache.iceberg.rest.requests.RenameTableRequest; -import org.apache.iceberg.rest.requests.ReportMetricsRequest; -import org.apache.iceberg.rest.requests.UpdateNamespacePropertiesRequest; -import org.apache.iceberg.rest.responses.ConfigResponse; -import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; -import org.apache.polaris.core.auth.PolarisAuthorizer; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.core.entity.PolarisEntity; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.core.persistence.PolarisEntityManager; -import org.apache.polaris.core.persistence.cache.EntityCacheEntry; -import org.apache.polaris.core.persistence.resolver.Resolver; -import org.apache.polaris.core.persistence.resolver.ResolverStatus; -import org.apache.polaris.service.catalog.api.IcebergRestCatalogApiService; -import org.apache.polaris.service.catalog.api.IcebergRestConfigurationApiService; -import org.apache.polaris.service.config.RealmEntityManagerFactory; -import org.apache.polaris.service.context.CallContextCatalogFactory; -import org.apache.polaris.service.types.CommitTableRequest; -import org.apache.polaris.service.types.CommitViewRequest; -import org.apache.polaris.service.types.NotificationRequest; - -/** - * {@link IcebergRestCatalogApiService} implementation that delegates operations to {@link - * org.apache.iceberg.rest.CatalogHandlers} after finding the appropriate {@link Catalog} for the - * current {@link RealmContext}. - */ -@RequestScoped -public class IcebergCatalogAdapter - implements IcebergRestCatalogApiService, IcebergRestConfigurationApiService { - - private final CallContextCatalogFactory catalogFactory; - private final MetaStoreManagerFactory metaStoreManagerFactory; - private final RealmEntityManagerFactory entityManagerFactory; - private final PolarisAuthorizer polarisAuthorizer; - - @Inject - public IcebergCatalogAdapter( - CallContextCatalogFactory catalogFactory, - RealmEntityManagerFactory entityManagerFactory, - MetaStoreManagerFactory metaStoreManagerFactory, - PolarisAuthorizer polarisAuthorizer) { - this.catalogFactory = catalogFactory; - this.entityManagerFactory = entityManagerFactory; - this.metaStoreManagerFactory = metaStoreManagerFactory; - this.polarisAuthorizer = polarisAuthorizer; - } - - private PolarisCatalogHandlerWrapper newHandlerWrapper( - SecurityContext securityContext, String catalogName) { - CallContext callContext = CallContext.getCurrentContext(); - AuthenticatedPolarisPrincipal authenticatedPrincipal = - (AuthenticatedPolarisPrincipal) securityContext.getUserPrincipal(); - if (authenticatedPrincipal == null) { - throw new NotAuthorizedException("Failed to find authenticatedPrincipal in SecurityContext"); - } - - PolarisEntityManager entityManager = - entityManagerFactory.getOrCreateEntityManager(callContext.getRealmContext()); - - return new PolarisCatalogHandlerWrapper( - callContext, - entityManager, - metaStoreManagerFactory.getOrCreateMetaStoreManager(callContext.getRealmContext()), - authenticatedPrincipal, - catalogFactory, - catalogName, - polarisAuthorizer); - } - - @Override - public Response createNamespace( - String prefix, - CreateNamespaceRequest createNamespaceRequest, - SecurityContext securityContext) { - return Response.ok( - newHandlerWrapper(securityContext, prefix).createNamespace(createNamespaceRequest)) - .build(); - } - - @Override - public Response listNamespaces( - String prefix, - String pageToken, - Integer pageSize, - String parent, - SecurityContext securityContext) { - Optional namespaceOptional = - Optional.ofNullable(parent).map(IcebergCatalogAdapter::decodeNamespace); - return Response.ok( - newHandlerWrapper(securityContext, prefix) - .listNamespaces(namespaceOptional.orElse(Namespace.of()))) - .build(); - } - - @Override - public Response loadNamespaceMetadata( - String prefix, String namespace, SecurityContext securityContext) { - Namespace ns = decodeNamespace(namespace); - return Response.ok(newHandlerWrapper(securityContext, prefix).loadNamespaceMetadata(ns)) - .build(); - } - - private static Namespace decodeNamespace(String namespace) { - return RESTUtil.decodeNamespace(URLEncoder.encode(namespace, Charset.defaultCharset())); - } - - @Override - public Response namespaceExists( - String prefix, String namespace, SecurityContext securityContext) { - Namespace ns = decodeNamespace(namespace); - newHandlerWrapper(securityContext, prefix).namespaceExists(ns); - return Response.ok().build(); - } - - @Override - public Response dropNamespace(String prefix, String namespace, SecurityContext securityContext) { - Namespace ns = decodeNamespace(namespace); - newHandlerWrapper(securityContext, prefix).dropNamespace(ns); - return Response.ok(Response.Status.NO_CONTENT).build(); - } - - @Override - public Response updateProperties( - String prefix, - String namespace, - UpdateNamespacePropertiesRequest updateNamespacePropertiesRequest, - SecurityContext securityContext) { - Namespace ns = decodeNamespace(namespace); - return Response.ok( - newHandlerWrapper(securityContext, prefix) - .updateNamespaceProperties(ns, updateNamespacePropertiesRequest)) - .build(); - } - - private EnumSet parseAccessDelegationModes(String accessDelegationMode) { - EnumSet delegationModes = - AccessDelegationMode.fromProtocolValuesList(accessDelegationMode); - Preconditions.checkArgument( - delegationModes.isEmpty() || delegationModes.contains(VENDED_CREDENTIALS), - "Unsupported access delegation mode: %s", - accessDelegationMode); - return delegationModes; - } - - @Override - public Response createTable( - String prefix, - String namespace, - CreateTableRequest createTableRequest, - String accessDelegationMode, - SecurityContext securityContext) { - EnumSet delegationModes = - parseAccessDelegationModes(accessDelegationMode); - Namespace ns = decodeNamespace(namespace); - if (createTableRequest.stageCreate()) { - if (delegationModes.isEmpty()) { - return Response.ok( - newHandlerWrapper(securityContext, prefix) - .createTableStaged(ns, createTableRequest)) - .build(); - } else { - return Response.ok( - newHandlerWrapper(securityContext, prefix) - .createTableStagedWithWriteDelegation(ns, createTableRequest)) - .build(); - } - } else if (delegationModes.isEmpty()) { - return Response.ok( - newHandlerWrapper(securityContext, prefix).createTableDirect(ns, createTableRequest)) - .build(); - } else { - return Response.ok( - newHandlerWrapper(securityContext, prefix) - .createTableDirectWithWriteDelegation(ns, createTableRequest)) - .build(); - } - } - - @Override - public Response listTables( - String prefix, - String namespace, - String pageToken, - Integer pageSize, - SecurityContext securityContext) { - Namespace ns = decodeNamespace(namespace); - return Response.ok(newHandlerWrapper(securityContext, prefix).listTables(ns)).build(); - } - - @Override - public Response loadTable( - String prefix, - String namespace, - String table, - String accessDelegationMode, - String snapshots, - SecurityContext securityContext) { - EnumSet delegationModes = - parseAccessDelegationModes(accessDelegationMode); - Namespace ns = decodeNamespace(namespace); - TableIdentifier tableIdentifier = TableIdentifier.of(ns, RESTUtil.decodeString(table)); - if (delegationModes.isEmpty()) { - return Response.ok( - newHandlerWrapper(securityContext, prefix).loadTable(tableIdentifier, snapshots)) - .build(); - } else { - return Response.ok( - newHandlerWrapper(securityContext, prefix) - .loadTableWithAccessDelegation(tableIdentifier, snapshots)) - .build(); - } - } - - @Override - public Response tableExists( - String prefix, String namespace, String table, SecurityContext securityContext) { - Namespace ns = decodeNamespace(namespace); - TableIdentifier tableIdentifier = TableIdentifier.of(ns, RESTUtil.decodeString(table)); - newHandlerWrapper(securityContext, prefix).tableExists(tableIdentifier); - return Response.ok().build(); - } - - @Override - public Response dropTable( - String prefix, - String namespace, - String table, - Boolean purgeRequested, - SecurityContext securityContext) { - Namespace ns = decodeNamespace(namespace); - TableIdentifier tableIdentifier = TableIdentifier.of(ns, RESTUtil.decodeString(table)); - - if (purgeRequested != null && purgeRequested) { - newHandlerWrapper(securityContext, prefix).dropTableWithPurge(tableIdentifier); - } else { - newHandlerWrapper(securityContext, prefix).dropTableWithoutPurge(tableIdentifier); - } - return Response.ok(Response.Status.NO_CONTENT).build(); - } - - @Override - public Response registerTable( - String prefix, - String namespace, - RegisterTableRequest registerTableRequest, - SecurityContext securityContext) { - Namespace ns = decodeNamespace(namespace); - return Response.ok( - newHandlerWrapper(securityContext, prefix).registerTable(ns, registerTableRequest)) - .build(); - } - - @Override - public Response renameTable( - String prefix, RenameTableRequest renameTableRequest, SecurityContext securityContext) { - newHandlerWrapper(securityContext, prefix).renameTable(renameTableRequest); - return Response.ok(Response.Status.NO_CONTENT).build(); - } - - @Override - public Response updateTable( - String prefix, - String namespace, - String table, - CommitTableRequest commitTableRequest, - SecurityContext securityContext) { - Namespace ns = decodeNamespace(namespace); - TableIdentifier tableIdentifier = TableIdentifier.of(ns, RESTUtil.decodeString(table)); - - if (PolarisCatalogHandlerWrapper.isCreate(commitTableRequest)) { - return Response.ok( - newHandlerWrapper(securityContext, prefix) - .updateTableForStagedCreate(tableIdentifier, commitTableRequest)) - .build(); - } else { - return Response.ok( - newHandlerWrapper(securityContext, prefix) - .updateTable(tableIdentifier, commitTableRequest)) - .build(); - } - } - - @Override - public Response createView( - String prefix, - String namespace, - CreateViewRequest createViewRequest, - SecurityContext securityContext) { - Namespace ns = decodeNamespace(namespace); - return Response.ok(newHandlerWrapper(securityContext, prefix).createView(ns, createViewRequest)) - .build(); - } - - @Override - public Response listViews( - String prefix, - String namespace, - String pageToken, - Integer pageSize, - SecurityContext securityContext) { - Namespace ns = decodeNamespace(namespace); - return Response.ok(newHandlerWrapper(securityContext, prefix).listViews(ns)).build(); - } - - @Override - public Response loadView( - String prefix, String namespace, String view, SecurityContext securityContext) { - Namespace ns = decodeNamespace(namespace); - TableIdentifier tableIdentifier = TableIdentifier.of(ns, RESTUtil.decodeString(view)); - return Response.ok(newHandlerWrapper(securityContext, prefix).loadView(tableIdentifier)) - .build(); - } - - @Override - public Response viewExists( - String prefix, String namespace, String view, SecurityContext securityContext) { - Namespace ns = decodeNamespace(namespace); - TableIdentifier tableIdentifier = TableIdentifier.of(ns, RESTUtil.decodeString(view)); - newHandlerWrapper(securityContext, prefix).viewExists(tableIdentifier); - return Response.ok().build(); - } - - @Override - public Response dropView( - String prefix, String namespace, String view, SecurityContext securityContext) { - Namespace ns = decodeNamespace(namespace); - TableIdentifier tableIdentifier = TableIdentifier.of(ns, RESTUtil.decodeString(view)); - newHandlerWrapper(securityContext, prefix).dropView(tableIdentifier); - return Response.ok(Response.Status.NO_CONTENT).build(); - } - - @Override - public Response renameView( - String prefix, RenameTableRequest renameTableRequest, SecurityContext securityContext) { - newHandlerWrapper(securityContext, prefix).renameView(renameTableRequest); - return Response.ok(Response.Status.NO_CONTENT).build(); - } - - @Override - public Response replaceView( - String prefix, - String namespace, - String view, - CommitViewRequest commitViewRequest, - SecurityContext securityContext) { - Namespace ns = decodeNamespace(namespace); - TableIdentifier tableIdentifier = TableIdentifier.of(ns, RESTUtil.decodeString(view)); - return Response.ok( - newHandlerWrapper(securityContext, prefix) - .replaceView(tableIdentifier, commitViewRequest)) - .build(); - } - - @Override - public Response commitTransaction( - String prefix, - CommitTransactionRequest commitTransactionRequest, - SecurityContext securityContext) { - newHandlerWrapper(securityContext, prefix).commitTransaction(commitTransactionRequest); - return Response.status(Response.Status.NO_CONTENT).build(); - } - - @Override - public Response reportMetrics( - String prefix, - String namespace, - String table, - ReportMetricsRequest reportMetricsRequest, - SecurityContext securityContext) { - return Response.status(Response.Status.NO_CONTENT).build(); - } - - @Override - public Response sendNotification( - String prefix, - String namespace, - String table, - NotificationRequest notificationRequest, - SecurityContext securityContext) { - Namespace ns = decodeNamespace(namespace); - TableIdentifier tableIdentifier = TableIdentifier.of(ns, RESTUtil.decodeString(table)); - newHandlerWrapper(securityContext, prefix) - .sendNotification(tableIdentifier, notificationRequest); - return Response.status(Response.Status.NO_CONTENT).build(); - } - - /** From IcebergRestConfigurationApiService. */ - @Override - public Response getConfig(String warehouse, SecurityContext securityContext) { - // 'warehouse' as an input here is catalogName. - // 'warehouse' as an output will be treated by the client as a default catalog - // storage - // base location. - // 'prefix' as an output is the REST subpath that routes to the catalog - // resource, - // which may be URL-escaped catalogName or potentially a different unique - // identifier for - // the catalog being accessed. - // TODO: Push this down into PolarisCatalogHandlerWrapper for authorizing "any" catalog - // role in this catalog. - PolarisEntityManager entityManager = - entityManagerFactory.getOrCreateEntityManager( - CallContext.getCurrentContext().getRealmContext()); - AuthenticatedPolarisPrincipal authenticatedPrincipal = - (AuthenticatedPolarisPrincipal) securityContext.getUserPrincipal(); - if (authenticatedPrincipal == null) { - throw new NotAuthorizedException("Failed to find authenticatedPrincipal in SecurityContext"); - } - if (warehouse == null) { - throw new BadRequestException("Please specify a warehouse"); - } - Resolver resolver = - entityManager.prepareResolver( - CallContext.getCurrentContext(), authenticatedPrincipal, warehouse); - ResolverStatus resolverStatus = resolver.resolveAll(); - if (!resolverStatus.getStatus().equals(ResolverStatus.StatusEnum.SUCCESS)) { - throw new NotFoundException("Unable to find warehouse %s", warehouse); - } - EntityCacheEntry resolvedReferenceCatalog = resolver.getResolvedReferenceCatalog(); - Map properties = - PolarisEntity.of(resolvedReferenceCatalog.getEntity()).getPropertiesAsMap(); - - return Response.ok( - ConfigResponse.builder() - .withDefaults(properties) // catalog properties are defaults - .withOverrides(ImmutableMap.of("prefix", warehouse)) - .build()) - .build(); - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java deleted file mode 100644 index b91ee0970..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java +++ /dev/null @@ -1,1150 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.catalog; - -import com.google.common.base.Preconditions; -import com.google.common.collect.Maps; -import java.io.Closeable; -import java.io.IOException; -import java.time.OffsetDateTime; -import java.time.ZoneOffset; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.function.Supplier; -import java.util.stream.Collectors; -import org.apache.iceberg.BaseMetadataTable; -import org.apache.iceberg.BaseTable; -import org.apache.iceberg.MetadataUpdate; -import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.SortOrder; -import org.apache.iceberg.Table; -import org.apache.iceberg.TableMetadata; -import org.apache.iceberg.TableOperations; -import org.apache.iceberg.UpdateRequirement; -import org.apache.iceberg.catalog.Catalog; -import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.catalog.SupportsNamespaces; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.catalog.ViewCatalog; -import org.apache.iceberg.exceptions.AlreadyExistsException; -import org.apache.iceberg.exceptions.BadRequestException; -import org.apache.iceberg.exceptions.CommitFailedException; -import org.apache.iceberg.exceptions.ForbiddenException; -import org.apache.iceberg.exceptions.NoSuchNamespaceException; -import org.apache.iceberg.exceptions.NoSuchTableException; -import org.apache.iceberg.exceptions.NoSuchViewException; -import org.apache.iceberg.rest.CatalogHandlers; -import org.apache.iceberg.rest.requests.CommitTransactionRequest; -import org.apache.iceberg.rest.requests.CreateNamespaceRequest; -import org.apache.iceberg.rest.requests.CreateTableRequest; -import org.apache.iceberg.rest.requests.CreateViewRequest; -import org.apache.iceberg.rest.requests.RegisterTableRequest; -import org.apache.iceberg.rest.requests.RenameTableRequest; -import org.apache.iceberg.rest.requests.UpdateNamespacePropertiesRequest; -import org.apache.iceberg.rest.requests.UpdateTableRequest; -import org.apache.iceberg.rest.responses.CreateNamespaceResponse; -import org.apache.iceberg.rest.responses.GetNamespaceResponse; -import org.apache.iceberg.rest.responses.ListNamespacesResponse; -import org.apache.iceberg.rest.responses.ListTablesResponse; -import org.apache.iceberg.rest.responses.LoadTableResponse; -import org.apache.iceberg.rest.responses.LoadViewResponse; -import org.apache.iceberg.rest.responses.UpdateNamespacePropertiesResponse; -import org.apache.polaris.core.PolarisConfiguration; -import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; -import org.apache.polaris.core.auth.PolarisAuthorizableOperation; -import org.apache.polaris.core.auth.PolarisAuthorizer; -import org.apache.polaris.core.catalog.PolarisCatalogHelpers; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.entity.CatalogEntity; -import org.apache.polaris.core.entity.PolarisEntitySubType; -import org.apache.polaris.core.entity.PolarisEntityType; -import org.apache.polaris.core.persistence.PolarisEntityManager; -import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; -import org.apache.polaris.core.persistence.TransactionWorkspaceMetaStoreManager; -import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest; -import org.apache.polaris.core.persistence.resolver.ResolverPath; -import org.apache.polaris.core.persistence.resolver.ResolverStatus; -import org.apache.polaris.core.storage.PolarisStorageActions; -import org.apache.polaris.service.context.CallContextCatalogFactory; -import org.apache.polaris.service.types.NotificationRequest; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Authorization-aware adapter between REST stubs and shared Iceberg SDK CatalogHandlers. - * - *

We must make authorization decisions based on entity resolution at this layer instead of the - * underlying BasePolarisCatalog layer, because this REST-adjacent layer captures intent of - * different REST calls that share underlying catalog calls (e.g. updateTable will call loadTable - * under the hood), and some features of the REST API aren't expressed at all in the underlying - * Catalog interfaces (e.g. credential-vending in createTable/loadTable). - * - *

We also want this layer to be independent of API-endpoint-specific idioms, such as dealing - * with jakarta.ws.rs.core.Response objects, and other implementations that expose different HTTP - * stubs or even tunnel the protocol over something like gRPC can still normalize on the Iceberg - * model objects used in this layer to still benefit from the shared implementation of - * authorization-aware catalog protocols. - */ -public class PolarisCatalogHandlerWrapper { - private static final Logger LOGGER = LoggerFactory.getLogger(PolarisCatalogHandlerWrapper.class); - - private final CallContext callContext; - private final PolarisEntityManager entityManager; - private final PolarisMetaStoreManager metaStoreManager; - private final String catalogName; - private final AuthenticatedPolarisPrincipal authenticatedPrincipal; - private final PolarisAuthorizer authorizer; - private final CallContextCatalogFactory catalogFactory; - - // Initialized in the authorize methods. - private PolarisResolutionManifest resolutionManifest = null; - - // Catalog instance will be initialized after authorizing resolver successfully resolves - // the catalog entity. - private Catalog baseCatalog = null; - private SupportsNamespaces namespaceCatalog = null; - private ViewCatalog viewCatalog = null; - - public PolarisCatalogHandlerWrapper( - CallContext callContext, - PolarisEntityManager entityManager, - PolarisMetaStoreManager metaStoreManager, - AuthenticatedPolarisPrincipal authenticatedPrincipal, - CallContextCatalogFactory catalogFactory, - String catalogName, - PolarisAuthorizer authorizer) { - this.callContext = callContext; - this.entityManager = entityManager; - this.metaStoreManager = metaStoreManager; - this.catalogName = catalogName; - this.authenticatedPrincipal = authenticatedPrincipal; - this.authorizer = authorizer; - this.catalogFactory = catalogFactory; - } - - /** - * TODO: Make the helper in org.apache.iceberg.rest.CatalogHandlers public instead of needing to - * copy/paste here. - */ - public static boolean isCreate(UpdateTableRequest request) { - boolean isCreate = - request.requirements().stream() - .anyMatch(UpdateRequirement.AssertTableDoesNotExist.class::isInstance); - - if (isCreate) { - List invalidRequirements = - request.requirements().stream() - .filter(req -> !(req instanceof UpdateRequirement.AssertTableDoesNotExist)) - .collect(Collectors.toList()); - Preconditions.checkArgument( - invalidRequirements.isEmpty(), "Invalid create requirements: %s", invalidRequirements); - } - - return isCreate; - } - - private void initializeCatalog() { - this.baseCatalog = - catalogFactory.createCallContextCatalog( - callContext, authenticatedPrincipal, resolutionManifest); - this.namespaceCatalog = - (baseCatalog instanceof SupportsNamespaces) ? (SupportsNamespaces) baseCatalog : null; - this.viewCatalog = (baseCatalog instanceof ViewCatalog) ? (ViewCatalog) baseCatalog : null; - } - - private void authorizeBasicNamespaceOperationOrThrow( - PolarisAuthorizableOperation op, Namespace namespace) { - authorizeBasicNamespaceOperationOrThrow(op, namespace, null, null); - } - - private void authorizeBasicNamespaceOperationOrThrow( - PolarisAuthorizableOperation op, - Namespace namespace, - List extraPassthroughNamespaces, - List extraPassthroughTableLikes) { - resolutionManifest = - entityManager.prepareResolutionManifest(callContext, authenticatedPrincipal, catalogName); - resolutionManifest.addPath( - new ResolverPath(Arrays.asList(namespace.levels()), PolarisEntityType.NAMESPACE), - namespace); - - if (extraPassthroughNamespaces != null) { - for (Namespace ns : extraPassthroughNamespaces) { - resolutionManifest.addPassthroughPath( - new ResolverPath( - Arrays.asList(ns.levels()), PolarisEntityType.NAMESPACE, true /* optional */), - ns); - } - } - if (extraPassthroughTableLikes != null) { - for (TableIdentifier id : extraPassthroughTableLikes) { - resolutionManifest.addPassthroughPath( - new ResolverPath( - PolarisCatalogHelpers.tableIdentifierToList(id), - PolarisEntityType.TABLE_LIKE, - true /* optional */), - id); - } - } - resolutionManifest.resolveAll(); - PolarisResolvedPathWrapper target = resolutionManifest.getResolvedPath(namespace, true); - if (target == null) { - throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace); - } - authorizer.authorizeOrThrow( - authenticatedPrincipal, - resolutionManifest.getAllActivatedCatalogRoleAndPrincipalRoles(), - op, - target, - null /* secondary */); - - initializeCatalog(); - } - - private void authorizeCreateNamespaceUnderNamespaceOperationOrThrow( - PolarisAuthorizableOperation op, Namespace namespace) { - resolutionManifest = - entityManager.prepareResolutionManifest(callContext, authenticatedPrincipal, catalogName); - - Namespace parentNamespace = PolarisCatalogHelpers.getParentNamespace(namespace); - resolutionManifest.addPath( - new ResolverPath(Arrays.asList(parentNamespace.levels()), PolarisEntityType.NAMESPACE), - parentNamespace); - - // When creating an entity under a namespace, the authz target is the parentNamespace, but we - // must also add the actual path that will be created as an "optional" passthrough resolution - // path to indicate that the underlying catalog is "allowed" to check the creation path for - // a conflicting entity. - resolutionManifest.addPassthroughPath( - new ResolverPath( - Arrays.asList(namespace.levels()), PolarisEntityType.NAMESPACE, true /* optional */), - namespace); - resolutionManifest.resolveAll(); - PolarisResolvedPathWrapper target = resolutionManifest.getResolvedPath(parentNamespace, true); - if (target == null) { - throw new NoSuchNamespaceException("Namespace does not exist: %s", parentNamespace); - } - authorizer.authorizeOrThrow( - authenticatedPrincipal, - resolutionManifest.getAllActivatedCatalogRoleAndPrincipalRoles(), - op, - target, - null /* secondary */); - - initializeCatalog(); - } - - private void authorizeCreateTableLikeUnderNamespaceOperationOrThrow( - PolarisAuthorizableOperation op, TableIdentifier identifier) { - Namespace namespace = identifier.namespace(); - - resolutionManifest = - entityManager.prepareResolutionManifest(callContext, authenticatedPrincipal, catalogName); - resolutionManifest.addPath( - new ResolverPath(Arrays.asList(namespace.levels()), PolarisEntityType.NAMESPACE), - namespace); - - // When creating an entity under a namespace, the authz target is the namespace, but we must - // also - // add the actual path that will be created as an "optional" passthrough resolution path to - // indicate that the underlying catalog is "allowed" to check the creation path for a - // conflicting - // entity. - resolutionManifest.addPassthroughPath( - new ResolverPath( - PolarisCatalogHelpers.tableIdentifierToList(identifier), - PolarisEntityType.TABLE_LIKE, - true /* optional */), - identifier); - resolutionManifest.resolveAll(); - PolarisResolvedPathWrapper target = resolutionManifest.getResolvedPath(namespace, true); - if (target == null) { - throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace); - } - authorizer.authorizeOrThrow( - authenticatedPrincipal, - resolutionManifest.getAllActivatedCatalogRoleAndPrincipalRoles(), - op, - target, - null /* secondary */); - - initializeCatalog(); - } - - private void authorizeBasicTableLikeOperationOrThrow( - PolarisAuthorizableOperation op, PolarisEntitySubType subType, TableIdentifier identifier) { - resolutionManifest = - entityManager.prepareResolutionManifest(callContext, authenticatedPrincipal, catalogName); - - // The underlying Catalog is also allowed to fetch "fresh" versions of the target entity. - resolutionManifest.addPassthroughPath( - new ResolverPath( - PolarisCatalogHelpers.tableIdentifierToList(identifier), - PolarisEntityType.TABLE_LIKE, - true /* optional */), - identifier); - resolutionManifest.resolveAll(); - PolarisResolvedPathWrapper target = - resolutionManifest.getResolvedPath(identifier, subType, true); - if (target == null) { - if (subType == PolarisEntitySubType.TABLE) { - throw new NoSuchTableException("Table does not exist: %s", identifier); - } else { - throw new NoSuchViewException("View does not exist: %s", identifier); - } - } - authorizer.authorizeOrThrow( - authenticatedPrincipal, - resolutionManifest.getAllActivatedCatalogRoleAndPrincipalRoles(), - op, - target, - null /* secondary */); - - initializeCatalog(); - } - - private void authorizeCollectionOfTableLikeOperationOrThrow( - PolarisAuthorizableOperation op, - final PolarisEntitySubType subType, - List ids) { - resolutionManifest = - entityManager.prepareResolutionManifest(callContext, authenticatedPrincipal, catalogName); - ids.forEach( - identifier -> - resolutionManifest.addPassthroughPath( - new ResolverPath( - PolarisCatalogHelpers.tableIdentifierToList(identifier), - PolarisEntityType.TABLE_LIKE), - identifier)); - - ResolverStatus status = resolutionManifest.resolveAll(); - - // If one of the paths failed to resolve, throw exception based on the one that - // we first failed to resolve. - if (status.getStatus() == ResolverStatus.StatusEnum.PATH_COULD_NOT_BE_FULLY_RESOLVED) { - TableIdentifier identifier = - PolarisCatalogHelpers.listToTableIdentifier( - status.getFailedToResolvePath().getEntityNames()); - if (subType == PolarisEntitySubType.TABLE) { - throw new NoSuchTableException("Table does not exist: %s", identifier); - } else { - throw new NoSuchViewException("View does not exist: %s", identifier); - } - } - - List targets = - ids.stream() - .map( - identifier -> - Optional.ofNullable( - resolutionManifest.getResolvedPath(identifier, subType, true)) - .orElseThrow( - () -> - subType == PolarisEntitySubType.TABLE - ? new NoSuchTableException( - "Table does not exist: %s", identifier) - : new NoSuchViewException( - "View does not exist: %s", identifier))) - .toList(); - authorizer.authorizeOrThrow( - authenticatedPrincipal, - resolutionManifest.getAllActivatedCatalogRoleAndPrincipalRoles(), - op, - targets, - null /* secondaries */); - - initializeCatalog(); - } - - private void authorizeRenameTableLikeOperationOrThrow( - PolarisAuthorizableOperation op, - PolarisEntitySubType subType, - TableIdentifier src, - TableIdentifier dst) { - resolutionManifest = - entityManager.prepareResolutionManifest(callContext, authenticatedPrincipal, catalogName); - // Add src, dstParent, and dst(optional) - resolutionManifest.addPath( - new ResolverPath( - PolarisCatalogHelpers.tableIdentifierToList(src), PolarisEntityType.TABLE_LIKE), - src); - resolutionManifest.addPath( - new ResolverPath(Arrays.asList(dst.namespace().levels()), PolarisEntityType.NAMESPACE), - dst.namespace()); - resolutionManifest.addPath( - new ResolverPath( - PolarisCatalogHelpers.tableIdentifierToList(dst), - PolarisEntityType.TABLE_LIKE, - true /* optional */), - dst); - ResolverStatus status = resolutionManifest.resolveAll(); - if (status.getStatus() == ResolverStatus.StatusEnum.PATH_COULD_NOT_BE_FULLY_RESOLVED - && status.getFailedToResolvePath().getLastEntityType() == PolarisEntityType.NAMESPACE) { - throw new NoSuchNamespaceException("Namespace does not exist: %s", dst.namespace()); - } else if (resolutionManifest.getResolvedPath(src, subType) == null) { - if (subType == PolarisEntitySubType.TABLE) { - throw new NoSuchTableException("Table does not exist: %s", src); - } else { - throw new NoSuchViewException("View does not exist: %s", src); - } - } - - // Normally, since we added the dst as an optional path, we'd expect it to only get resolved - // up to its parent namespace, and for there to be no TABLE_LIKE already in the dst in which - // case the leafSubType will be NULL_SUBTYPE. - // If there is a conflicting TABLE or VIEW, this leafSubType will indicate that conflicting - // type. - // TODO: Possibly modify the exception thrown depending on whether the caller has privileges - // on the parent namespace. - PolarisEntitySubType dstLeafSubType = resolutionManifest.getLeafSubType(dst); - if (dstLeafSubType == PolarisEntitySubType.TABLE) { - throw new AlreadyExistsException("Cannot rename %s to %s. Table already exists", src, dst); - } else if (dstLeafSubType == PolarisEntitySubType.VIEW) { - throw new AlreadyExistsException("Cannot rename %s to %s. View already exists", src, dst); - } - - PolarisResolvedPathWrapper target = resolutionManifest.getResolvedPath(src, subType, true); - PolarisResolvedPathWrapper secondary = - resolutionManifest.getResolvedPath(dst.namespace(), true); - authorizer.authorizeOrThrow( - authenticatedPrincipal, - resolutionManifest.getAllActivatedCatalogRoleAndPrincipalRoles(), - op, - target, - secondary); - - initializeCatalog(); - } - - public ListNamespacesResponse listNamespaces(Namespace parent) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.LIST_NAMESPACES; - authorizeBasicNamespaceOperationOrThrow(op, parent); - - return doCatalogOperation(() -> CatalogHandlers.listNamespaces(namespaceCatalog, parent)); - } - - public CreateNamespaceResponse createNamespace(CreateNamespaceRequest request) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.CREATE_NAMESPACE; - - Namespace namespace = request.namespace(); - if (namespace.isEmpty()) { - throw new AlreadyExistsException( - "Cannot create root namespace, as it already exists implicitly."); - } - authorizeCreateNamespaceUnderNamespaceOperationOrThrow(op, namespace); - - if (namespaceCatalog instanceof BasePolarisCatalog) { - // Note: The CatalogHandlers' default implementation will non-atomically create the - // namespace and then fetch its properties using loadNamespaceMetadata for the response. - // However, the latest namespace metadata technically isn't the same authorized instance, - // so we don't want all cals to loadNamespaceMetadata to automatically use the manifest - // in "passthrough" mode. - // - // For CreateNamespace, we consider this a special case in that the creator is able to - // retrieve the latest namespace metadata for the duration of the CreateNamespace - // operation, even if the entityVersion and/or grantsVersion update in the interim. - return doCatalogOperation( - () -> { - namespaceCatalog.createNamespace(namespace, request.properties()); - return CreateNamespaceResponse.builder() - .withNamespace(namespace) - .setProperties( - resolutionManifest - .getPassthroughResolvedPath(namespace) - .getRawLeafEntity() - .getPropertiesAsMap()) - .build(); - }); - } else { - return doCatalogOperation(() -> CatalogHandlers.createNamespace(namespaceCatalog, request)); - } - } - - private static boolean isExternal(CatalogEntity catalog) { - return org.apache.polaris.core.admin.model.Catalog.TypeEnum.EXTERNAL.equals( - catalog.getCatalogType()); - } - - private void doCatalogOperation(Runnable handler) { - doCatalogOperation( - () -> { - handler.run(); - return null; - }); - } - - /** - * Execute a catalog function and ensure we close the BaseCatalog afterward. This will typically - * ensure the underlying FileIO is closed - */ - private T doCatalogOperation(Supplier handler) { - try { - return handler.get(); - } finally { - if (baseCatalog instanceof Closeable closeable) { - try { - closeable.close(); - } catch (IOException e) { - LOGGER.error("Error while closing BaseCatalog", e); - } - } - } - } - - public GetNamespaceResponse loadNamespaceMetadata(Namespace namespace) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.LOAD_NAMESPACE_METADATA; - authorizeBasicNamespaceOperationOrThrow(op, namespace); - - return doCatalogOperation(() -> CatalogHandlers.loadNamespace(namespaceCatalog, namespace)); - } - - public void namespaceExists(Namespace namespace) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.NAMESPACE_EXISTS; - - // TODO: This authz check doesn't accomplish true authz in terms of blocking the ability - // for a caller to ascertain whether the namespace exists or not, but instead just behaves - // according to convention -- if existence is going to be privileged, we must instead - // add a base layer that throws NotFound exceptions instead of NotAuthorizedException - // for *all* operations in which we determine that the basic privilege for determining - // existence is also missing. - authorizeBasicNamespaceOperationOrThrow(op, namespace); - - // TODO: Just skip CatalogHandlers for this one maybe - doCatalogOperation(() -> CatalogHandlers.loadNamespace(namespaceCatalog, namespace)); - } - - public void dropNamespace(Namespace namespace) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.DROP_NAMESPACE; - authorizeBasicNamespaceOperationOrThrow(op, namespace); - - doCatalogOperation(() -> CatalogHandlers.dropNamespace(namespaceCatalog, namespace)); - } - - public UpdateNamespacePropertiesResponse updateNamespaceProperties( - Namespace namespace, UpdateNamespacePropertiesRequest request) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.UPDATE_NAMESPACE_PROPERTIES; - authorizeBasicNamespaceOperationOrThrow(op, namespace); - - return doCatalogOperation( - () -> CatalogHandlers.updateNamespaceProperties(namespaceCatalog, namespace, request)); - } - - public ListTablesResponse listTables(Namespace namespace) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.LIST_TABLES; - authorizeBasicNamespaceOperationOrThrow(op, namespace); - - return doCatalogOperation(() -> CatalogHandlers.listTables(baseCatalog, namespace)); - } - - public LoadTableResponse createTableDirect(Namespace namespace, CreateTableRequest request) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.CREATE_TABLE_DIRECT; - authorizeCreateTableLikeUnderNamespaceOperationOrThrow( - op, TableIdentifier.of(namespace, request.name())); - - CatalogEntity catalog = - CatalogEntity.of( - resolutionManifest - .getResolvedReferenceCatalogEntity() - .getResolvedLeafEntity() - .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot create table on external catalogs."); - } - return doCatalogOperation(() -> CatalogHandlers.createTable(baseCatalog, namespace, request)); - } - - public LoadTableResponse createTableDirectWithWriteDelegation( - Namespace namespace, CreateTableRequest request) { - PolarisAuthorizableOperation op = - PolarisAuthorizableOperation.CREATE_TABLE_DIRECT_WITH_WRITE_DELEGATION; - authorizeCreateTableLikeUnderNamespaceOperationOrThrow( - op, TableIdentifier.of(namespace, request.name())); - - CatalogEntity catalog = - CatalogEntity.of( - resolutionManifest - .getResolvedReferenceCatalogEntity() - .getResolvedLeafEntity() - .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot create table on external catalogs."); - } - return doCatalogOperation( - () -> { - request.validate(); - - TableIdentifier tableIdentifier = TableIdentifier.of(namespace, request.name()); - if (baseCatalog.tableExists(tableIdentifier)) { - throw new AlreadyExistsException("Table already exists: %s", tableIdentifier); - } - - Map properties = Maps.newHashMap(); - properties.put("created-at", OffsetDateTime.now(ZoneOffset.UTC).toString()); - properties.putAll(request.properties()); - - Table table = - baseCatalog - .buildTable(tableIdentifier, request.schema()) - .withLocation(request.location()) - .withPartitionSpec(request.spec()) - .withSortOrder(request.writeOrder()) - .withProperties(properties) - .create(); - - if (table instanceof BaseTable baseTable) { - TableMetadata tableMetadata = baseTable.operations().current(); - LoadTableResponse.Builder responseBuilder = - LoadTableResponse.builder().withTableMetadata(tableMetadata); - if (baseCatalog instanceof SupportsCredentialDelegation credentialDelegation) { - LOGGER - .atDebug() - .addKeyValue("tableIdentifier", tableIdentifier) - .addKeyValue("tableLocation", tableMetadata.location()) - .log("Fetching client credentials for table"); - responseBuilder.addAllConfig( - credentialDelegation.getCredentialConfig( - tableIdentifier, - tableMetadata, - Set.of( - PolarisStorageActions.READ, - PolarisStorageActions.WRITE, - PolarisStorageActions.LIST))); - } - return responseBuilder.build(); - } else if (table instanceof BaseMetadataTable) { - // metadata tables are loaded on the client side, return NoSuchTableException for now - throw new NoSuchTableException("Table does not exist: %s", tableIdentifier.toString()); - } - - throw new IllegalStateException("Cannot wrap catalog that does not produce BaseTable"); - }); - } - - private TableMetadata stageTableCreateHelper(Namespace namespace, CreateTableRequest request) { - request.validate(); - - TableIdentifier ident = TableIdentifier.of(namespace, request.name()); - if (baseCatalog.tableExists(ident)) { - throw new AlreadyExistsException("Table already exists: %s", ident); - } - - Map properties = Maps.newHashMap(); - properties.put("created-at", OffsetDateTime.now(ZoneOffset.UTC).toString()); - properties.putAll(request.properties()); - - String location; - if (request.location() != null) { - // Even if the request provides a location, run it through the catalog's TableBuilder - // to inherit any override behaviors if applicable. - if (baseCatalog instanceof BasePolarisCatalog) { - location = - ((BasePolarisCatalog) baseCatalog).transformTableLikeLocation(request.location()); - } else { - location = request.location(); - } - } else { - location = - baseCatalog - .buildTable(ident, request.schema()) - .withPartitionSpec(request.spec()) - .withSortOrder(request.writeOrder()) - .withProperties(properties) - .createTransaction() - .table() - .location(); - } - - TableMetadata metadata = - TableMetadata.newTableMetadata( - request.schema(), - request.spec() != null ? request.spec() : PartitionSpec.unpartitioned(), - request.writeOrder() != null ? request.writeOrder() : SortOrder.unsorted(), - location, - properties); - return metadata; - } - - public LoadTableResponse createTableStaged(Namespace namespace, CreateTableRequest request) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.CREATE_TABLE_STAGED; - authorizeCreateTableLikeUnderNamespaceOperationOrThrow( - op, TableIdentifier.of(namespace, request.name())); - - CatalogEntity catalog = - CatalogEntity.of( - resolutionManifest - .getResolvedReferenceCatalogEntity() - .getResolvedLeafEntity() - .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot create table on external catalogs."); - } - return doCatalogOperation( - () -> { - TableMetadata metadata = stageTableCreateHelper(namespace, request); - return LoadTableResponse.builder().withTableMetadata(metadata).build(); - }); - } - - public LoadTableResponse createTableStagedWithWriteDelegation( - Namespace namespace, CreateTableRequest request) { - PolarisAuthorizableOperation op = - PolarisAuthorizableOperation.CREATE_TABLE_STAGED_WITH_WRITE_DELEGATION; - authorizeCreateTableLikeUnderNamespaceOperationOrThrow( - op, TableIdentifier.of(namespace, request.name())); - - CatalogEntity catalog = - CatalogEntity.of( - resolutionManifest - .getResolvedReferenceCatalogEntity() - .getResolvedLeafEntity() - .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot create table on external catalogs."); - } - return doCatalogOperation( - () -> { - TableIdentifier ident = TableIdentifier.of(namespace, request.name()); - TableMetadata metadata = stageTableCreateHelper(namespace, request); - - LoadTableResponse.Builder responseBuilder = - LoadTableResponse.builder().withTableMetadata(metadata); - - if (baseCatalog instanceof SupportsCredentialDelegation credentialDelegation) { - LOGGER - .atDebug() - .addKeyValue("tableIdentifier", ident) - .addKeyValue("tableLocation", metadata.location()) - .log("Fetching client credentials for table"); - responseBuilder.addAllConfig( - credentialDelegation.getCredentialConfig( - ident, metadata, Set.of(PolarisStorageActions.ALL))); - } - return responseBuilder.build(); - }); - } - - public LoadTableResponse registerTable(Namespace namespace, RegisterTableRequest request) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.REGISTER_TABLE; - authorizeCreateTableLikeUnderNamespaceOperationOrThrow( - op, TableIdentifier.of(namespace, request.name())); - - return doCatalogOperation(() -> CatalogHandlers.registerTable(baseCatalog, namespace, request)); - } - - public boolean sendNotification(TableIdentifier identifier, NotificationRequest request) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.SEND_NOTIFICATIONS; - - // For now, just require the full set of privileges on the base Catalog entity, which we can - // also express just as the "root" Namespace for purposes of the BasePolarisCatalog being - // able to fetch Namespace.empty() as path key. - List extraPassthroughTableLikes = List.of(identifier); - List extraPassthroughNamespaces = new ArrayList<>(); - extraPassthroughNamespaces.add(Namespace.empty()); - for (int i = 1; i <= identifier.namespace().length(); i++) { - Namespace nsLevel = - Namespace.of( - Arrays.stream(identifier.namespace().levels()).limit(i).toArray(String[]::new)); - extraPassthroughNamespaces.add(nsLevel); - } - authorizeBasicNamespaceOperationOrThrow( - op, Namespace.empty(), extraPassthroughNamespaces, extraPassthroughTableLikes); - - CatalogEntity catalog = - CatalogEntity.of( - resolutionManifest - .getResolvedReferenceCatalogEntity() - .getResolvedLeafEntity() - .getEntity()); - if (catalog - .getCatalogType() - .equals(org.apache.polaris.core.admin.model.Catalog.TypeEnum.INTERNAL)) { - LOGGER - .atWarn() - .addKeyValue("catalog", catalog) - .addKeyValue("notification", request) - .log("Attempted notification on internal catalog"); - throw new BadRequestException("Cannot update internal catalog via notifications"); - } - return baseCatalog instanceof SupportsNotifications notificationCatalog - && notificationCatalog.sendNotification(identifier, request); - } - - public LoadTableResponse loadTable(TableIdentifier tableIdentifier, String snapshots) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.LOAD_TABLE; - authorizeBasicTableLikeOperationOrThrow(op, PolarisEntitySubType.TABLE, tableIdentifier); - - return doCatalogOperation(() -> CatalogHandlers.loadTable(baseCatalog, tableIdentifier)); - } - - public LoadTableResponse loadTableWithAccessDelegation( - TableIdentifier tableIdentifier, String snapshots) { - // Here we have a single method that falls through multiple candidate - // PolarisAuthorizableOperations because instead of identifying the desired operation up-front - // and - // failing the authz check if grants aren't found, we find the first most-privileged authz match - // and respond according to that. - PolarisAuthorizableOperation read = - PolarisAuthorizableOperation.LOAD_TABLE_WITH_READ_DELEGATION; - PolarisAuthorizableOperation write = - PolarisAuthorizableOperation.LOAD_TABLE_WITH_WRITE_DELEGATION; - - Set actionsRequested = - new HashSet<>(Set.of(PolarisStorageActions.READ, PolarisStorageActions.LIST)); - try { - // TODO: Refactor to have a boolean-return version of the helpers so we can fallthrough - // easily. - authorizeBasicTableLikeOperationOrThrow(write, PolarisEntitySubType.TABLE, tableIdentifier); - actionsRequested.add(PolarisStorageActions.WRITE); - } catch (ForbiddenException e) { - authorizeBasicTableLikeOperationOrThrow(read, PolarisEntitySubType.TABLE, tableIdentifier); - } - - // TODO: Find a way for the configuration or caller to better express whether to fail or omit - // when data-access is specified but access delegation grants are not found. - return doCatalogOperation( - () -> { - Table table = baseCatalog.loadTable(tableIdentifier); - - if (table instanceof BaseTable baseTable) { - TableMetadata tableMetadata = baseTable.operations().current(); - LoadTableResponse.Builder responseBuilder = - LoadTableResponse.builder().withTableMetadata(tableMetadata); - if (baseCatalog instanceof SupportsCredentialDelegation credentialDelegation) { - LOGGER - .atDebug() - .addKeyValue("tableIdentifier", tableIdentifier) - .addKeyValue("tableLocation", tableMetadata.location()) - .log("Fetching client credentials for table"); - responseBuilder.addAllConfig( - credentialDelegation.getCredentialConfig( - tableIdentifier, tableMetadata, actionsRequested)); - } - return responseBuilder.build(); - } else if (table instanceof BaseMetadataTable) { - // metadata tables are loaded on the client side, return NoSuchTableException for now - throw new NoSuchTableException("Table does not exist: %s", tableIdentifier.toString()); - } - - throw new IllegalStateException("Cannot wrap catalog that does not produce BaseTable"); - }); - } - - private UpdateTableRequest applyUpdateFilters(UpdateTableRequest request) { - // Certain MetadataUpdates need to be explicitly transformed to achieve the same behavior - // as using a local Catalog client via TableBuilder. - TableIdentifier identifier = request.identifier(); - List requirements = request.requirements(); - List updates = - request.updates().stream() - .map( - update -> { - if (baseCatalog instanceof BasePolarisCatalog - && update instanceof MetadataUpdate.SetLocation) { - String requestedLocation = ((MetadataUpdate.SetLocation) update).location(); - String filteredLocation = - ((BasePolarisCatalog) baseCatalog) - .transformTableLikeLocation(requestedLocation); - return new MetadataUpdate.SetLocation(filteredLocation); - } else { - return update; - } - }) - .toList(); - return UpdateTableRequest.create(identifier, requirements, updates); - } - - public LoadTableResponse updateTable( - TableIdentifier tableIdentifier, UpdateTableRequest request) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.UPDATE_TABLE; - authorizeBasicTableLikeOperationOrThrow(op, PolarisEntitySubType.TABLE, tableIdentifier); - - CatalogEntity catalog = - CatalogEntity.of( - resolutionManifest - .getResolvedReferenceCatalogEntity() - .getResolvedLeafEntity() - .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot update table on external catalogs."); - } - return doCatalogOperation( - () -> - CatalogHandlers.updateTable(baseCatalog, tableIdentifier, applyUpdateFilters(request))); - } - - public LoadTableResponse updateTableForStagedCreate( - TableIdentifier tableIdentifier, UpdateTableRequest request) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.UPDATE_TABLE_FOR_STAGED_CREATE; - authorizeCreateTableLikeUnderNamespaceOperationOrThrow(op, tableIdentifier); - - CatalogEntity catalog = - CatalogEntity.of( - resolutionManifest - .getResolvedReferenceCatalogEntity() - .getResolvedLeafEntity() - .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot update table on external catalogs."); - } - return doCatalogOperation( - () -> - CatalogHandlers.updateTable(baseCatalog, tableIdentifier, applyUpdateFilters(request))); - } - - public void dropTableWithoutPurge(TableIdentifier tableIdentifier) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.DROP_TABLE_WITHOUT_PURGE; - authorizeBasicTableLikeOperationOrThrow(op, PolarisEntitySubType.TABLE, tableIdentifier); - - doCatalogOperation(() -> CatalogHandlers.dropTable(baseCatalog, tableIdentifier)); - } - - public void dropTableWithPurge(TableIdentifier tableIdentifier) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.DROP_TABLE_WITH_PURGE; - authorizeBasicTableLikeOperationOrThrow(op, PolarisEntitySubType.TABLE, tableIdentifier); - - CatalogEntity catalog = - CatalogEntity.of( - resolutionManifest - .getResolvedReferenceCatalogEntity() - .getResolvedLeafEntity() - .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot drop table on external catalogs."); - } - doCatalogOperation(() -> CatalogHandlers.purgeTable(baseCatalog, tableIdentifier)); - } - - public void tableExists(TableIdentifier tableIdentifier) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.TABLE_EXISTS; - authorizeBasicTableLikeOperationOrThrow(op, PolarisEntitySubType.TABLE, tableIdentifier); - - // TODO: Just skip CatalogHandlers for this one maybe - doCatalogOperation(() -> CatalogHandlers.loadTable(baseCatalog, tableIdentifier)); - } - - public void renameTable(RenameTableRequest request) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.RENAME_TABLE; - authorizeRenameTableLikeOperationOrThrow( - op, PolarisEntitySubType.TABLE, request.source(), request.destination()); - - CatalogEntity catalog = - CatalogEntity.of( - resolutionManifest - .getResolvedReferenceCatalogEntity() - .getResolvedLeafEntity() - .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot rename table on external catalogs."); - } - doCatalogOperation(() -> CatalogHandlers.renameTable(baseCatalog, request)); - } - - public void commitTransaction(CommitTransactionRequest commitTransactionRequest) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.COMMIT_TRANSACTION; - // TODO: The authz actually needs to detect hidden updateForStagedCreate UpdateTableRequests - // and have some kind of per-item conditional privilege requirement if we want to make it - // so that only the stageCreate updates need TABLE_CREATE whereas everything else only - // needs TABLE_WRITE_PROPERTIES. - authorizeCollectionOfTableLikeOperationOrThrow( - op, - PolarisEntitySubType.TABLE, - commitTransactionRequest.tableChanges().stream() - .map(UpdateTableRequest::identifier) - .toList()); - CatalogEntity catalog = - CatalogEntity.of( - resolutionManifest - .getResolvedReferenceCatalogEntity() - .getResolvedLeafEntity() - .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot update table on external catalogs."); - } - - if (!(baseCatalog instanceof BasePolarisCatalog)) { - throw new BadRequestException( - "Unsupported operation: commitTransaction with baseCatalog type: %s", - baseCatalog.getClass().getName()); - } - - // Swap in TransactionWorkspaceMetaStoreManager for all mutations made by this baseCatalog to - // only go into an in-memory collection that we can commit as a single atomic unit after all - // validations. - TransactionWorkspaceMetaStoreManager transactionMetaStoreManager = - new TransactionWorkspaceMetaStoreManager(metaStoreManager); - ((BasePolarisCatalog) baseCatalog).setMetaStoreManager(transactionMetaStoreManager); - - commitTransactionRequest.tableChanges().stream() - .forEach( - change -> { - Table table = baseCatalog.loadTable(change.identifier()); - if (!(table instanceof BaseTable)) { - throw new IllegalStateException( - "Cannot wrap catalog that does not produce BaseTable"); - } - if (isCreate(change)) { - throw new BadRequestException( - "Unsupported operation: commitTranaction with updateForStagedCreate: %s", - change); - } - - TableOperations tableOps = ((BaseTable) table).operations(); - TableMetadata currentMetadata = tableOps.current(); - - // Validate requirements; any CommitFailedExceptions will fail the overall request - change.requirements().forEach(requirement -> requirement.validate(currentMetadata)); - - // Apply changes - TableMetadata.Builder metadataBuilder = TableMetadata.buildFrom(currentMetadata); - change.updates().stream() - .forEach( - singleUpdate -> { - // Note: If location-overlap checking is refactored to be atomic, we could - // support validation within a single multi-table transaction as well, but - // will need to update the TransactionWorkspaceMetaStoreManager to better - // expose the concept of being able to read uncommitted updates. - if (singleUpdate instanceof MetadataUpdate.SetLocation) { - if (!currentMetadata - .location() - .equals(((MetadataUpdate.SetLocation) singleUpdate).location()) - && !callContext - .getPolarisCallContext() - .getConfigurationStore() - .getConfiguration( - callContext.getPolarisCallContext(), - PolarisConfiguration.ALLOW_NAMESPACE_LOCATION_OVERLAP)) { - throw new BadRequestException( - "Unsupported operation: commitTransaction containing SetLocation" - + " for table '%s' and new location '%s'", - change.identifier(), - ((MetadataUpdate.SetLocation) singleUpdate).location()); - } - } - - // Apply updates to builder - singleUpdate.applyTo(metadataBuilder); - }); - - // Commit into transaction workspace we swapped the baseCatalog to use - TableMetadata updatedMetadata = metadataBuilder.build(); - if (!updatedMetadata.changes().isEmpty()) { - tableOps.commit(currentMetadata, updatedMetadata); - } - }); - - // Commit the collected updates in a single atomic operation - List pendingUpdates = - transactionMetaStoreManager.getPendingUpdates(); - PolarisMetaStoreManager.EntitiesResult result = - metaStoreManager.updateEntitiesPropertiesIfNotChanged( - callContext.getPolarisCallContext(), pendingUpdates); - if (!result.isSuccess()) { - // TODO: Retries and server-side cleanup on failure - throw new CommitFailedException( - "Transaction commit failed with status: %s, extraInfo: %s", - result.getReturnStatus(), result.getExtraInformation()); - } - } - - public ListTablesResponse listViews(Namespace namespace) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.LIST_VIEWS; - authorizeBasicNamespaceOperationOrThrow(op, namespace); - - return doCatalogOperation(() -> CatalogHandlers.listViews(viewCatalog, namespace)); - } - - public LoadViewResponse createView(Namespace namespace, CreateViewRequest request) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.CREATE_VIEW; - authorizeCreateTableLikeUnderNamespaceOperationOrThrow( - op, TableIdentifier.of(namespace, request.name())); - - CatalogEntity catalog = - CatalogEntity.of( - resolutionManifest - .getResolvedReferenceCatalogEntity() - .getResolvedLeafEntity() - .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot create view on external catalogs."); - } - return doCatalogOperation(() -> CatalogHandlers.createView(viewCatalog, namespace, request)); - } - - public LoadViewResponse loadView(TableIdentifier viewIdentifier) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.LOAD_VIEW; - authorizeBasicTableLikeOperationOrThrow(op, PolarisEntitySubType.VIEW, viewIdentifier); - - return doCatalogOperation(() -> CatalogHandlers.loadView(viewCatalog, viewIdentifier)); - } - - public LoadViewResponse replaceView(TableIdentifier viewIdentifier, UpdateTableRequest request) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.REPLACE_VIEW; - authorizeBasicTableLikeOperationOrThrow(op, PolarisEntitySubType.VIEW, viewIdentifier); - - CatalogEntity catalog = - CatalogEntity.of( - resolutionManifest - .getResolvedReferenceCatalogEntity() - .getResolvedLeafEntity() - .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot replace view on external catalogs."); - } - return doCatalogOperation( - () -> CatalogHandlers.updateView(viewCatalog, viewIdentifier, applyUpdateFilters(request))); - } - - public void dropView(TableIdentifier viewIdentifier) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.DROP_VIEW; - authorizeBasicTableLikeOperationOrThrow(op, PolarisEntitySubType.VIEW, viewIdentifier); - - doCatalogOperation(() -> CatalogHandlers.dropView(viewCatalog, viewIdentifier)); - } - - public void viewExists(TableIdentifier viewIdentifier) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.VIEW_EXISTS; - authorizeBasicTableLikeOperationOrThrow(op, PolarisEntitySubType.VIEW, viewIdentifier); - - // TODO: Just skip CatalogHandlers for this one maybe - doCatalogOperation(() -> CatalogHandlers.loadView(viewCatalog, viewIdentifier)); - } - - public void renameView(RenameTableRequest request) { - PolarisAuthorizableOperation op = PolarisAuthorizableOperation.RENAME_VIEW; - authorizeRenameTableLikeOperationOrThrow( - op, PolarisEntitySubType.VIEW, request.source(), request.destination()); - - CatalogEntity catalog = - CatalogEntity.of( - resolutionManifest - .getResolvedReferenceCatalogEntity() - .getResolvedLeafEntity() - .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot rename view on external catalogs."); - } - doCatalogOperation(() -> CatalogHandlers.renameView(viewCatalog, request)); - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/SupportsCredentialDelegation.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/SupportsCredentialDelegation.java deleted file mode 100644 index 554358765..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/SupportsCredentialDelegation.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.catalog; - -import java.util.Map; -import java.util.Set; -import org.apache.iceberg.TableMetadata; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.polaris.core.storage.PolarisStorageActions; - -/** - * Adds support for credential vending for (typically) {@link org.apache.iceberg.TableOperations} to - * fetch access credentials that are inserted into the {@link - * org.apache.iceberg.rest.responses.LoadTableResponse#config()} property. See the - * rest-catalog-open-api.yaml spec for details on the expected format of vended credential - * configuration. - */ -public interface SupportsCredentialDelegation { - Map getCredentialConfig( - TableIdentifier tableIdentifier, - TableMetadata tableMetadata, - Set storageActions); -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/SupportsNotifications.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/SupportsNotifications.java deleted file mode 100644 index 525dfaaae..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/SupportsNotifications.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.catalog; - -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.polaris.service.types.NotificationRequest; - -public interface SupportsNotifications { - - boolean sendNotification(TableIdentifier table, NotificationRequest notificationRequest); -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java deleted file mode 100644 index 2dbda360f..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.catalog.io; - -import io.quarkus.arc.lookup.LookupIfProperty; -import jakarta.enterprise.context.ApplicationScoped; -import java.util.Map; -import org.apache.hadoop.conf.Configuration; -import org.apache.iceberg.CatalogUtil; -import org.apache.iceberg.io.FileIO; -import org.apache.polaris.service.config.RuntimeCandidate; - -/** A simple FileIOFactory implementation that defers all the work to the Iceberg SDK */ -@ApplicationScoped -@RuntimeCandidate -@LookupIfProperty(name = "polaris.io.file-io-factory.type", stringValue = "default") -public class DefaultFileIOFactory implements FileIOFactory { - @Override - public FileIO loadFileIO(String impl, Map properties) { - return CatalogUtil.loadFileIO(impl, properties, new Configuration()); - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/io/FileIOFactory.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/io/FileIOFactory.java deleted file mode 100644 index ca3c08511..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/io/FileIOFactory.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.catalog.io; - -import java.util.Map; -import org.apache.iceberg.io.FileIO; - -/** Interface for providing a way to construct FileIO objects, such as for reading/writing S3. */ -public interface FileIOFactory { - FileIO loadFileIO(String impl, Map properties); -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/io/WasbTranslatingFileIO.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/io/WasbTranslatingFileIO.java deleted file mode 100644 index 48cf890a4..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/io/WasbTranslatingFileIO.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.catalog.io; - -import java.util.Map; -import org.apache.iceberg.io.FileIO; -import org.apache.iceberg.io.InputFile; -import org.apache.iceberg.io.OutputFile; -import org.apache.polaris.core.storage.StorageLocation; -import org.apache.polaris.core.storage.azure.AzureLocation; - -/** - * A {@link FileIO} implementation that translates WASB paths into ABFS paths and then delegates to - * another underlying FileIO implementation - */ -public class WasbTranslatingFileIO implements FileIO { - private final FileIO io; - - private static final String WASB_SCHEME = "wasb"; - private static final String ABFS_SCHEME = "abfs"; - - public WasbTranslatingFileIO(FileIO io) { - this.io = io; - } - - private static String translate(String path) { - if (path == null) { - return null; - } else { - StorageLocation storageLocation = StorageLocation.of(path); - if (storageLocation instanceof AzureLocation azureLocation) { - String scheme = azureLocation.getScheme(); - if (scheme.startsWith(WASB_SCHEME)) { - scheme = scheme.replaceFirst(WASB_SCHEME, ABFS_SCHEME); - } - return String.format( - "%s://%s@%s.%s/%s", - scheme, - azureLocation.getContainer(), - azureLocation.getStorageAccount(), - azureLocation.getEndpoint(), - azureLocation.getFilePath()); - } else { - return path; - } - } - } - - @Override - public InputFile newInputFile(String path) { - return io.newInputFile(translate(path)); - } - - @Override - public OutputFile newOutputFile(String path) { - return io.newOutputFile(translate(path)); - } - - @Override - public void deleteFile(String path) { - io.deleteFile(translate(path)); - } - - @Override - public Map properties() { - return io.properties(); - } - - @Override - public void initialize(Map properties) { - io.initialize(properties); - } - - @Override - public void close() { - io.close(); - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/io/WasbTranslatingFileIOFactory.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/io/WasbTranslatingFileIOFactory.java deleted file mode 100644 index e832544bf..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/catalog/io/WasbTranslatingFileIOFactory.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.catalog.io; - -import io.quarkus.arc.lookup.LookupIfProperty; -import jakarta.enterprise.context.ApplicationScoped; -import java.util.Map; -import org.apache.hadoop.conf.Configuration; -import org.apache.iceberg.CatalogUtil; -import org.apache.iceberg.io.FileIO; -import org.apache.polaris.service.config.RuntimeCandidate; - -/** A {@link FileIOFactory} that translates WASB paths to ABFS ones */ -@ApplicationScoped -@RuntimeCandidate -@LookupIfProperty(name = "polaris.io.file-io-factory.type", stringValue = "wasb") -public class WasbTranslatingFileIOFactory implements FileIOFactory { - @Override - public FileIO loadFileIO(String ioImpl, Map properties) { - return new WasbTranslatingFileIO( - CatalogUtil.loadFileIO(ioImpl, properties, new Configuration())); - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/config/DefaultConfigurationStore.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/config/DefaultConfigurationStore.java deleted file mode 100644 index 63496ef20..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/config/DefaultConfigurationStore.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.config; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import jakarta.annotation.Nullable; -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.inject.Inject; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.PolarisConfigurationStore; -import org.eclipse.microprofile.config.inject.ConfigProperty; - -@ApplicationScoped -public class DefaultConfigurationStore implements PolarisConfigurationStore { - - private final Map properties; - - // FIXME the whole PolarisConfigurationStore + PolarisConfiguration needs to be refactored - // to become a proper Quarkus configuration object - @Inject - public DefaultConfigurationStore( - ObjectMapper objectMapper, - @ConfigProperty(name = "polaris.config.feature-configurations") - Map properties) { - this(convertMap(objectMapper, properties)); - } - - public DefaultConfigurationStore(Map properties) { - this.properties = Map.copyOf(properties); - } - - private static Map convertMap( - ObjectMapper objectMapper, Map properties) { - Map m = new HashMap<>(); - for (String configName : properties.keySet()) { - String json = properties.get(configName); - try { - JsonNode node = objectMapper.readTree(json); - m.put(configName, configValue(node)); - } catch (JsonProcessingException e) { - throw new RuntimeException( - "Invalid JSON value for feature configuration: " + configName, e); - } - } - return m; - } - - private static Object configValue(JsonNode node) { - return switch (node.getNodeType()) { - case BOOLEAN -> node.asBoolean(); - case STRING -> node.asText(); - case NUMBER -> - switch (node.numberType()) { - case INT, LONG -> node.asLong(); - case FLOAT, DOUBLE -> node.asDouble(); - default -> - throw new IllegalArgumentException("Unsupported number type: " + node.numberType()); - }; - case ARRAY -> { - List list = new ArrayList<>(); - node.elements().forEachRemaining(n -> list.add(configValue(n))); - yield List.copyOf(list); - } - default -> - throw new IllegalArgumentException( - "Unsupported feature configuration JSON type: " + node.getNodeType()); - }; - } - - @Override - public @Nullable T getConfiguration(PolarisCallContext ctx, String configName) { - @SuppressWarnings("unchecked") - T o = (T) properties.get(configName); - return o; - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/config/RealmEntityManagerFactory.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/config/RealmEntityManagerFactory.java deleted file mode 100644 index afa1da612..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/config/RealmEntityManagerFactory.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.config; - -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.inject.Inject; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.core.persistence.PolarisEntityManager; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@ApplicationScoped -public class RealmEntityManagerFactory { - - private static final Logger LOGGER = - LoggerFactory.getLogger(RealmEntityManagerFactory.class.getName()); - - private final MetaStoreManagerFactory metaStoreManagerFactory; - - // Key: realmIdentifier - private final Map cachedEntityManagers = new ConcurrentHashMap<>(); - - @Inject - public RealmEntityManagerFactory(MetaStoreManagerFactory metaStoreManagerFactory) { - this.metaStoreManagerFactory = metaStoreManagerFactory; - } - - public PolarisEntityManager getOrCreateEntityManager(RealmContext context) { - String realm = context.getRealmIdentifier(); - - LOGGER.debug("Looking up PolarisEntityManager for realm {}", realm); - - return cachedEntityManagers.computeIfAbsent( - realm, - r -> { - LOGGER.info("Initializing new PolarisEntityManager for realm {}", r); - return new PolarisEntityManager( - metaStoreManagerFactory.getOrCreateMetaStoreManager(context), - metaStoreManagerFactory.getOrCreateStorageCredentialCache(context)); - }); - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/config/Serializers.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/config/Serializers.java deleted file mode 100644 index a2a84eef6..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/config/Serializers.java +++ /dev/null @@ -1,246 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.config; - -import com.fasterxml.jackson.core.JacksonException; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.TreeNode; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.module.SimpleModule; -import com.fasterxml.jackson.databind.node.ObjectNode; -import java.io.IOException; -import org.apache.polaris.core.admin.model.AddGrantRequest; -import org.apache.polaris.core.admin.model.Catalog; -import org.apache.polaris.core.admin.model.CatalogRole; -import org.apache.polaris.core.admin.model.CreateCatalogRequest; -import org.apache.polaris.core.admin.model.CreateCatalogRoleRequest; -import org.apache.polaris.core.admin.model.CreatePrincipalRequest; -import org.apache.polaris.core.admin.model.CreatePrincipalRoleRequest; -import org.apache.polaris.core.admin.model.GrantCatalogRoleRequest; -import org.apache.polaris.core.admin.model.GrantPrincipalRoleRequest; -import org.apache.polaris.core.admin.model.GrantResource; -import org.apache.polaris.core.admin.model.Principal; -import org.apache.polaris.core.admin.model.PrincipalRole; -import org.apache.polaris.core.admin.model.RevokeGrantRequest; - -public final class Serializers { - private Serializers() {} - - public static void registerSerializers(ObjectMapper mapper) { - SimpleModule module = new SimpleModule(); - module.addDeserializer(CreateCatalogRequest.class, new CreateCatalogRequestDeserializer()); - module.addDeserializer(CreatePrincipalRequest.class, new CreatePrincipalRequestDeserializer()); - module.addDeserializer( - CreatePrincipalRoleRequest.class, new CreatePrincipalRoleRequestDeserializer()); - module.addDeserializer( - GrantPrincipalRoleRequest.class, new GrantPrincipalRoleRequestDeserializer()); - module.addDeserializer( - CreateCatalogRoleRequest.class, new CreateCatalogRoleRequestDeserializer()); - module.addDeserializer( - GrantCatalogRoleRequest.class, new GrantCatalogRoleRequestDeserializer()); - module.addDeserializer(AddGrantRequest.class, new AddGrantRequestDeserializer()); - module.addDeserializer(RevokeGrantRequest.class, new RevokeGrantRequestDeserializer()); - mapper.registerModule(module); - } - - /** - * Deserializer for {@link CreateCatalogRequest}. Backward compatible with the previous version of - * the api - */ - public static final class CreateCatalogRequestDeserializer - extends JsonDeserializer { - @Override - public CreateCatalogRequest deserialize(JsonParser p, DeserializationContext ctxt) - throws IOException, JacksonException { - TreeNode treeNode = p.readValueAsTree(); - if (treeNode.isObject() && ((ObjectNode) treeNode).has("catalog")) { - return CreateCatalogRequest.builder() - .setCatalog(ctxt.readTreeAsValue((JsonNode) treeNode.get("catalog"), Catalog.class)) - .build(); - } else { - return CreateCatalogRequest.builder() - .setCatalog(ctxt.readTreeAsValue((JsonNode) treeNode, Catalog.class)) - .build(); - } - } - } - - /** - * Deserializer for {@link CreatePrincipalRequest}. Backward compatible with the previous version - * of the api - */ - public static final class CreatePrincipalRequestDeserializer - extends JsonDeserializer { - @Override - public CreatePrincipalRequest deserialize(JsonParser p, DeserializationContext ctxt) - throws IOException, JacksonException { - TreeNode treeNode = p.readValueAsTree(); - if (treeNode.isObject() && ((ObjectNode) treeNode).has("principal")) { - return CreatePrincipalRequest.builder() - .setPrincipal( - ctxt.readTreeAsValue((JsonNode) treeNode.get("principal"), Principal.class)) - .setCredentialRotationRequired( - ctxt.readTreeAsValue( - (JsonNode) treeNode.get("credentialRotationRequired"), Boolean.class)) - .build(); - } else { - return CreatePrincipalRequest.builder() - .setPrincipal(ctxt.readTreeAsValue((JsonNode) treeNode, Principal.class)) - .build(); - } - } - } - - /** - * Deserializer for {@link CreatePrincipalRoleRequest}. Backward compatible with the previous - * version of the api - */ - public static final class CreatePrincipalRoleRequestDeserializer - extends JsonDeserializer { - @Override - public CreatePrincipalRoleRequest deserialize(JsonParser p, DeserializationContext ctxt) - throws IOException, JacksonException { - TreeNode treeNode = p.readValueAsTree(); - if (treeNode.isObject() && ((ObjectNode) treeNode).has("principalRole")) { - return CreatePrincipalRoleRequest.builder() - .setPrincipalRole( - ctxt.readTreeAsValue((JsonNode) treeNode.get("principalRole"), PrincipalRole.class)) - .build(); - } else { - return CreatePrincipalRoleRequest.builder() - .setPrincipalRole(ctxt.readTreeAsValue((JsonNode) treeNode, PrincipalRole.class)) - .build(); - } - } - } - - /** - * Deserializer for {@link GrantPrincipalRoleRequest}. Backward compatible with the previous - * version of the api - */ - public static final class GrantPrincipalRoleRequestDeserializer - extends JsonDeserializer { - @Override - public GrantPrincipalRoleRequest deserialize(JsonParser p, DeserializationContext ctxt) - throws IOException, JacksonException { - TreeNode treeNode = p.readValueAsTree(); - if (treeNode.isObject() && ((ObjectNode) treeNode).has("principalRole")) { - return GrantPrincipalRoleRequest.builder() - .setPrincipalRole( - ctxt.readTreeAsValue((JsonNode) treeNode.get("principalRole"), PrincipalRole.class)) - .build(); - } else { - return GrantPrincipalRoleRequest.builder() - .setPrincipalRole(ctxt.readTreeAsValue((JsonNode) treeNode, PrincipalRole.class)) - .build(); - } - } - } - - /** - * Deserializer for {@link CreateCatalogRoleRequest} Backward compatible with the previous version - * of the api - */ - public static final class CreateCatalogRoleRequestDeserializer - extends JsonDeserializer { - @Override - public CreateCatalogRoleRequest deserialize(JsonParser p, DeserializationContext ctxt) - throws IOException, JacksonException { - TreeNode treeNode = p.readValueAsTree(); - if (treeNode.isObject() && ((ObjectNode) treeNode).has("catalogRole")) { - return CreateCatalogRoleRequest.builder() - .setCatalogRole( - ctxt.readTreeAsValue((JsonNode) treeNode.get("catalogRole"), CatalogRole.class)) - .build(); - } else { - return CreateCatalogRoleRequest.builder() - .setCatalogRole(ctxt.readTreeAsValue((JsonNode) treeNode, CatalogRole.class)) - .build(); - } - } - } - - /** - * Deserializer for {@link GrantCatalogRoleRequest} Backward compatible with the previous version - * of the api - */ - public static final class GrantCatalogRoleRequestDeserializer - extends JsonDeserializer { - @Override - public GrantCatalogRoleRequest deserialize(JsonParser p, DeserializationContext ctxt) - throws IOException, JacksonException { - TreeNode treeNode = p.readValueAsTree(); - if (treeNode.isObject() && ((ObjectNode) treeNode).has("catalogRole")) { - return GrantCatalogRoleRequest.builder() - .setCatalogRole( - ctxt.readTreeAsValue((JsonNode) treeNode.get("catalogRole"), CatalogRole.class)) - .build(); - } else { - return GrantCatalogRoleRequest.builder() - .setCatalogRole(ctxt.readTreeAsValue((JsonNode) treeNode, CatalogRole.class)) - .build(); - } - } - } - - /** - * Deserializer for {@link AddGrantRequest} Backward compatible with previous version of the api - */ - public static final class AddGrantRequestDeserializer extends JsonDeserializer { - @Override - public AddGrantRequest deserialize(JsonParser p, DeserializationContext ctxt) - throws IOException, JacksonException { - TreeNode treeNode = p.readValueAsTree(); - if (treeNode.isObject() && ((ObjectNode) treeNode).has("grant")) { - return AddGrantRequest.builder() - .setGrant(ctxt.readTreeAsValue((JsonNode) treeNode.get("grant"), GrantResource.class)) - .build(); - } else { - return AddGrantRequest.builder() - .setGrant(ctxt.readTreeAsValue((JsonNode) treeNode, GrantResource.class)) - .build(); - } - } - } - - /** - * Deserializer for {@link RevokeGrantRequest} Backward compatible with previous version of the - * api - */ - public static final class RevokeGrantRequestDeserializer - extends JsonDeserializer { - @Override - public RevokeGrantRequest deserialize(JsonParser p, DeserializationContext ctxt) - throws IOException, JacksonException { - TreeNode treeNode = p.readValueAsTree(); - if (treeNode.isObject() && ((ObjectNode) treeNode).has("grant")) { - return RevokeGrantRequest.builder() - .setGrant(ctxt.readTreeAsValue((JsonNode) treeNode.get("grant"), GrantResource.class)) - .build(); - } else { - return RevokeGrantRequest.builder() - .setGrant(ctxt.readTreeAsValue((JsonNode) treeNode, GrantResource.class)) - .build(); - } - } - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/config/TaskHandlerConfiguration.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/config/TaskHandlerConfiguration.java deleted file mode 100644 index bc8cab508..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/config/TaskHandlerConfiguration.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.config; - -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import jakarta.enterprise.context.ApplicationScoped; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ThreadFactory; -import org.eclipse.microprofile.config.inject.ConfigProperty; - -@ApplicationScoped -public class TaskHandlerConfiguration { - - private final int poolSize; - private final boolean fixedSize; - private final String threadNamePattern; - - public TaskHandlerConfiguration( - @ConfigProperty(name = "polaris.tasks.pool-size") int poolSize, - @ConfigProperty(name = "polaris.tasks.fixed-size") boolean fixedSize, - @ConfigProperty(name = "polaris.tasks.thread-name-pattern") String threadNamePattern) { - this.poolSize = poolSize; - this.fixedSize = fixedSize; - this.threadNamePattern = threadNamePattern; - } - - public ExecutorService executorService() { - return fixedSize - ? Executors.newFixedThreadPool(poolSize, threadFactory()) - : Executors.newCachedThreadPool(threadFactory()); - } - - private ThreadFactory threadFactory() { - return new ThreadFactoryBuilder().setNameFormat(threadNamePattern).setDaemon(true).build(); - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/context/CallContextCatalogFactory.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/context/CallContextCatalogFactory.java deleted file mode 100644 index 24551a2d0..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/context/CallContextCatalogFactory.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.context; - -import org.apache.iceberg.catalog.Catalog; -import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest; - -public interface CallContextCatalogFactory { - Catalog createCallContextCatalog( - CallContext context, - AuthenticatedPolarisPrincipal authenticatedPrincipal, - PolarisResolutionManifest resolvedManifest); -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/context/CallContextResolver.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/context/CallContextResolver.java deleted file mode 100644 index 71b21d0f3..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/context/CallContextResolver.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.context; - -import java.util.Map; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.context.RealmContext; - -/** Uses the resolved RealmContext to further resolve elements of the CallContext. */ -public interface CallContextResolver { - CallContext resolveCallContext( - RealmContext realmContext, - String method, - String path, - Map queryParams, - Map headers); -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java deleted file mode 100644 index f5ea10fb2..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.context; - -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.inject.Inject; -import java.nio.file.Paths; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; -import org.apache.iceberg.CatalogProperties; -import org.apache.iceberg.catalog.Catalog; -import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.entity.CatalogEntity; -import org.apache.polaris.core.entity.PolarisBaseEntity; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.core.persistence.PolarisEntityManager; -import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest; -import org.apache.polaris.service.catalog.BasePolarisCatalog; -import org.apache.polaris.service.catalog.io.FileIOFactory; -import org.apache.polaris.service.config.RealmEntityManagerFactory; -import org.apache.polaris.service.task.TaskExecutor; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@ApplicationScoped -public class PolarisCallContextCatalogFactory implements CallContextCatalogFactory { - - private static final Logger LOGGER = - LoggerFactory.getLogger(PolarisCallContextCatalogFactory.class); - - private static final String WAREHOUSE_LOCATION_BASEDIR = - "/tmp/iceberg_rest_server_warehouse_data/"; - - private final RealmEntityManagerFactory entityManagerFactory; - private final TaskExecutor taskExecutor; - private final FileIOFactory fileIOFactory; - private final MetaStoreManagerFactory metaStoreManagerFactory; - - @Inject - public PolarisCallContextCatalogFactory( - RealmEntityManagerFactory entityManagerFactory, - MetaStoreManagerFactory metaStoreManagerFactory, - TaskExecutor taskExecutor, - FileIOFactory fileIOFactory) { - this.entityManagerFactory = entityManagerFactory; - this.taskExecutor = taskExecutor; - this.fileIOFactory = fileIOFactory; - this.metaStoreManagerFactory = metaStoreManagerFactory; - } - - @Override - public Catalog createCallContextCatalog( - CallContext context, - AuthenticatedPolarisPrincipal authenticatedPrincipal, - final PolarisResolutionManifest resolvedManifest) { - PolarisBaseEntity baseCatalogEntity = - resolvedManifest.getResolvedReferenceCatalogEntity().getRawLeafEntity(); - String catalogName = baseCatalogEntity.getName(); - - String realm = context.getRealmContext().getRealmIdentifier(); - String catalogKey = realm + "/" + catalogName; - LOGGER.info("Initializing new BasePolarisCatalog for key: {}", catalogKey); - - PolarisEntityManager entityManager = - entityManagerFactory.getOrCreateEntityManager(context.getRealmContext()); - - BasePolarisCatalog catalogInstance = - new BasePolarisCatalog( - entityManager, - metaStoreManagerFactory.getOrCreateMetaStoreManager(context.getRealmContext()), - context, - resolvedManifest, - authenticatedPrincipal, - taskExecutor, - fileIOFactory); - - context.contextVariables().put(CallContext.REQUEST_PATH_CATALOG_INSTANCE_KEY, catalogInstance); - - CatalogEntity catalog = CatalogEntity.of(baseCatalogEntity); - Map catalogProperties = new HashMap<>(catalog.getPropertiesAsMap()); - String defaultBaseLocation = catalog.getDefaultBaseLocation(); - LOGGER.info("Looked up defaultBaseLocation {} for catalog {}", defaultBaseLocation, catalogKey); - catalogProperties.put( - CatalogProperties.WAREHOUSE_LOCATION, - Objects.requireNonNullElseGet( - defaultBaseLocation, - () -> Paths.get(WAREHOUSE_LOCATION_BASEDIR, catalogKey).toString())); - - // TODO: The initialize properties might need to take more from CallContext and the - // CatalogEntity. - catalogInstance.initialize(catalogName, catalogProperties); - - return catalogInstance; - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/context/RealmContextResolver.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/context/RealmContextResolver.java deleted file mode 100644 index fedc1fde2..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/context/RealmContextResolver.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.context; - -import java.util.Map; -import org.apache.polaris.core.context.RealmContext; - -public interface RealmContextResolver { - - RealmContext resolveRealmContext( - String requestURL, - String method, - String path, - Map queryParams, - Map headers); - - String getDefaultRealm(); -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/exception/IcebergExceptionMapper.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/exception/IcebergExceptionMapper.java deleted file mode 100644 index 22032c30d..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/exception/IcebergExceptionMapper.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.exception; - -import com.azure.core.exception.AzureException; -import com.google.cloud.storage.StorageException; -import com.google.common.collect.ImmutableSet; -import jakarta.ws.rs.WebApplicationException; -import jakarta.ws.rs.core.MediaType; -import jakarta.ws.rs.core.Response; -import jakarta.ws.rs.ext.ExceptionMapper; -import jakarta.ws.rs.ext.Provider; -import java.util.Arrays; -import java.util.Collection; -import java.util.Locale; -import java.util.Set; -import org.apache.commons.lang3.exception.ExceptionUtils; -import org.apache.iceberg.exceptions.AlreadyExistsException; -import org.apache.iceberg.exceptions.CherrypickAncestorCommitException; -import org.apache.iceberg.exceptions.CleanableFailure; -import org.apache.iceberg.exceptions.CommitFailedException; -import org.apache.iceberg.exceptions.CommitStateUnknownException; -import org.apache.iceberg.exceptions.DuplicateWAPCommitException; -import org.apache.iceberg.exceptions.ForbiddenException; -import org.apache.iceberg.exceptions.NamespaceNotEmptyException; -import org.apache.iceberg.exceptions.NoSuchIcebergTableException; -import org.apache.iceberg.exceptions.NoSuchNamespaceException; -import org.apache.iceberg.exceptions.NoSuchTableException; -import org.apache.iceberg.exceptions.NoSuchViewException; -import org.apache.iceberg.exceptions.NotAuthorizedException; -import org.apache.iceberg.exceptions.NotFoundException; -import org.apache.iceberg.exceptions.RESTException; -import org.apache.iceberg.exceptions.RuntimeIOException; -import org.apache.iceberg.exceptions.ServiceFailureException; -import org.apache.iceberg.exceptions.ServiceUnavailableException; -import org.apache.iceberg.exceptions.UnprocessableEntityException; -import org.apache.iceberg.exceptions.ValidationException; -import org.apache.iceberg.rest.responses.ErrorResponse; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import software.amazon.awssdk.services.s3.model.S3Exception; - -@Provider -public class IcebergExceptionMapper implements ExceptionMapper { - private static final Logger LOGGER = LoggerFactory.getLogger(IcebergExceptionMapper.class); - - // Case-insensitive parts of exception messages that a request to a cloud provider was denied due - // to lack of permissions - // We may want to consider a change to Iceberg Core to wrap cloud provider IO exceptions to - // Iceberg ForbiddenException - private static final Set ACCESS_DENIED_HINTS = - Set.of("access denied", "not authorized", "forbidden"); - - public IcebergExceptionMapper() {} - - @Override - public Response toResponse(RuntimeException runtimeException) { - LOGGER.info("Handling runtimeException {}", runtimeException.getMessage()); - int responseCode = - switch (runtimeException) { - case NoSuchNamespaceException e -> Response.Status.NOT_FOUND.getStatusCode(); - case NoSuchIcebergTableException e -> Response.Status.NOT_FOUND.getStatusCode(); - case NoSuchTableException e -> Response.Status.NOT_FOUND.getStatusCode(); - case NoSuchViewException e -> Response.Status.NOT_FOUND.getStatusCode(); - case NotFoundException e -> Response.Status.NOT_FOUND.getStatusCode(); - case AlreadyExistsException e -> Response.Status.CONFLICT.getStatusCode(); - case CommitFailedException e -> Response.Status.CONFLICT.getStatusCode(); - case UnprocessableEntityException e -> 422; - case CherrypickAncestorCommitException e -> Response.Status.BAD_REQUEST.getStatusCode(); - case CommitStateUnknownException e -> Response.Status.BAD_REQUEST.getStatusCode(); - case DuplicateWAPCommitException e -> Response.Status.BAD_REQUEST.getStatusCode(); - case ForbiddenException e -> Response.Status.FORBIDDEN.getStatusCode(); - case jakarta.ws.rs.ForbiddenException e -> Response.Status.FORBIDDEN.getStatusCode(); - case NotAuthorizedException e -> Response.Status.UNAUTHORIZED.getStatusCode(); - case NamespaceNotEmptyException e -> Response.Status.BAD_REQUEST.getStatusCode(); - case ValidationException e -> Response.Status.BAD_REQUEST.getStatusCode(); - case ServiceUnavailableException e -> Response.Status.SERVICE_UNAVAILABLE.getStatusCode(); - case RuntimeIOException e -> Response.Status.SERVICE_UNAVAILABLE.getStatusCode(); - case ServiceFailureException e -> Response.Status.SERVICE_UNAVAILABLE.getStatusCode(); - case CleanableFailure e -> Response.Status.BAD_REQUEST.getStatusCode(); - case RESTException e -> Response.Status.SERVICE_UNAVAILABLE.getStatusCode(); - case IllegalArgumentException e -> Response.Status.BAD_REQUEST.getStatusCode(); - case UnsupportedOperationException e -> Response.Status.NOT_ACCEPTABLE.getStatusCode(); - case S3Exception e when doesAnyThrowableContainAccessDeniedHint(e) -> - Response.Status.FORBIDDEN.getStatusCode(); - case AzureException e when doesAnyThrowableContainAccessDeniedHint(e) -> - Response.Status.FORBIDDEN.getStatusCode(); - case StorageException e when doesAnyThrowableContainAccessDeniedHint(e) -> - Response.Status.FORBIDDEN.getStatusCode(); - case WebApplicationException e -> e.getResponse().getStatus(); - default -> Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(); - }; - if (responseCode == Response.Status.INTERNAL_SERVER_ERROR.getStatusCode()) { - LOGGER.error("Unhandled exception returning INTERNAL_SERVER_ERROR", runtimeException); - } - - ErrorResponse icebergErrorResponse = - ErrorResponse.builder() - .responseCode(responseCode) - .withType(runtimeException.getClass().getSimpleName()) - .withMessage(runtimeException.getMessage()) - .build(); - Response errorResp = - Response.status(responseCode) - .entity(icebergErrorResponse) - .type(MediaType.APPLICATION_JSON_TYPE) - .build(); - LOGGER.debug("Mapped exception to errorResp: {}", errorResp); - return errorResp; - } - - /** - * @return whether any throwable in the exception chain case-insensitive-contains the given - * message - */ - static boolean doesAnyThrowableContainAccessDeniedHint(Exception e) { - return Arrays.stream(ExceptionUtils.getThrowables(e)) - .anyMatch(t -> containsAnyAccessDeniedHint(t.getMessage())); - } - - public static boolean containsAnyAccessDeniedHint(String message) { - String messageLower = message.toLowerCase(Locale.ENGLISH); - return ACCESS_DENIED_HINTS.stream().anyMatch(messageLower::contains); - } - - public static Collection getAccessDeniedHints() { - return ImmutableSet.copyOf(ACCESS_DENIED_HINTS); - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/exception/IcebergJerseyViolationExceptionMapper.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/exception/IcebergJerseyViolationExceptionMapper.java deleted file mode 100644 index d5ec76841..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/exception/IcebergJerseyViolationExceptionMapper.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.exception; - -import jakarta.validation.ConstraintViolationException; -import jakarta.ws.rs.core.MediaType; -import jakarta.ws.rs.core.Response; -import jakarta.ws.rs.ext.ExceptionMapper; -import jakarta.ws.rs.ext.Provider; -import org.apache.iceberg.rest.responses.ErrorResponse; - -/** See {@code io.dropwizard.jersey.validation.JerseyViolationException} */ -@Provider -public class IcebergJerseyViolationExceptionMapper - implements ExceptionMapper { - @Override - public Response toResponse(ConstraintViolationException exception) { - final String message = "Invalid value: " + exception.getMessage(); - ErrorResponse icebergErrorResponse = - ErrorResponse.builder() - .responseCode(Response.Status.BAD_REQUEST.getStatusCode()) - .withType(exception.getClass().getSimpleName()) - .withMessage(message) - .build(); - return Response.status(Response.Status.BAD_REQUEST) - .type(MediaType.APPLICATION_JSON_TYPE) - .entity(icebergErrorResponse) - .build(); - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/exception/IcebergJsonProcessingExceptionMapper.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/exception/IcebergJsonProcessingExceptionMapper.java deleted file mode 100644 index b77e9138e..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/exception/IcebergJsonProcessingExceptionMapper.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.exception; - -import com.fasterxml.jackson.core.JsonGenerationException; -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.exc.InvalidDefinitionException; -import com.fasterxml.jackson.databind.exc.ValueInstantiationException; -import io.dropwizard.jersey.errors.ErrorMessage; -import jakarta.ws.rs.core.MediaType; -import jakarta.ws.rs.core.Response; -import jakarta.ws.rs.core.Response.Status; -import jakarta.ws.rs.ext.ExceptionMapper; -import jakarta.ws.rs.ext.Provider; -import java.util.Locale; -import java.util.concurrent.ThreadLocalRandom; -import org.apache.iceberg.rest.responses.ErrorResponse; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** See Dropwizard's {@code io.dropwizard.jersey.jackson.JsonProcessingExceptionMapper} */ -@Provider -public final class IcebergJsonProcessingExceptionMapper - implements ExceptionMapper { - - private static final Logger LOGGER = - LoggerFactory.getLogger(IcebergJsonProcessingExceptionMapper.class); - - @Override - public Response toResponse(JsonProcessingException exception) { - /* - * If the error is in the JSON generation or an invalid definition, it's a server error. - */ - if (exception instanceof JsonGenerationException - || exception instanceof InvalidDefinitionException) { - long id = ThreadLocalRandom.current().nextLong(); - LOGGER.error(String.format(Locale.ROOT, "Error handling a request: %016x", id), exception); - String message = - String.format( - Locale.ROOT, - "There was an error processing your request. It has been logged (ID %016x).", - id); - return Response.status(Status.INTERNAL_SERVER_ERROR.getStatusCode()) - .type(MediaType.APPLICATION_JSON_TYPE) - .entity(new ErrorMessage(message)) - .build(); - } - - /* - * Otherwise, it's those pesky users. - */ - LOGGER.info("Unable to process JSON: {}", exception.getMessage()); - - String messagePrefix = - switch (exception) { - case JsonParseException e -> "Invalid JSON: "; - case ValueInstantiationException ve -> "Invalid value: "; - default -> ""; - }; - final String message = messagePrefix + exception.getOriginalMessage(); - ErrorResponse icebergErrorResponse = - ErrorResponse.builder() - .responseCode(Response.Status.BAD_REQUEST.getStatusCode()) - .withType(exception.getClass().getSimpleName()) - .withMessage(message) - .build(); - return Response.status(Response.Status.BAD_REQUEST) - .type(MediaType.APPLICATION_JSON_TYPE) - .entity(icebergErrorResponse) - .build(); - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/exception/PolarisExceptionMapper.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/exception/PolarisExceptionMapper.java deleted file mode 100644 index 489b37351..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/exception/PolarisExceptionMapper.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.exception; - -import jakarta.ws.rs.core.MediaType; -import jakarta.ws.rs.core.Response; -import jakarta.ws.rs.ext.ExceptionMapper; -import jakarta.ws.rs.ext.Provider; -import org.apache.iceberg.rest.responses.ErrorResponse; -import org.apache.polaris.core.exceptions.AlreadyExistsException; -import org.apache.polaris.core.exceptions.PolarisException; - -/** - * An {@link ExceptionMapper} implementation for {@link PolarisException}s modeled after {@link - * IcebergExceptionMapper} - */ -@Provider -public class PolarisExceptionMapper implements ExceptionMapper { - - private Response.Status getStatus(PolarisException exception) { - if (exception instanceof AlreadyExistsException) { - return Response.Status.CONFLICT; - } else { - return Response.Status.INTERNAL_SERVER_ERROR; - } - } - - @Override - public Response toResponse(PolarisException exception) { - Response.Status status = getStatus(exception); - ErrorResponse errorResponse = - ErrorResponse.builder() - .responseCode(status.getStatusCode()) - .withType(exception.getClass().getSimpleName()) - .withMessage(exception.getMessage()) - .build(); - return Response.status(status) - .entity(errorResponse) - .type(MediaType.APPLICATION_JSON_TYPE) - .build(); - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java deleted file mode 100644 index b16793c49..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.persistence; - -import io.quarkus.arc.lookup.LookupIfProperty; -import io.quarkus.runtime.Startup; -import jakarta.annotation.Nonnull; -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.inject.Inject; -import java.util.Collections; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.function.Supplier; -import org.apache.polaris.core.PolarisDiagnostics; -import org.apache.polaris.core.auth.PolarisSecretsManager.PrincipalSecretsResult; -import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.core.persistence.*; -import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; -import org.apache.polaris.service.config.RuntimeCandidate; -import org.apache.polaris.service.context.RealmContextResolver; - -@ApplicationScoped -@RuntimeCandidate -@LookupIfProperty(name = "polaris.persistence.metastore-manager.type", stringValue = "in-memory") -public class InMemoryPolarisMetaStoreManagerFactory - extends LocalPolarisMetaStoreManagerFactory { - - private final Set bootstrappedRealms = new HashSet<>(); - private final RealmContextResolver realmContextResolver; - - @Inject - public InMemoryPolarisMetaStoreManagerFactory( - PolarisStorageIntegrationProvider storageIntegration, - RealmContextResolver realmContextResolver) { - this.storageIntegration = storageIntegration; - this.realmContextResolver = realmContextResolver; - } - - @Startup - public void init() { - // For in-memory metastore we need to bootstrap Service and Service principal at startup - // (for default realm) - getOrCreateMetaStoreManager(realmContextResolver::getDefaultRealm); - } - - @Override - protected PolarisTreeMapStore createBackingStore(@Nonnull PolarisDiagnostics diagnostics) { - return new PolarisTreeMapStore(diagnostics); - } - - @Override - protected PolarisMetaStoreSession createMetaStoreSession( - @Nonnull PolarisTreeMapStore store, @Nonnull RealmContext realmContext) { - return new PolarisTreeMapMetaStoreSessionImpl(store, storageIntegration); - } - - @Override - public synchronized PolarisMetaStoreManager getOrCreateMetaStoreManager( - RealmContext realmContext) { - String realmId = realmContext.getRealmIdentifier(); - if (!bootstrappedRealms.contains(realmId)) { - bootstrapRealmAndPrintCredentials(realmId); - } - return super.getOrCreateMetaStoreManager(realmContext); - } - - @Override - public synchronized Supplier getOrCreateSessionSupplier( - RealmContext realmContext) { - String realmId = realmContext.getRealmIdentifier(); - if (!bootstrappedRealms.contains(realmId)) { - bootstrapRealmAndPrintCredentials(realmId); - } - return super.getOrCreateSessionSupplier(realmContext); - } - - private void bootstrapRealmAndPrintCredentials(String realmId) { - Map results = - this.bootstrapRealms(Collections.singletonList(realmId)); - bootstrappedRealms.add(realmId); - - PrincipalSecretsResult principalSecrets = results.get(realmId); - - String msg = - String.format( - "realm: %1s root principal credentials: %2s:%3s", - realmId, - principalSecrets.getPrincipalSecrets().getPrincipalClientId(), - principalSecrets.getPrincipalSecrets().getMainSecret()); - System.out.println(msg); - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/ratelimiter/NoOpRateLimiter.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/ratelimiter/NoOpRateLimiter.java deleted file mode 100644 index 6ac76638a..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/ratelimiter/NoOpRateLimiter.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.ratelimiter; - -import io.quarkus.arc.lookup.LookupIfProperty; -import jakarta.enterprise.context.ApplicationScoped; -import org.apache.polaris.service.config.RuntimeCandidate; - -/** Rate limiter that always allows the request */ -@ApplicationScoped -@RuntimeCandidate -@LookupIfProperty(name = "polaris.rate-limiter.type", stringValue = "no-op") -public class NoOpRateLimiter implements RateLimiter { - @Override - public boolean tryAcquire() { - return true; - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiter.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiter.java deleted file mode 100644 index be2017d32..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiter.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.ratelimiter; - -/** Interface for rate limiting requests */ -public interface RateLimiter { - /** - * This signifies that a request is being made. That is, the rate limiter should count the request - * at this point. - * - * @return Whether the request is allowed to proceed by the rate limiter - */ - boolean tryAcquire(); -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiterFilter.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiterFilter.java deleted file mode 100644 index 725aa8734..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiterFilter.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.ratelimiter; - -import jakarta.annotation.Priority; -import jakarta.inject.Inject; -import jakarta.ws.rs.Priorities; -import jakarta.ws.rs.container.ContainerRequestContext; -import jakarta.ws.rs.container.ContainerRequestFilter; -import jakarta.ws.rs.core.Response; -import jakarta.ws.rs.ext.Provider; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** Request filter that returns a 429 Too Many Requests if the rate limiter says so */ -@Priority(Priorities.AUTHORIZATION + 1) -@Provider -public class RateLimiterFilter implements ContainerRequestFilter { - private static final Logger LOGGER = LoggerFactory.getLogger(RateLimiterFilter.class); - - private final RateLimiter rateLimiter; - - @Inject - public RateLimiterFilter(RateLimiter rateLimiter) { - this.rateLimiter = rateLimiter; - } - - /** Returns a 429 if the rate limiter says so. Otherwise, forwards the request along. */ - @Override - public void filter(ContainerRequestContext containerRequestContext) { - if (!rateLimiter.tryAcquire()) { - containerRequestContext.abortWith(Response.status(Response.Status.TOO_MANY_REQUESTS).build()); - LOGGER.atDebug().log("Rate limiting request"); - } - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/ratelimiter/RealmTokenBucketRateLimiter.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/ratelimiter/RealmTokenBucketRateLimiter.java deleted file mode 100644 index 56053a0c2..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/ratelimiter/RealmTokenBucketRateLimiter.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.ratelimiter; - -import io.quarkus.arc.lookup.LookupIfProperty; -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.inject.Inject; -import java.time.Clock; -import java.time.Duration; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.service.config.RuntimeCandidate; -import org.eclipse.microprofile.config.inject.ConfigProperty; - -/** - * Rate limiter that maps the request's realm identifier to its own TokenBucketRateLimiter, with its - * own capacity. - */ -@ApplicationScoped -@RuntimeCandidate -@LookupIfProperty(name = "polaris.rate-limiter.type", stringValue = "realm-token-bucket") -public class RealmTokenBucketRateLimiter implements RateLimiter { - private final long requestsPerSecond; - private final Duration window; - private final Map perRealmLimiters; - private final Clock clock; - - @Inject - public RealmTokenBucketRateLimiter( - @ConfigProperty(name = "polaris.rate-limiter.realm-token-bucket.requests-per-second") - long requestsPerSecond, - @ConfigProperty(name = "polaris.rate-limiter.realm-token-bucket.window") Duration window, - Clock clock) { - this.requestsPerSecond = requestsPerSecond; - this.window = window; - this.clock = clock; - this.perRealmLimiters = new ConcurrentHashMap<>(); - } - - /** - * This signifies that a request is being made. That is, the rate limiter should count the request - * at this point. - * - * @return Whether the request is allowed to proceed by the rate limiter - */ - @Override - public boolean tryAcquire() { - String key = - Optional.ofNullable(CallContext.getCurrentContext()) - .map(CallContext::getRealmContext) - .map(RealmContext::getRealmIdentifier) - .orElse(""); - - return perRealmLimiters - .computeIfAbsent( - key, - (k) -> - new TokenBucketRateLimiter( - requestsPerSecond, - Math.multiplyExact(requestsPerSecond, window.getSeconds()), - clock)) - .tryAcquire(); - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/ratelimiter/TokenBucketRateLimiter.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/ratelimiter/TokenBucketRateLimiter.java deleted file mode 100644 index 65e635f48..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/ratelimiter/TokenBucketRateLimiter.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.ratelimiter; - -import jakarta.enterprise.inject.Vetoed; -import java.time.InstantSource; - -/** - * Token bucket implementation of a Polaris RateLimiter. Acquires tokens at a fixed rate and has a - * maximum amount of tokens. Each successful "tryAcquire" costs 1 token. - */ -@Vetoed -public class TokenBucketRateLimiter implements RateLimiter { - private final double tokensPerMilli; - private final long maxTokens; - private final InstantSource instantSource; - - private double tokens; - private long lastTokenGenerationMillis; - - public TokenBucketRateLimiter(long tokensPerSecond, long maxTokens, InstantSource instantSource) { - this.tokensPerMilli = tokensPerSecond / 1000D; - this.maxTokens = maxTokens; - this.instantSource = instantSource; - - tokens = maxTokens; - lastTokenGenerationMillis = instantSource.millis(); - } - - /** - * Tries to acquire and spend 1 token. Doesn't block if a token isn't available. - * - * @return whether a token was successfully acquired and spent - */ - @Override - public synchronized boolean tryAcquire() { - // Grant tokens for the time that has passed since our last tryAcquire() - long t = instantSource.millis(); - long millisPassed = Math.subtractExact(t, lastTokenGenerationMillis); - lastTokenGenerationMillis = t; - tokens = Math.min(maxTokens, tokens + (millisPassed * tokensPerMilli)); - - // Take a token if they have one available - if (tokens >= 1) { - tokens--; - return true; - } - return false; - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java deleted file mode 100644 index 52917dddf..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.storage; - -import com.google.api.client.http.javanet.NetHttpTransport; -import com.google.auth.http.HttpTransportFactory; -import com.google.auth.oauth2.AccessToken; -import com.google.auth.oauth2.GoogleCredentials; -import com.google.cloud.ServiceOptions; -import jakarta.annotation.Nonnull; -import jakarta.annotation.Nullable; -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.inject.Inject; -import java.io.IOException; -import java.time.Duration; -import java.time.Instant; -import java.util.*; -import java.util.function.Supplier; -import org.apache.polaris.core.PolarisDiagnostics; -import org.apache.polaris.core.storage.PolarisCredentialProperty; -import org.apache.polaris.core.storage.PolarisStorageActions; -import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; -import org.apache.polaris.core.storage.PolarisStorageIntegration; -import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; -import org.apache.polaris.core.storage.aws.AwsCredentialsStorageIntegration; -import org.apache.polaris.core.storage.azure.AzureCredentialsStorageIntegration; -import org.apache.polaris.core.storage.gcp.GcpCredentialsStorageIntegration; -import org.eclipse.microprofile.config.inject.ConfigProperty; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.services.sts.StsClient; -import software.amazon.awssdk.services.sts.StsClientBuilder; - -@ApplicationScoped -public class PolarisStorageIntegrationProviderImpl implements PolarisStorageIntegrationProvider { - - private static final Logger LOGGER = - LoggerFactory.getLogger(PolarisStorageIntegrationProviderImpl.class); - - private final Supplier stsClientSupplier; - private final Supplier gcpCredsProvider; - - @Inject - public PolarisStorageIntegrationProviderImpl( - @ConfigProperty(name = "polaris.storage.aws.awsAccessKey") Optional awsAccessKey, - @ConfigProperty(name = "polaris.storage.aws.awsSecretKey") Optional awsSecretKey, - @ConfigProperty(name = "polaris.storage.gcp.token") Optional gcpAccessToken, - @ConfigProperty(name = "polaris.storage.gcp.lifespan") Optional lifespan) { - // TODO clean up this constructor, use bean injection with qualifier for configuration for each - // provider (AWS, GCP, ...) - this( - () -> { - StsClientBuilder stsClientBuilder = StsClient.builder(); - if (!awsAccessKey.get().isBlank() && !awsSecretKey.get().isBlank()) { - LOGGER.warn( - "Using hard-coded AWS credentials - this is not recommended for production"); - StaticCredentialsProvider awsCredentialsProvider = - StaticCredentialsProvider.create( - AwsBasicCredentials.create(awsAccessKey.get(), awsSecretKey.get())); - stsClientBuilder.credentialsProvider(awsCredentialsProvider); - } - return stsClientBuilder.build(); - }, - () -> { - if (gcpAccessToken.get().isBlank()) { - try { - return GoogleCredentials.getApplicationDefault(); - } catch (IOException e) { - throw new RuntimeException("Failed to get GCP credentials", e); - } - } else { - AccessToken accessToken = - new AccessToken( - gcpAccessToken.get(), - new Date(Instant.now().plus(lifespan.get()).toEpochMilli())); - return GoogleCredentials.create(accessToken); - } - }); - } - - public PolarisStorageIntegrationProviderImpl( - Supplier stsClientSupplier, Supplier gcpCredsProvider) { - this.stsClientSupplier = stsClientSupplier; - this.gcpCredsProvider = gcpCredsProvider; - } - - @Override - @SuppressWarnings("unchecked") - public @Nullable - PolarisStorageIntegration getStorageIntegrationForConfig( - PolarisStorageConfigurationInfo polarisStorageConfigurationInfo) { - if (polarisStorageConfigurationInfo == null) { - return null; - } - PolarisStorageIntegration storageIntegration; - switch (polarisStorageConfigurationInfo.getStorageType()) { - case S3: - storageIntegration = - (PolarisStorageIntegration) - new AwsCredentialsStorageIntegration(stsClientSupplier.get()); - break; - case GCS: - storageIntegration = - (PolarisStorageIntegration) - new GcpCredentialsStorageIntegration( - gcpCredsProvider.get(), - ServiceOptions.getFromServiceLoader( - HttpTransportFactory.class, NetHttpTransport::new)); - break; - case AZURE: - storageIntegration = - (PolarisStorageIntegration) new AzureCredentialsStorageIntegration(); - break; - case FILE: - storageIntegration = - new PolarisStorageIntegration<>("file") { - @Override - public EnumMap getSubscopedCreds( - @Nonnull PolarisDiagnostics diagnostics, - @Nonnull T storageConfig, - boolean allowListOperation, - @Nonnull Set allowedReadLocations, - @Nonnull Set allowedWriteLocations) { - return new EnumMap<>(PolarisCredentialProperty.class); - } - - @Override - public @Nonnull Map> - validateAccessToLocations( - @Nonnull T storageConfig, - @Nonnull Set actions, - @Nonnull Set locations) { - return Map.of(); - } - }; - break; - default: - throw new IllegalArgumentException( - "Unknown storage type " + polarisStorageConfigurationInfo.getStorageType()); - } - return storageIntegration; - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/task/ManifestFileCleanupTaskHandler.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/task/ManifestFileCleanupTaskHandler.java deleted file mode 100644 index e17e24fd7..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/task/ManifestFileCleanupTaskHandler.java +++ /dev/null @@ -1,225 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.task; - -import java.io.IOException; -import java.util.List; -import java.util.Objects; -import java.util.Spliterator; -import java.util.Spliterators; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.function.Function; -import java.util.stream.StreamSupport; -import org.apache.commons.codec.binary.Base64; -import org.apache.iceberg.DataFile; -import org.apache.iceberg.ManifestFile; -import org.apache.iceberg.ManifestFiles; -import org.apache.iceberg.ManifestReader; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.io.FileIO; -import org.apache.polaris.core.entity.AsyncTaskType; -import org.apache.polaris.core.entity.TaskEntity; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * {@link TaskHandler} responsible for deleting all of the files in a manifest and the manifest - * itself. Since data files may be present in multiple manifests across different snapshots, we - * assume a data file that doesn't exist is missing because it was already deleted by another task. - */ -public class ManifestFileCleanupTaskHandler implements TaskHandler { - public static final int MAX_ATTEMPTS = 3; - public static final int FILE_DELETION_RETRY_MILLIS = 100; - private static final Logger LOGGER = - LoggerFactory.getLogger(ManifestFileCleanupTaskHandler.class); - private final Function fileIOSupplier; - private final ExecutorService executorService; - - public ManifestFileCleanupTaskHandler( - Function fileIOSupplier, ExecutorService executorService) { - this.fileIOSupplier = fileIOSupplier; - this.executorService = executorService; - } - - @Override - public boolean canHandleTask(TaskEntity task) { - return task.getTaskType() == AsyncTaskType.FILE_CLEANUP; - } - - @Override - public boolean handleTask(TaskEntity task) { - ManifestCleanupTask cleanupTask = task.readData(ManifestCleanupTask.class); - ManifestFile manifestFile = decodeManifestData(cleanupTask.getManifestFileData()); - TableIdentifier tableId = cleanupTask.getTableId(); - try (FileIO authorizedFileIO = fileIOSupplier.apply(task)) { - - // if the file doesn't exist, we assume that another task execution was successful, but failed - // to drop the task entity. Log a warning and return success - if (!TaskUtils.exists(manifestFile.path(), authorizedFileIO)) { - LOGGER - .atWarn() - .addKeyValue("manifestFile", manifestFile.path()) - .addKeyValue("tableId", tableId) - .log("Manifest cleanup task scheduled, but manifest file doesn't exist"); - return true; - } - - ManifestReader dataFiles = ManifestFiles.read(manifestFile, authorizedFileIO); - List> dataFileDeletes = - StreamSupport.stream( - Spliterators.spliteratorUnknownSize(dataFiles.iterator(), Spliterator.IMMUTABLE), - false) - .map( - file -> - tryDelete( - tableId, authorizedFileIO, manifestFile, file.path().toString(), null, 1)) - .toList(); - LOGGER.debug( - "Scheduled {} data files to be deleted from manifest {}", - dataFileDeletes.size(), - manifestFile.path()); - try { - // wait for all data files to be deleted, then wait for the manifest itself to be deleted - CompletableFuture.allOf(dataFileDeletes.toArray(CompletableFuture[]::new)) - .thenCompose( - (v) -> { - LOGGER - .atInfo() - .addKeyValue("manifestFile", manifestFile.path()) - .log("All data files in manifest deleted - deleting manifest"); - return tryDelete( - tableId, authorizedFileIO, manifestFile, manifestFile.path(), null, 1); - }) - .get(); - return true; - } catch (InterruptedException e) { - LOGGER.error( - "Interrupted exception deleting data files from manifest {}", manifestFile.path(), e); - throw new RuntimeException(e); - } catch (ExecutionException e) { - LOGGER.error("Unable to delete data files from manifest {}", manifestFile.path(), e); - return false; - } - } - } - - private static ManifestFile decodeManifestData(String manifestFileData) { - try { - return ManifestFiles.decode(Base64.decodeBase64(manifestFileData)); - } catch (IOException e) { - throw new RuntimeException("Unable to decode base64 encoded manifest", e); - } - } - - private CompletableFuture tryDelete( - TableIdentifier tableId, - FileIO fileIO, - ManifestFile manifestFile, - String dataFile, - Throwable e, - int attempt) { - if (e != null && attempt <= MAX_ATTEMPTS) { - LOGGER - .atWarn() - .addKeyValue("dataFile", dataFile) - .addKeyValue("attempt", attempt) - .addKeyValue("error", e.getMessage()) - .log("Error encountered attempting to delete data file"); - } - if (attempt > MAX_ATTEMPTS && e != null) { - return CompletableFuture.failedFuture(e); - } - return CompletableFuture.runAsync( - () -> { - // totally normal for a file to already be missing, as a data file - // may be in multiple manifests. There's a possibility we check the - // file's existence, but then it is deleted before we have a chance to - // send the delete request. In such a case, we should retry - // and find - if (TaskUtils.exists(dataFile, fileIO)) { - fileIO.deleteFile(dataFile); - } else { - LOGGER - .atInfo() - .addKeyValue("dataFile", dataFile) - .addKeyValue("manifestFile", manifestFile.path()) - .addKeyValue("tableId", tableId) - .log("Manifest cleanup task scheduled, but data file doesn't exist"); - } - }, - executorService) - .exceptionallyComposeAsync( - newEx -> { - LOGGER - .atWarn() - .addKeyValue("dataFile", dataFile) - .addKeyValue("tableIdentifer", tableId) - .addKeyValue("manifestFile", manifestFile.path()) - .log("Exception caught deleting data file from manifest", newEx); - return tryDelete(tableId, fileIO, manifestFile, dataFile, newEx, attempt + 1); - }, - CompletableFuture.delayedExecutor( - FILE_DELETION_RETRY_MILLIS, TimeUnit.MILLISECONDS, executorService)); - } - - /** Serialized Task data sent from the {@link TableCleanupTaskHandler} */ - public static final class ManifestCleanupTask { - private TableIdentifier tableId; - private String manifestFileData; - - public ManifestCleanupTask(TableIdentifier tableId, String manifestFileData) { - this.tableId = tableId; - this.manifestFileData = manifestFileData; - } - - public ManifestCleanupTask() {} - - public TableIdentifier getTableId() { - return tableId; - } - - public void setTableId(TableIdentifier tableId) { - this.tableId = tableId; - } - - public String getManifestFileData() { - return manifestFileData; - } - - public void setManifestFileData(String manifestFileData) { - this.manifestFileData = manifestFileData; - } - - @Override - public boolean equals(Object object) { - if (this == object) return true; - if (!(object instanceof ManifestCleanupTask that)) return false; - return Objects.equals(tableId, that.tableId) - && Objects.equals(manifestFileData, that.manifestFileData); - } - - @Override - public int hashCode() { - return Objects.hash(tableId, manifestFileData); - } - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/task/TableCleanupTaskHandler.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/task/TableCleanupTaskHandler.java deleted file mode 100644 index 7f323174b..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/task/TableCleanupTaskHandler.java +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.task; - -import java.util.List; -import java.util.UUID; -import java.util.function.Function; -import java.util.stream.Collectors; -import org.apache.iceberg.ManifestFile; -import org.apache.iceberg.TableMetadata; -import org.apache.iceberg.TableMetadataParser; -import org.apache.iceberg.io.FileIO; -import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.entity.AsyncTaskType; -import org.apache.polaris.core.entity.PolarisBaseEntity; -import org.apache.polaris.core.entity.PolarisEntity; -import org.apache.polaris.core.entity.PolarisEntityType; -import org.apache.polaris.core.entity.TableLikeEntity; -import org.apache.polaris.core.entity.TaskEntity; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Table cleanup handler resolves the latest {@link TableMetadata} file for a dropped table and - * schedules a deletion task for each Snapshot found in the {@link TableMetadata}. Manifest - * cleanup tasks are scheduled in a batch so tasks should be stored atomically. - */ -public class TableCleanupTaskHandler implements TaskHandler { - private static final Logger LOGGER = LoggerFactory.getLogger(TableCleanupTaskHandler.class); - private final TaskExecutor taskExecutor; - private final MetaStoreManagerFactory metaStoreManagerFactory; - private final Function fileIOSupplier; - - public TableCleanupTaskHandler( - TaskExecutor taskExecutor, - MetaStoreManagerFactory metaStoreManagerFactory, - Function fileIOSupplier) { - this.taskExecutor = taskExecutor; - this.metaStoreManagerFactory = metaStoreManagerFactory; - this.fileIOSupplier = fileIOSupplier; - } - - @Override - public boolean canHandleTask(TaskEntity task) { - return task.getTaskType() == AsyncTaskType.ENTITY_CLEANUP_SCHEDULER && taskEntityIsTable(task); - } - - private boolean taskEntityIsTable(TaskEntity task) { - PolarisEntity entity = PolarisEntity.of((task.readData(PolarisBaseEntity.class))); - return entity.getType().equals(PolarisEntityType.TABLE_LIKE); - } - - @Override - public boolean handleTask(TaskEntity cleanupTask) { - PolarisBaseEntity entity = cleanupTask.readData(PolarisBaseEntity.class); - PolarisMetaStoreManager metaStoreManager = - metaStoreManagerFactory.getOrCreateMetaStoreManager( - CallContext.getCurrentContext().getRealmContext()); - TableLikeEntity tableEntity = TableLikeEntity.of(entity); - PolarisCallContext polarisCallContext = CallContext.getCurrentContext().getPolarisCallContext(); - LOGGER - .atInfo() - .addKeyValue("tableIdentifier", tableEntity.getTableIdentifier()) - .addKeyValue("metadataLocation", tableEntity.getMetadataLocation()) - .log("Handling table metadata cleanup task"); - - // It's likely the cleanupTask has already been completed, but wasn't dropped successfully. - // Log a - // warning and move on - try (FileIO fileIO = fileIOSupplier.apply(cleanupTask)) { - if (!TaskUtils.exists(tableEntity.getMetadataLocation(), fileIO)) { - LOGGER - .atWarn() - .addKeyValue("tableIdentifier", tableEntity.getTableIdentifier()) - .addKeyValue("metadataLocation", tableEntity.getMetadataLocation()) - .log("Table metadata cleanup scheduled, but metadata file does not exist"); - return true; - } - - TableMetadata tableMetadata = - TableMetadataParser.read(fileIO, tableEntity.getMetadataLocation()); - - // read the manifest list for each snapshot. dedupe the manifest files and schedule a - // cleanupTask - // for each manifest file and its data files to be deleted - List taskEntities = - tableMetadata.snapshots().stream() - .flatMap(sn -> sn.allManifests(fileIO).stream()) - // distinct by manifest path, since multiple snapshots will contain the same - // manifest - .collect(Collectors.toMap(ManifestFile::path, Function.identity(), (mf1, mf2) -> mf1)) - .values() - .stream() - .filter(mf -> TaskUtils.exists(mf.path(), fileIO)) - .map( - mf -> { - // append a random uuid to the task name to avoid any potential conflict - // when - // storing the task entity. It's better to have duplicate tasks than to risk - // not storing the rest of the task entities. If a duplicate deletion task - // is - // queued, it will check for the manifest file's existence and simply exit - // if - // the task has already been handled. - String taskName = - cleanupTask.getName() + "_" + mf.path() + "_" + UUID.randomUUID(); - LOGGER - .atDebug() - .addKeyValue("taskName", taskName) - .addKeyValue("tableIdentifier", tableEntity.getTableIdentifier()) - .addKeyValue("metadataLocation", tableEntity.getMetadataLocation()) - .addKeyValue("manifestFile", mf.path()) - .log("Queueing task to delete manifest file"); - return new TaskEntity.Builder() - .setName(taskName) - .setId(metaStoreManager.generateNewEntityId(polarisCallContext).getId()) - .setCreateTimestamp(polarisCallContext.getClock().millis()) - .withTaskType(AsyncTaskType.FILE_CLEANUP) - .withData( - new ManifestFileCleanupTaskHandler.ManifestCleanupTask( - tableEntity.getTableIdentifier(), TaskUtils.encodeManifestFile(mf))) - .setId(metaStoreManager.generateNewEntityId(polarisCallContext).getId()) - // copy the internal properties, which will have storage info - .setInternalProperties(cleanupTask.getInternalPropertiesAsMap()) - .build(); - }) - .toList(); - List createdTasks = - metaStoreManager - .createEntitiesIfNotExist(polarisCallContext, null, taskEntities) - .getEntities(); - if (createdTasks != null) { - LOGGER - .atInfo() - .addKeyValue("tableIdentifier", tableEntity.getTableIdentifier()) - .addKeyValue("metadataLocation", tableEntity.getMetadataLocation()) - .addKeyValue("taskCount", taskEntities.size()) - .log("Successfully queued tasks to delete manifests - deleting table metadata file"); - for (PolarisBaseEntity createdTask : createdTasks) { - taskExecutor.addTaskHandlerContext(createdTask.getId(), CallContext.getCurrentContext()); - } - fileIO.deleteFile(tableEntity.getMetadataLocation()); - - return true; - } - } - return false; - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/task/TaskExecutor.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/task/TaskExecutor.java deleted file mode 100644 index 016518e6f..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/task/TaskExecutor.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.task; - -import org.apache.polaris.core.context.CallContext; - -/** - * Execute a task asynchronously with a provided context. The context must be cloned so that callers - * can close their own context and closables - */ -public interface TaskExecutor { - void addTaskHandlerContext(long taskEntityId, CallContext callContext); -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/task/TaskExecutorImpl.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/task/TaskExecutorImpl.java deleted file mode 100644 index 748bcadd2..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/task/TaskExecutorImpl.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.task; - -import io.quarkus.runtime.Startup; -import jakarta.annotation.Nonnull; -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.inject.Inject; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.entity.PolarisBaseEntity; -import org.apache.polaris.core.entity.PolarisEntity; -import org.apache.polaris.core.entity.PolarisEntityType; -import org.apache.polaris.core.entity.TaskEntity; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.service.config.TaskHandlerConfiguration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@ApplicationScoped -public class TaskExecutorImpl implements TaskExecutor { - private static final Logger LOGGER = LoggerFactory.getLogger(TaskExecutorImpl.class); - public static final long TASK_RETRY_DELAY = 1000; - private final ExecutorService executorService; - private final MetaStoreManagerFactory metaStoreManagerFactory; - private final TaskFileIOSupplier fileIOSupplier; - private final List taskHandlers = new ArrayList<>(); - - @Inject - public TaskExecutorImpl( - TaskHandlerConfiguration taskHandlerConfiguration, - MetaStoreManagerFactory metaStoreManagerFactory, - TaskFileIOSupplier fileIOSupplier) { - this.executorService = taskHandlerConfiguration.executorService(); - this.metaStoreManagerFactory = metaStoreManagerFactory; - this.fileIOSupplier = fileIOSupplier; - } - - @Startup - public void init() { - addTaskHandler(new TableCleanupTaskHandler(this, metaStoreManagerFactory, fileIOSupplier)); - addTaskHandler( - new ManifestFileCleanupTaskHandler( - fileIOSupplier, Executors.newVirtualThreadPerTaskExecutor())); - } - - /** - * Add a {@link TaskHandler}. {@link TaskEntity}s will be tested against the {@link - * TaskHandler#canHandleTask(TaskEntity)} method and will be handled by the first handler that - * responds true. - */ - public void addTaskHandler(TaskHandler taskHandler) { - taskHandlers.add(taskHandler); - } - - /** - * Register a {@link CallContext} for a specific task id. That task will be loaded and executed - * asynchronously with a clone of the provided {@link CallContext}. - */ - @Override - public void addTaskHandlerContext(long taskEntityId, CallContext callContext) { - CallContext clone = CallContext.copyOf(callContext); - tryHandleTask(taskEntityId, clone, null, 1); - } - - private @Nonnull CompletableFuture tryHandleTask( - long taskEntityId, CallContext clone, Throwable e, int attempt) { - if (attempt > 3) { - return CompletableFuture.failedFuture(e); - } - return CompletableFuture.runAsync( - () -> { - // set the call context INSIDE the async task - try (CallContext ctx = CallContext.setCurrentContext(CallContext.copyOf(clone))) { - PolarisMetaStoreManager metaStoreManager = - metaStoreManagerFactory.getOrCreateMetaStoreManager(ctx.getRealmContext()); - PolarisBaseEntity taskEntity = - metaStoreManager - .loadEntity(ctx.getPolarisCallContext(), 0L, taskEntityId) - .getEntity(); - if (!PolarisEntityType.TASK.equals(taskEntity.getType())) { - throw new IllegalArgumentException("Provided taskId must be a task entity type"); - } - TaskEntity task = TaskEntity.of(taskEntity); - Optional handlerOpt = - taskHandlers.stream().filter(th -> th.canHandleTask(task)).findFirst(); - if (handlerOpt.isEmpty()) { - LOGGER - .atWarn() - .addKeyValue("taskEntityId", taskEntityId) - .addKeyValue("taskType", task.getTaskType()) - .log("Unable to find handler for task type"); - return; - } - TaskHandler handler = handlerOpt.get(); - boolean success = handler.handleTask(task); - if (success) { - LOGGER - .atInfo() - .addKeyValue("taskEntityId", taskEntityId) - .addKeyValue("handlerClass", handler.getClass()) - .log("Task successfully handled"); - metaStoreManager.dropEntityIfExists( - ctx.getPolarisCallContext(), - null, - PolarisEntity.toCore(taskEntity), - Map.of(), - false); - } else { - LOGGER - .atWarn() - .addKeyValue("taskEntityId", taskEntityId) - .addKeyValue("taskEntityName", taskEntity.getName()) - .log("Unable to execute async task"); - } - } - }, - executorService) - .exceptionallyComposeAsync( - (t) -> { - LOGGER.warn("Failed to handle task entity id {}", taskEntityId, t); - return tryHandleTask(taskEntityId, clone, t, attempt + 1); - }, - CompletableFuture.delayedExecutor( - TASK_RETRY_DELAY * (long) attempt, TimeUnit.MILLISECONDS, executorService)); - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java deleted file mode 100644 index e63c2e987..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.task; - -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.inject.Inject; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import java.util.function.Function; -import org.apache.iceberg.CatalogProperties; -import org.apache.iceberg.io.FileIO; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.entity.PolarisTaskConstants; -import org.apache.polaris.core.entity.TaskEntity; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.service.catalog.io.FileIOFactory; - -@ApplicationScoped -public class TaskFileIOSupplier implements Function { - private final MetaStoreManagerFactory metaStoreManagerFactory; - private final FileIOFactory fileIOFactory; - - @Inject - public TaskFileIOSupplier( - MetaStoreManagerFactory metaStoreManagerFactory, FileIOFactory fileIOFactory) { - this.metaStoreManagerFactory = metaStoreManagerFactory; - this.fileIOFactory = fileIOFactory; - } - - @Override - public FileIO apply(TaskEntity task) { - Map internalProperties = task.getInternalPropertiesAsMap(); - String location = internalProperties.get(PolarisTaskConstants.STORAGE_LOCATION); - PolarisMetaStoreManager metaStoreManager = - metaStoreManagerFactory.getOrCreateMetaStoreManager( - CallContext.getCurrentContext().getRealmContext()); - Map properties = new HashMap<>(internalProperties); - properties.putAll( - metaStoreManagerFactory - .getOrCreateStorageCredentialCache(CallContext.getCurrentContext().getRealmContext()) - .getOrGenerateSubScopeCreds( - metaStoreManager, - CallContext.getCurrentContext().getPolarisCallContext(), - task, - true, - Set.of(location), - Set.of(location))); - String ioImpl = - properties.getOrDefault( - CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.io.ResolvingFileIO"); - return fileIOFactory.loadFileIO(ioImpl, properties); - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/task/TaskHandler.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/task/TaskHandler.java deleted file mode 100644 index f903ddf80..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/task/TaskHandler.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.task; - -import org.apache.polaris.core.entity.TaskEntity; - -public interface TaskHandler { - boolean canHandleTask(TaskEntity task); - - boolean handleTask(TaskEntity task); -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/task/TaskUtils.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/task/TaskUtils.java deleted file mode 100644 index 7c1de08be..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/task/TaskUtils.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.task; - -import java.io.IOException; -import org.apache.commons.codec.binary.Base64; -import org.apache.iceberg.ManifestFile; -import org.apache.iceberg.ManifestFiles; -import org.apache.iceberg.exceptions.NotFoundException; -import org.apache.iceberg.io.FileIO; - -public class TaskUtils { - static boolean exists(String path, FileIO fileIO) { - try { - return fileIO.newInputFile(path).exists(); - } catch (NotFoundException e) { - // in-memory FileIO throws this exception - return false; - } catch (Exception e) { - // typically, clients will catch a 404 and simply return false, so any other exception - // means something probably went wrong - throw new RuntimeException(e); - } - } - - /** - * base64 encode the serialized manifest file entry so we can deserialize it and read the manifest - * in the {@link ManifestFileCleanupTaskHandler} - */ - static String encodeManifestFile(ManifestFile mf) { - try { - return Base64.encodeBase64String(ManifestFiles.encode(mf)); - } catch (IOException e) { - throw new RuntimeException("Unable to encode binary data in memory", e); - } - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/types/CommitTableRequest.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/types/CommitTableRequest.java deleted file mode 100644 index 3d36f923e..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/types/CommitTableRequest.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.types; - -import org.apache.iceberg.rest.requests.UpdateTableRequest; - -public class CommitTableRequest extends UpdateTableRequest {} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/types/CommitViewRequest.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/types/CommitViewRequest.java deleted file mode 100644 index b6f623b65..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/types/CommitViewRequest.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.types; - -import org.apache.iceberg.rest.requests.UpdateTableRequest; - -public class CommitViewRequest extends UpdateTableRequest {} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/types/NotificationRequest.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/types/NotificationRequest.java deleted file mode 100644 index c73304bf8..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/types/NotificationRequest.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.types; - -import com.fasterxml.jackson.annotation.JsonProperty; -import io.swagger.annotations.ApiModelProperty; -import java.util.Objects; - -@jakarta.annotation.Generated( - value = "org.openapitools.codegen.languages.JavaResteasyServerCodegen", - date = "2024-05-25T00:53:53.298853423Z[UTC]", - comments = "Generator version: 7.5.0") -public class NotificationRequest { - - private NotificationType notificationType; - private TableUpdateNotification payload; - - /** */ - @ApiModelProperty(required = true, value = "") - @JsonProperty("notification-type") - public NotificationType getNotificationType() { - return notificationType; - } - - public void setNotificationType(NotificationType notificationType) { - this.notificationType = notificationType; - } - - /** */ - @ApiModelProperty(value = "") - @JsonProperty("payload") - public TableUpdateNotification getPayload() { - return payload; - } - - public void setPayload(TableUpdateNotification payload) { - this.payload = payload; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - NotificationRequest notificationRequest = (NotificationRequest) o; - return Objects.equals(this.notificationType, notificationRequest.notificationType) - && Objects.equals(this.payload, notificationRequest.payload); - } - - @Override - public int hashCode() { - return Objects.hash(notificationType, payload); - } - - @Override - public String toString() { - return """ - class NotificationRequest { - notificationType: %s - payload: %s - }""" - .formatted(toIndentedString(notificationType), toIndentedString(payload)); - } - - /** - * Convert the given object to string with each line indented by 4 spaces (except the first line). - */ - private static String toIndentedString(Object o) { - if (o == null) { - return "null"; - } - return o.toString().replace("\n", "\n "); - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/types/NotificationType.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/types/NotificationType.java deleted file mode 100644 index 53d7d4777..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/types/NotificationType.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.types; - -import java.util.Arrays; -import java.util.Locale; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; - -public enum NotificationType { - - /** Supported notification types for the update table notification. */ - UNKNOWN(0, "UNKNOWN"), - CREATE(1, "CREATE"), - UPDATE(2, "UPDATE"), - DROP(3, "DROP"), - VALIDATE(4, "VALIDATE"); - - NotificationType(int id, String displayName) { - this.id = id; - this.displayName = displayName; - } - - /** Internal id of the notification type. */ - private final int id; - - /** Display name of the notification type */ - private final String displayName; - - /** Internal ids and their corresponding sources of notification types. */ - private static final Map idToNotificationTypeMap = - Arrays.stream(NotificationType.values()) - .collect(Collectors.toMap(NotificationType::getId, tf -> tf)); - - /** - * Lookup a notification type using its internal id representation - * - * @param id internal id of the notification type - * @return The notification type, if it exists, or empty - */ - public static Optional lookupById(int id) { - return Optional.ofNullable(idToNotificationTypeMap.get(id)); - } - - /** - * Return the internal id of the notification type - * - * @return id - */ - public int getId() { - return id; - } - - /** Return the display name of the notification type */ - public String getDisplayName() { - return displayName; - } - - /** - * Find the notification type by name, or return an empty optional - * - * @param name name of the notification type - * @return The notification type, if it exists, or empty - */ - public static Optional lookupByName(String name) { - if (name == null) { - return Optional.empty(); - } - - for (NotificationType NotificationType : NotificationType.values()) { - if (name.toUpperCase(Locale.ROOT).equals(NotificationType.name())) { - return Optional.of(NotificationType); - } - } - return Optional.empty(); - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/types/TableUpdateNotification.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/types/TableUpdateNotification.java deleted file mode 100644 index 4a6326872..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/types/TableUpdateNotification.java +++ /dev/null @@ -1,203 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.types; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Preconditions; -import io.swagger.annotations.ApiModelProperty; -import java.util.Objects; -import org.apache.iceberg.TableMetadata; - -public class TableUpdateNotification { - - private String tableName; - private Long timestamp; - private String tableUuid; - private String metadataLocation; - private TableMetadata metadata; - - /** */ - @ApiModelProperty(required = true, value = "") - @JsonProperty("table-name") - public String getTableName() { - return tableName; - } - - public void setTableName(String tableName) { - this.tableName = tableName; - } - - /** */ - @ApiModelProperty(required = true, value = "") - @JsonProperty("timestamp") - public Long getTimestamp() { - return timestamp; - } - - public void setTimestamp(Long timestamp) { - this.timestamp = timestamp; - } - - /** */ - @ApiModelProperty(required = true, value = "") - @JsonProperty("table-uuid") - public String getTableUuid() { - return tableUuid; - } - - public void setTableUuid(String tableUuid) { - this.tableUuid = tableUuid; - } - - /** */ - @ApiModelProperty(required = true, value = "") - @JsonProperty("metadata-location") - public String getMetadataLocation() { - return metadataLocation; - } - - public void setMetadataLocation(String metadataLocation) { - this.metadataLocation = metadataLocation; - } - - /** */ - @ApiModelProperty(required = true, value = "") - @JsonProperty("metadata") - public TableMetadata getMetadata() { - return metadata; - } - - public void setMetadata(TableMetadata metadata) { - this.metadata = metadata; - } - - public TableUpdateNotification() {} - - public TableUpdateNotification( - String tableName, - Long timestamp, - String tableUuid, - String metadataLocation, - TableMetadata metadata) { - this.tableName = tableName; - this.timestamp = timestamp; - this.tableUuid = tableUuid; - this.metadataLocation = metadataLocation; - this.metadata = metadata; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - TableUpdateNotification tableUpdateNotification = (TableUpdateNotification) o; - return Objects.equals(this.tableName, tableUpdateNotification.tableName) - && Objects.equals(this.timestamp, tableUpdateNotification.timestamp) - && Objects.equals(this.tableUuid, tableUpdateNotification.tableUuid) - && Objects.equals(this.metadataLocation, tableUpdateNotification.metadataLocation) - && Objects.equals(this.metadata, tableUpdateNotification.metadata); - } - - @Override - public int hashCode() { - return Objects.hash(tableName, timestamp, tableUuid, metadataLocation, metadata); - } - - @Override - public String toString() { - return """ - class TableUpdateNotification { - tableName: %s - timestamp: %s - tableUuid: %s - metadataLocation: %s - metadata: %s - }""" - .formatted( - toIndentedString(tableName), - toIndentedString(timestamp), - toIndentedString(tableUuid), - toIndentedString(metadataLocation), - toIndentedString(metadata)); - } - - /** - * Convert the given object to string with each line indented by 4 spaces (except the first line). - */ - private static String toIndentedString(Object o) { - if (o == null) { - return "null"; - } - return o.toString().replace("\n", "\n "); - } - - public static Builder builder() { - return new Builder(); - } - - public static final class Builder { - - private String tableName; - private Long timestamp; - private String tableUuid; - private String metadataLocation; - private TableMetadata metadata; - - private Builder() {} - - public final Builder tableName(String tableName) { - Preconditions.checkArgument(tableName != null, "Null table name supplied"); - this.tableName = tableName; - return this; - } - - public final Builder timestamp(Long timestamp) { - Preconditions.checkArgument(timestamp != null, "timestamp can't be null"); - this.timestamp = timestamp; - return this; - } - - public final Builder metadataLocation(String metadataLocation) { - Preconditions.checkArgument(metadataLocation != null, "metadataLocation can't be null"); - this.metadataLocation = metadataLocation; - return this; - } - - public final Builder metadata(TableMetadata metadata) { - this.metadata = metadata; - return this; - } - - public final Builder tableUuid(String tableUuid) { - Preconditions.checkArgument(tableUuid != null, "timestamp can't be null"); - this.tableUuid = tableUuid; - return this; - } - - public TableUpdateNotification build() { - - return new TableUpdateNotification( - tableName, timestamp, tableUuid, metadataLocation, metadata); - } - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/types/TokenType.java b/polaris-service-quarkus/src/main/java/org/apache/polaris/service/types/TokenType.java deleted file mode 100644 index 1ef42c670..000000000 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/types/TokenType.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.types; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonValue; - -/** - * Token type identifier, from RFC - * 8693 Section 3 - */ -public enum TokenType { - ACCESS_TOKEN("urn:ietf:params:oauth:token-type:access_token"), - - REFRESH_TOKEN("urn:ietf:params:oauth:token-type:refresh_token"), - - ID_TOKEN("urn:ietf:params:oauth:token-type:id_token"), - - SAML1("urn:ietf:params:oauth:token-type:saml1"), - - SAML2("urn:ietf:params:oauth:token-type:saml2"), - - JWT("urn:ietf:params:oauth:token-type:jwt"); - - private final String value; - - TokenType(String value) { - this.value = value; - } - - @JsonValue - public String getValue() { - return value; - } - - @Override - public String toString() { - return String.valueOf(value); - } - - @JsonCreator - public static TokenType fromValue(String value) { - for (TokenType b : TokenType.values()) { - if (b.value.equals(value)) { - return b; - } - } - throw new IllegalArgumentException("Unexpected value '" + value + "'"); - } -} diff --git a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/PolarisApplicationIntegrationTest.java b/polaris-service-quarkus/src/test/java/org/apache/polaris/service/PolarisApplicationIntegrationTest.java deleted file mode 100644 index 01d8abf47..000000000 --- a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/PolarisApplicationIntegrationTest.java +++ /dev/null @@ -1,723 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service; - -import static org.apache.polaris.service.auth.BasePolarisAuthenticator.PRINCIPAL_ROLE_ALL; -import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.when; - -import io.quarkus.test.junit.QuarkusTest; -import jakarta.inject.Inject; -import jakarta.ws.rs.ProcessingException; -import jakarta.ws.rs.client.Entity; -import jakarta.ws.rs.client.Invocation; -import jakarta.ws.rs.core.Response; -import java.io.IOException; -import java.nio.file.Path; -import java.util.List; -import java.util.Map; -import org.apache.hadoop.conf.Configuration; -import org.apache.iceberg.BaseTable; -import org.apache.iceberg.PartitionData; -import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.Schema; -import org.apache.iceberg.SortOrder; -import org.apache.iceberg.Table; -import org.apache.iceberg.TableMetadata; -import org.apache.iceberg.TableMetadataParser; -import org.apache.iceberg.TestHelpers; -import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.catalog.SessionCatalog; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.exceptions.BadRequestException; -import org.apache.iceberg.exceptions.ForbiddenException; -import org.apache.iceberg.exceptions.NoSuchNamespaceException; -import org.apache.iceberg.exceptions.NoSuchTableException; -import org.apache.iceberg.exceptions.RESTException; -import org.apache.iceberg.hadoop.HadoopFileIO; -import org.apache.iceberg.io.ResolvingFileIO; -import org.apache.iceberg.rest.HTTPClient; -import org.apache.iceberg.rest.RESTClient; -import org.apache.iceberg.rest.RESTSessionCatalog; -import org.apache.iceberg.rest.auth.AuthConfig; -import org.apache.iceberg.rest.auth.OAuth2Properties; -import org.apache.iceberg.rest.auth.OAuth2Util; -import org.apache.iceberg.types.Types; -import org.apache.iceberg.util.EnvironmentUtil; -import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; -import org.apache.polaris.core.admin.model.Catalog; -import org.apache.polaris.core.admin.model.CatalogProperties; -import org.apache.polaris.core.admin.model.CatalogRole; -import org.apache.polaris.core.admin.model.ExternalCatalog; -import org.apache.polaris.core.admin.model.FileStorageConfigInfo; -import org.apache.polaris.core.admin.model.PolarisCatalog; -import org.apache.polaris.core.admin.model.PrincipalRole; -import org.apache.polaris.core.admin.model.StorageConfigInfo; -import org.apache.polaris.core.entity.CatalogEntity; -import org.apache.polaris.core.entity.PolarisEntityConstants; -import org.apache.polaris.service.test.PolarisIntegrationTestHelper; -import org.assertj.core.api.Assertions; -import org.assertj.core.api.InstanceOfAssertFactories; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInfo; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.io.TempDir; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@QuarkusTest -@TestInstance(TestInstance.Lifecycle.PER_CLASS) -public class PolarisApplicationIntegrationTest { - - private static final Logger LOGGER = - LoggerFactory.getLogger(PolarisApplicationIntegrationTest.class); - - public static final String PRINCIPAL_ROLE_NAME = "admin"; - - @Inject PolarisIntegrationTestHelper testHelper; - - @BeforeAll - public void setUp(TestInfo testInfo) { - testHelper.setUp(testInfo); - PrincipalRole principalRole = new PrincipalRole(PRINCIPAL_ROLE_NAME); - try (Response createPrResponse = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/management/v1/principal-roles", testHelper.localPort)) - .request("application/json") - .header("Authorization", "Bearer " + testHelper.adminToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .post(Entity.json(principalRole))) { - assertThat(createPrResponse) - .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - - try (Response assignPrResponse = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/management/v1/principals/snowman/principal-roles", - testHelper.localPort)) - .request("application/json") - .header("Authorization", "Bearer " + testHelper.adminToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .put(Entity.json(principalRole))) { - assertThat(assignPrResponse) - .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - } - - @AfterAll - public void tearDown() { - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/management/v1/principal-roles/%s", - testHelper.localPort, PRINCIPAL_ROLE_NAME)) - .request("application/json") - .header("Authorization", "Bearer " + testHelper.adminToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .delete() - .close(); - testHelper.tearDown(); - } - - /** - * Create a new catalog for each test case. Assign the snowman catalog-admin principal role the - * admin role of the new catalog. - * - * @param testInfo - */ - @BeforeEach - public void createTestCatalog(TestInfo testInfo) { - testInfo - .getTestMethod() - .ifPresent( - method -> { - String catalogName = method.getName(); - Catalog.TypeEnum catalogType = Catalog.TypeEnum.INTERNAL; - createCatalog(catalogName, catalogType, PRINCIPAL_ROLE_NAME); - }); - } - - private void createCatalog( - String catalogName, Catalog.TypeEnum catalogType, String principalRoleName) { - createCatalog( - catalogName, - catalogType, - principalRoleName, - AwsStorageConfigInfo.builder() - .setRoleArn("arn:aws:iam::123456789012:role/my-role") - .setExternalId("externalId") - .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) - .setAllowedLocations(List.of("s3://my-old-bucket/path/to/data")) - .build(), - "s3://my-bucket/path/to/data"); - } - - private void createCatalog( - String catalogName, - Catalog.TypeEnum catalogType, - String principalRoleName, - StorageConfigInfo storageConfig, - String defaultBaseLocation) { - CatalogProperties props = - CatalogProperties.builder(defaultBaseLocation) - .addProperty( - CatalogEntity.REPLACE_NEW_LOCATION_PREFIX_WITH_CATALOG_DEFAULT_KEY, "file:/") - .build(); - Catalog catalog = - catalogType.equals(Catalog.TypeEnum.INTERNAL) - ? PolarisCatalog.builder() - .setName(catalogName) - .setType(catalogType) - .setProperties(props) - .setStorageConfigInfo(storageConfig) - .build() - : ExternalCatalog.builder() - .setRemoteUrl("http://faraway.com") - .setName(catalogName) - .setType(catalogType) - .setProperties(props) - .setStorageConfigInfo(storageConfig) - .build(); - try (Response response = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs", testHelper.localPort)) - .request("application/json") - .header("Authorization", "Bearer " + testHelper.adminToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .post(Entity.json(catalog))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - try (Response response = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/%s", - testHelper.localPort, - catalogName, - PolarisEntityConstants.getNameOfCatalogAdminRole())) - .request("application/json") - .header("Authorization", "Bearer " + testHelper.adminToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .get()) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - CatalogRole catalogRole = response.readEntity(CatalogRole.class); - - try (Response assignResponse = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/management/v1/principal-roles/%s/catalog-roles/%s", - testHelper.localPort, principalRoleName, catalogName)) - .request("application/json") - .header("Authorization", "Bearer " + testHelper.adminToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .put(Entity.json(catalogRole))) { - assertThat(assignResponse) - .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - } - } - - private RESTSessionCatalog newSessionCatalog(String catalog) { - RESTSessionCatalog sessionCatalog = new RESTSessionCatalog(); - sessionCatalog.initialize( - "polaris_catalog_test", - Map.of( - "uri", - "http://localhost:" + testHelper.localPort + "/api/catalog", - OAuth2Properties.CREDENTIAL, - testHelper.snowmanCredentials.clientId() - + ":" - + testHelper.snowmanCredentials.clientSecret(), - OAuth2Properties.SCOPE, - PRINCIPAL_ROLE_ALL, - "warehouse", - catalog, - "header." + REALM_PROPERTY_KEY, - testHelper.realm)); - return sessionCatalog; - } - - @Test - public void testIcebergListNamespaces() throws IOException { - try (RESTSessionCatalog sessionCatalog = newSessionCatalog("testIcebergListNamespaces")) { - SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); - List namespaces = sessionCatalog.listNamespaces(sessionContext); - assertThat(namespaces).isNotNull().isEmpty(); - } - } - - @Test - public void testConfigureCatalogCaseSensitive() throws IOException { - assertThatThrownBy(() -> newSessionCatalog("TESTCONFIGURECATALOGCASESENSITIVE")) - .isInstanceOf(RESTException.class) - .hasMessage( - "Unable to process: Unable to find warehouse TESTCONFIGURECATALOGCASESENSITIVE"); - } - - @Test - public void testIcebergListNamespacesNotFound() throws IOException { - try (RESTSessionCatalog sessionCatalog = - newSessionCatalog("testIcebergListNamespacesNotFound")) { - SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); - assertThatThrownBy( - () -> sessionCatalog.listNamespaces(sessionContext, Namespace.of("whoops"))) - .isInstanceOf(NoSuchNamespaceException.class) - .hasMessage("Namespace does not exist: whoops"); - } - } - - @Test - public void testIcebergListNamespacesNestedNotFound() throws IOException { - try (RESTSessionCatalog sessionCatalog = - newSessionCatalog("testIcebergListNamespacesNestedNotFound")) { - SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); - Namespace topLevelNamespace = Namespace.of("top_level"); - sessionCatalog.createNamespace(sessionContext, topLevelNamespace); - sessionCatalog.loadNamespaceMetadata(sessionContext, Namespace.of("top_level")); - assertThatThrownBy( - () -> - sessionCatalog.listNamespaces( - sessionContext, Namespace.of("top_level", "whoops"))) - .isInstanceOf(NoSuchNamespaceException.class) - .hasMessage("Namespace does not exist: top_level.whoops"); - } - } - - @Test - public void testIcebergListTablesNamespaceNotFound() throws IOException { - try (RESTSessionCatalog sessionCatalog = - newSessionCatalog("testIcebergListTablesNamespaceNotFound")) { - SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); - assertThatThrownBy(() -> sessionCatalog.listTables(sessionContext, Namespace.of("whoops"))) - .isInstanceOf(NoSuchNamespaceException.class) - .hasMessage("Namespace does not exist: whoops"); - } - } - - @Test - public void testIcebergCreateNamespace() throws IOException { - try (RESTSessionCatalog sessionCatalog = newSessionCatalog("testIcebergCreateNamespace")) { - SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); - Namespace topLevelNamespace = Namespace.of("top_level"); - sessionCatalog.createNamespace(sessionContext, topLevelNamespace); - List namespaces = sessionCatalog.listNamespaces(sessionContext); - assertThat(namespaces).isNotNull().hasSize(1).containsExactly(topLevelNamespace); - Namespace nestedNamespace = Namespace.of("top_level", "second_level"); - sessionCatalog.createNamespace(sessionContext, nestedNamespace); - namespaces = sessionCatalog.listNamespaces(sessionContext, topLevelNamespace); - assertThat(namespaces).isNotNull().hasSize(1).containsExactly(nestedNamespace); - } - } - - @Test - public void testIcebergCreateNamespaceInExternalCatalog(TestInfo testInfo) throws IOException { - String catalogName = testInfo.getTestMethod().get().getName() + "External"; - createCatalog(catalogName, Catalog.TypeEnum.EXTERNAL, PRINCIPAL_ROLE_NAME); - try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName)) { - SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); - Namespace ns = Namespace.of("db1"); - sessionCatalog.createNamespace(sessionContext, ns); - List namespaces = sessionCatalog.listNamespaces(sessionContext); - assertThat(namespaces).isNotNull().hasSize(1).containsExactly(ns); - Map metadata = sessionCatalog.loadNamespaceMetadata(sessionContext, ns); - assertThat(metadata) - .isNotNull() - .isNotEmpty() - .containsEntry( - PolarisEntityConstants.ENTITY_BASE_LOCATION, "s3://my-bucket/path/to/data/db1"); - } - } - - @Test - public void testIcebergDropNamespaceInExternalCatalog(TestInfo testInfo) throws IOException { - String catalogName = testInfo.getTestMethod().get().getName() + "External"; - createCatalog(catalogName, Catalog.TypeEnum.EXTERNAL, PRINCIPAL_ROLE_NAME); - try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName)) { - SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); - Namespace ns = Namespace.of("db1"); - sessionCatalog.createNamespace(sessionContext, ns); - List namespaces = sessionCatalog.listNamespaces(sessionContext); - assertThat(namespaces).isNotNull().hasSize(1).containsExactly(ns); - sessionCatalog.dropNamespace(sessionContext, ns); - assertThatThrownBy(() -> sessionCatalog.loadNamespaceMetadata(sessionContext, ns)) - .isInstanceOf(NoSuchNamespaceException.class) - .hasMessage("Namespace does not exist: db1"); - } - } - - @Test - public void testIcebergCreateTablesInExternalCatalog(TestInfo testInfo) throws IOException { - String catalogName = testInfo.getTestMethod().get().getName() + "External"; - createCatalog(catalogName, Catalog.TypeEnum.EXTERNAL, PRINCIPAL_ROLE_NAME); - try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName)) { - SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); - Namespace ns = Namespace.of("db1"); - sessionCatalog.createNamespace(sessionContext, ns); - assertThatThrownBy( - () -> - sessionCatalog - .buildTable( - sessionContext, - TableIdentifier.of(ns, "the_table"), - new Schema( - List.of( - Types.NestedField.of( - 1, false, "theField", Types.StringType.get())))) - .withLocation("file:///tmp/tables") - .withSortOrder(SortOrder.unsorted()) - .withPartitionSpec(PartitionSpec.unpartitioned()) - .create()) - .isInstanceOf(BadRequestException.class) - .hasMessage("Malformed request: Cannot create table on external catalogs."); - } - } - - @Test - public void testIcebergCreateTablesWithWritePathBlocked(TestInfo testInfo) throws IOException { - String catalogName = testInfo.getTestMethod().get().getName() + "Internal"; - createCatalog(catalogName, Catalog.TypeEnum.INTERNAL, PRINCIPAL_ROLE_NAME); - try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName)) { - SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); - Namespace ns = Namespace.of("db1"); - sessionCatalog.createNamespace(sessionContext, ns); - try { - Assertions.assertThatThrownBy( - () -> - sessionCatalog - .buildTable( - sessionContext, - TableIdentifier.of(ns, "the_table"), - new Schema( - List.of( - Types.NestedField.of( - 1, false, "theField", Types.StringType.get())))) - .withSortOrder(SortOrder.unsorted()) - .withPartitionSpec(PartitionSpec.unpartitioned()) - .withProperties(Map.of("write.data.path", "s3://my-bucket/path/to/data")) - .create()) - .isInstanceOf(ForbiddenException.class) - .hasMessageContaining("Forbidden: Invalid locations"); - - Assertions.assertThatThrownBy( - () -> - sessionCatalog - .buildTable( - sessionContext, - TableIdentifier.of(ns, "the_table"), - new Schema( - List.of( - Types.NestedField.of( - 1, false, "theField", Types.StringType.get())))) - .withSortOrder(SortOrder.unsorted()) - .withPartitionSpec(PartitionSpec.unpartitioned()) - .withProperties( - Map.of("write.metadata.path", "s3://my-bucket/path/to/data")) - .create()) - .isInstanceOf(ForbiddenException.class) - .hasMessageContaining("Forbidden: Invalid locations"); - } catch (BadRequestException e) { - LOGGER.info("Received expected exception {}", e.getMessage()); - } - } - } - - @Test - public void testIcebergRegisterTableInExternalCatalog(TestInfo testInfo, @TempDir Path tempDir) - throws IOException { - String catalogName = testInfo.getTestMethod().get().getName() + "External"; - createCatalog( - catalogName, - Catalog.TypeEnum.EXTERNAL, - PRINCIPAL_ROLE_NAME, - FileStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.FILE) - .setAllowedLocations(List.of("file://" + tempDir.toFile().getAbsolutePath())) - .build(), - "file://" + tempDir.toFile().getAbsolutePath()); - try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName); - HadoopFileIO fileIo = new HadoopFileIO(new Configuration()); ) { - SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); - Namespace ns = Namespace.of("db1"); - sessionCatalog.createNamespace(sessionContext, ns); - TableIdentifier tableIdentifier = TableIdentifier.of(ns, "the_table"); - String location = - "file://" - + tempDir.toFile().getAbsolutePath() - + "/" - + testInfo.getTestMethod().get().getName(); - String metadataLocation = location + "/metadata/000001-494949494949494949.metadata.json"; - - TableMetadata tableMetadata = - TableMetadata.buildFromEmpty() - .setLocation(location) - .assignUUID() - .addPartitionSpec(PartitionSpec.unpartitioned()) - .addSortOrder(SortOrder.unsorted()) - .addSchema( - new Schema(Types.NestedField.of(1, false, "col1", Types.StringType.get())), 1) - .build(); - TableMetadataParser.write(tableMetadata, fileIo.newOutputFile(metadataLocation)); - - sessionCatalog.registerTable(sessionContext, tableIdentifier, metadataLocation); - Table table = sessionCatalog.loadTable(sessionContext, tableIdentifier); - assertThat(table) - .isNotNull() - .isInstanceOf(BaseTable.class) - .asInstanceOf(InstanceOfAssertFactories.type(BaseTable.class)) - .returns(tableMetadata.location(), BaseTable::location) - .returns(tableMetadata.uuid(), bt -> bt.uuid().toString()) - .returns(tableMetadata.schema().columns(), bt -> bt.schema().columns()); - } - } - - @Test - public void testIcebergUpdateTableInExternalCatalog(TestInfo testInfo, @TempDir Path tempDir) - throws IOException { - String catalogName = testInfo.getTestMethod().get().getName() + "External"; - createCatalog( - catalogName, - Catalog.TypeEnum.EXTERNAL, - PRINCIPAL_ROLE_NAME, - FileStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.FILE) - .setAllowedLocations(List.of("file://" + tempDir.toFile().getAbsolutePath())) - .build(), - "file://" + tempDir.toFile().getAbsolutePath()); - try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName); - HadoopFileIO fileIo = new HadoopFileIO(new Configuration()); ) { - SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); - Namespace ns = Namespace.of("db1"); - sessionCatalog.createNamespace(sessionContext, ns); - TableIdentifier tableIdentifier = TableIdentifier.of(ns, "the_table"); - String location = - "file://" - + tempDir.toFile().getAbsolutePath() - + "/" - + testInfo.getTestMethod().get().getName(); - String metadataLocation = location + "/metadata/000001-494949494949494949.metadata.json"; - - Types.NestedField col1 = Types.NestedField.of(1, false, "col1", Types.StringType.get()); - TableMetadata tableMetadata = - TableMetadata.buildFromEmpty() - .setLocation(location) - .assignUUID() - .addPartitionSpec(PartitionSpec.unpartitioned()) - .addSortOrder(SortOrder.unsorted()) - .addSchema(new Schema(col1), 1) - .build(); - TableMetadataParser.write(tableMetadata, fileIo.newOutputFile(metadataLocation)); - - sessionCatalog.registerTable(sessionContext, tableIdentifier, metadataLocation); - Table table = sessionCatalog.loadTable(sessionContext, tableIdentifier); - ((ResolvingFileIO) table.io()).setConf(new Configuration()); - assertThatThrownBy( - () -> - table - .newAppend() - .appendFile( - new TestHelpers.TestDataFile( - location + "/path/to/file.parquet", - new PartitionData(PartitionSpec.unpartitioned().partitionType()), - 10L)) - .commit()) - .isInstanceOf(BadRequestException.class) - .hasMessage("Malformed request: Cannot update table on external catalogs."); - } - } - - @Test - public void testIcebergDropTableInExternalCatalog(TestInfo testInfo, @TempDir Path tempDir) - throws IOException { - String catalogName = testInfo.getTestMethod().get().getName() + "External"; - createCatalog( - catalogName, - Catalog.TypeEnum.EXTERNAL, - PRINCIPAL_ROLE_NAME, - FileStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.FILE) - .setAllowedLocations(List.of("file://" + tempDir.toFile().getAbsolutePath())) - .build(), - "file://" + tempDir.toFile().getAbsolutePath()); - try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName); - HadoopFileIO fileIo = new HadoopFileIO(new Configuration()); ) { - SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); - Namespace ns = Namespace.of("db1"); - sessionCatalog.createNamespace(sessionContext, ns); - TableIdentifier tableIdentifier = TableIdentifier.of(ns, "the_table"); - String location = - "file://" - + tempDir.toFile().getAbsolutePath() - + "/" - + testInfo.getTestMethod().get().getName(); - String metadataLocation = location + "/metadata/000001-494949494949494949.metadata.json"; - - TableMetadata tableMetadata = - TableMetadata.buildFromEmpty() - .setLocation(location) - .assignUUID() - .addPartitionSpec(PartitionSpec.unpartitioned()) - .addSortOrder(SortOrder.unsorted()) - .addSchema( - new Schema(Types.NestedField.of(1, false, "col1", Types.StringType.get())), 1) - .build(); - TableMetadataParser.write(tableMetadata, fileIo.newOutputFile(metadataLocation)); - - sessionCatalog.registerTable(sessionContext, tableIdentifier, metadataLocation); - Table table = sessionCatalog.loadTable(sessionContext, tableIdentifier); - assertThat(table).isNotNull(); - sessionCatalog.dropTable(sessionContext, tableIdentifier); - assertThatThrownBy(() -> sessionCatalog.loadTable(sessionContext, tableIdentifier)) - .isInstanceOf(NoSuchTableException.class) - .hasMessage("Table does not exist: db1.the_table"); - } - } - - @Test - public void testWarehouseNotSpecified() throws IOException { - try (RESTSessionCatalog sessionCatalog = new RESTSessionCatalog()) { - String emptyEnvironmentVariable = "env:__NULL_ENV_VARIABLE__"; - assertThat(EnvironmentUtil.resolveAll(Map.of("", emptyEnvironmentVariable)).get("")).isNull(); - assertThatThrownBy( - () -> - sessionCatalog.initialize( - "polaris_catalog_test", - Map.of( - "uri", - "http://localhost:" + testHelper.localPort + "/api/catalog", - OAuth2Properties.CREDENTIAL, - testHelper.snowmanCredentials.clientId() - + ":" - + testHelper.snowmanCredentials.clientSecret(), - OAuth2Properties.SCOPE, - PRINCIPAL_ROLE_ALL, - "warehouse", - emptyEnvironmentVariable, - "header." + REALM_PROPERTY_KEY, - testHelper.realm))) - .isInstanceOf(BadRequestException.class) - .hasMessage("Malformed request: Please specify a warehouse"); - } - } - - @Test - public void testRequestHeaderTooLarge() { - Invocation.Builder request = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/management/v1/principal-roles", testHelper.localPort)) - .request("application/json"); - - // The default limit is 8KiB and each of these headers is at least 8 bytes, so 1500 definitely - // exceeds the limit - for (int i = 0; i < 1500; i++) { - request = request.header("header" + i, "" + i); - } - - try { - try (Response response = - request - .header("Authorization", "Bearer " + testHelper.adminToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .post(Entity.json(new PrincipalRole("r")))) { - assertThat(response) - .returns( - Response.Status.REQUEST_HEADER_FIELDS_TOO_LARGE.getStatusCode(), - Response::getStatus); - } - } catch (ProcessingException e) { - // In some runtime environments the request above will return a 431 but in others it'll result - // in a ProcessingException from the socket being closed. The test asserts that one of those - // things happens. - } - } - - @Test - public void testRequestBodyTooLarge() { - // The size is set to be higher than the limit in polaris-server-integrationtest.yml - Entity largeRequest = Entity.json(new PrincipalRole("r".repeat(1000001))); - - try (Response response = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/management/v1/principal-roles", testHelper.localPort)) - .request("application/json") - .header("Authorization", "Bearer " + testHelper.adminToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .post(largeRequest)) { - assertThat(response) - .returns(Response.Status.REQUEST_ENTITY_TOO_LARGE.getStatusCode(), Response::getStatus); - } - } - - @Test - public void testRefreshToken() throws IOException { - String path = - String.format("http://localhost:%d/api/catalog/v1/oauth/tokens", testHelper.localPort); - try (RESTClient client = - HTTPClient.builder(Map.of()) - .withHeader(REALM_PROPERTY_KEY, testHelper.realm) - .uri(path) - .build()) { - String credentialString = - testHelper.snowmanCredentials.clientId() - + ":" - + testHelper.snowmanCredentials.clientSecret(); - AuthConfig configMock = mock(AuthConfig.class); - when(configMock.credential()).thenReturn(credentialString); - when(configMock.scope()).thenReturn(PRINCIPAL_ROLE_ALL); - when(configMock.expiresAtMillis()).thenReturn(0L); - when(configMock.oauth2ServerUri()).thenReturn(path); - - var parentSession = new OAuth2Util.AuthSession(Map.of(), configMock); - var session = - OAuth2Util.AuthSession.fromAccessToken( - client, null, testHelper.adminToken, 0L, parentSession); - - OAuth2Util.AuthSession sessionSpy = spy(session); - when(sessionSpy.expiresAtMillis()).thenReturn(0L); - assertThat(sessionSpy.expiresAtMillis()).isEqualTo(0L); - assertThat(sessionSpy.token()).isEqualTo(testHelper.adminToken); - - sessionSpy.refresh(client); - assertThat(sessionSpy.credential()).isNotNull(); - assertThat(sessionSpy.credential()).isNotEqualTo(testHelper.adminToken); - } - } -} diff --git a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/admin/PolarisAdminServiceAuthzTest.java b/polaris-service-quarkus/src/test/java/org/apache/polaris/service/admin/PolarisAdminServiceAuthzTest.java deleted file mode 100644 index 556ca7468..000000000 --- a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/admin/PolarisAdminServiceAuthzTest.java +++ /dev/null @@ -1,1842 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.admin; - -import io.quarkus.test.junit.QuarkusTest; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Function; -import org.apache.polaris.core.admin.model.UpdateCatalogRequest; -import org.apache.polaris.core.admin.model.UpdateCatalogRoleRequest; -import org.apache.polaris.core.admin.model.UpdatePrincipalRequest; -import org.apache.polaris.core.admin.model.UpdatePrincipalRoleRequest; -import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; -import org.apache.polaris.core.entity.CatalogEntity; -import org.apache.polaris.core.entity.CatalogRoleEntity; -import org.apache.polaris.core.entity.PolarisPrivilege; -import org.apache.polaris.core.entity.PrincipalEntity; -import org.apache.polaris.core.entity.PrincipalRoleEntity; -import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.Test; - -@QuarkusTest -public class PolarisAdminServiceAuthzTest extends PolarisAuthzTestBase { - private PolarisAdminService newTestAdminService() { - return newTestAdminService(Set.of()); - } - - private PolarisAdminService newTestAdminService(Set activatedPrincipalRoles) { - final AuthenticatedPolarisPrincipal authenticatedPrincipal = - new AuthenticatedPolarisPrincipal(principalEntity, activatedPrincipalRoles); - return new PolarisAdminService( - callContext, entityManager, metaStoreManager, authenticatedPrincipal, polarisAuthorizer); - } - - private void doTestSufficientPrivileges( - List sufficientPrivileges, - Runnable action, - Runnable cleanupAction, - Function grantAction, - Function revokeAction) { - doTestSufficientPrivilegeSets( - sufficientPrivileges.stream().map(priv -> Set.of(priv)).toList(), - action, - cleanupAction, - PRINCIPAL_NAME, - grantAction, - revokeAction); - } - - private void doTestInsufficientPrivileges( - List insufficientPrivileges, - Runnable action, - Function grantAction, - Function revokeAction) { - doTestInsufficientPrivileges( - insufficientPrivileges, PRINCIPAL_NAME, action, grantAction, revokeAction); - } - - @Test - public void testListCatalogsSufficientPrivileges() { - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.SERVICE_MANAGE_ACCESS, - PolarisPrivilege.CATALOG_LIST, - PolarisPrivilege.CATALOG_READ_PROPERTIES, - PolarisPrivilege.CATALOG_WRITE_PROPERTIES, - PolarisPrivilege.CATALOG_CREATE, - PolarisPrivilege.CATALOG_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT), - () -> newTestAdminService().listCatalogs(), - null, // cleanupAction - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testListCatalogsInsufficientPrivileges() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_ROLE_FULL_METADATA, - PolarisPrivilege.CATALOG_DROP, - PolarisPrivilege.CATALOG_ROLE_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_ACCESS), - () -> newTestAdminService().listCatalogs(), - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testCreateCatalogSufficientPrivileges() { - // Cleanup with PRINCIPAL_ROLE2 - Assertions.assertThat( - adminService.grantPrivilegeOnRootContainerToPrincipalRole( - PRINCIPAL_ROLE2, PolarisPrivilege.CATALOG_DROP)) - .isTrue(); - final CatalogEntity newCatalog = new CatalogEntity.Builder().setName("new_catalog").build(); - - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.SERVICE_MANAGE_ACCESS, - PolarisPrivilege.CATALOG_CREATE, - PolarisPrivilege.CATALOG_FULL_METADATA), - () -> newTestAdminService(Set.of(PRINCIPAL_ROLE1)).createCatalog(newCatalog), - () -> newTestAdminService(Set.of(PRINCIPAL_ROLE2)).deleteCatalog(newCatalog.getName()), - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testCreateCatalogInsufficientPrivileges() { - final CatalogEntity newCatalog = new CatalogEntity.Builder().setName("new_catalog").build(); - - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_ROLE_FULL_METADATA, - PolarisPrivilege.CATALOG_ROLE_FULL_METADATA, - PolarisPrivilege.CATALOG_LIST, - PolarisPrivilege.CATALOG_DROP, - PolarisPrivilege.CATALOG_READ_PROPERTIES, - PolarisPrivilege.CATALOG_WRITE_PROPERTIES, - PolarisPrivilege.CATALOG_MANAGE_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT, - PolarisPrivilege.CATALOG_MANAGE_ACCESS), - () -> newTestAdminService(Set.of(PRINCIPAL_ROLE1)).createCatalog(newCatalog), - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testGetCatalogSufficientPrivileges() { - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.SERVICE_MANAGE_ACCESS, - PolarisPrivilege.CATALOG_READ_PROPERTIES, - PolarisPrivilege.CATALOG_WRITE_PROPERTIES, - PolarisPrivilege.CATALOG_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT), - () -> newTestAdminService().getCatalog(CATALOG_NAME), - null, // cleanupAction - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testGetCatalogInsufficientPrivileges() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_ROLE_FULL_METADATA, - PolarisPrivilege.CATALOG_LIST, - PolarisPrivilege.CATALOG_CREATE, - PolarisPrivilege.CATALOG_DROP, - PolarisPrivilege.CATALOG_ROLE_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_ACCESS), - () -> newTestAdminService().getCatalog(CATALOG_NAME), - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testUpdateCatalogSufficientPrivileges() { - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.SERVICE_MANAGE_ACCESS, - PolarisPrivilege.CATALOG_WRITE_PROPERTIES, - PolarisPrivilege.CATALOG_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT), - () -> { - // Use the test-permission admin service instead of the root adminService to also - // perform the initial GET to illustrate that the actual user workflow for update - // *must* also encompass GET privileges to be able to set entityVersion properly. - UpdateCatalogRequest updateRequest = - UpdateCatalogRequest.builder() - .setCurrentEntityVersion( - newTestAdminService().getCatalog(CATALOG_NAME).getEntityVersion()) - .setProperties(Map.of("foo", Long.toString(System.currentTimeMillis()))) - .build(); - newTestAdminService().updateCatalog(CATALOG_NAME, updateRequest); - }, - null, // cleanupAction - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testUpdateCatalogInsufficientPrivileges() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_ROLE_FULL_METADATA, - PolarisPrivilege.CATALOG_READ_PROPERTIES, - PolarisPrivilege.CATALOG_LIST, - PolarisPrivilege.CATALOG_CREATE, - PolarisPrivilege.CATALOG_DROP, - PolarisPrivilege.CATALOG_ROLE_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_ACCESS), - () -> { - UpdateCatalogRequest updateRequest = - UpdateCatalogRequest.builder() - .setCurrentEntityVersion( - newTestAdminService().getCatalog(CATALOG_NAME).getEntityVersion()) - .setProperties(Map.of("foo", Long.toString(System.currentTimeMillis()))) - .build(); - newTestAdminService().updateCatalog(CATALOG_NAME, updateRequest); - }, - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testDeleteCatalogSufficientPrivileges() { - // Cleanup with PRINCIPAL_ROLE2 - Assertions.assertThat( - adminService.grantPrivilegeOnRootContainerToPrincipalRole( - PRINCIPAL_ROLE2, PolarisPrivilege.CATALOG_CREATE)) - .isTrue(); - final CatalogEntity newCatalog = new CatalogEntity.Builder().setName("new_catalog").build(); - adminService.createCatalog(newCatalog); - - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.SERVICE_MANAGE_ACCESS, - PolarisPrivilege.CATALOG_DROP, - PolarisPrivilege.CATALOG_FULL_METADATA), - () -> newTestAdminService(Set.of(PRINCIPAL_ROLE1)).deleteCatalog(newCatalog.getName()), - () -> newTestAdminService(Set.of(PRINCIPAL_ROLE2)).createCatalog(newCatalog), - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testDeleteCatalogInsufficientPrivileges() { - final CatalogEntity newCatalog = new CatalogEntity.Builder().setName("new_catalog").build(); - adminService.createCatalog(newCatalog); - - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_ROLE_FULL_METADATA, - PolarisPrivilege.CATALOG_ROLE_FULL_METADATA, - PolarisPrivilege.CATALOG_CREATE, - PolarisPrivilege.CATALOG_LIST, - PolarisPrivilege.CATALOG_READ_PROPERTIES, - PolarisPrivilege.CATALOG_WRITE_PROPERTIES, - PolarisPrivilege.CATALOG_MANAGE_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT, - PolarisPrivilege.CATALOG_MANAGE_ACCESS), - () -> newTestAdminService(Set.of(PRINCIPAL_ROLE1)).deleteCatalog(newCatalog.getName()), - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testListPrincipalsSufficientPrivileges() { - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.SERVICE_MANAGE_ACCESS, - PolarisPrivilege.PRINCIPAL_LIST, - PolarisPrivilege.PRINCIPAL_READ_PROPERTIES, - PolarisPrivilege.PRINCIPAL_WRITE_PROPERTIES, - PolarisPrivilege.PRINCIPAL_CREATE, - PolarisPrivilege.PRINCIPAL_FULL_METADATA), - () -> newTestAdminService().listPrincipals(), - null, // cleanupAction - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testListPrincipalsInsufficientPrivileges() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.CATALOG_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_ROLE_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_DROP, - PolarisPrivilege.CATALOG_ROLE_FULL_METADATA), - () -> newTestAdminService().listPrincipals(), - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testCreatePrincipalSufficientPrivileges() { - // Cleanup with PRINCIPAL_ROLE2 - Assertions.assertThat( - adminService.grantPrivilegeOnRootContainerToPrincipalRole( - PRINCIPAL_ROLE2, PolarisPrivilege.PRINCIPAL_DROP)) - .isTrue(); - final PrincipalEntity newPrincipal = - new PrincipalEntity.Builder().setName("new_principal").build(); - - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.SERVICE_MANAGE_ACCESS, - PolarisPrivilege.PRINCIPAL_CREATE, - PolarisPrivilege.PRINCIPAL_FULL_METADATA), - () -> newTestAdminService(Set.of(PRINCIPAL_ROLE1)).createPrincipal(newPrincipal), - () -> newTestAdminService(Set.of(PRINCIPAL_ROLE2)).deletePrincipal(newPrincipal.getName()), - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testCreatePrincipalInsufficientPrivileges() { - final PrincipalEntity newPrincipal = - new PrincipalEntity.Builder().setName("new_principal").build(); - - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.CATALOG_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_ROLE_FULL_METADATA, - PolarisPrivilege.CATALOG_ROLE_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_LIST, - PolarisPrivilege.PRINCIPAL_DROP, - PolarisPrivilege.PRINCIPAL_READ_PROPERTIES, - PolarisPrivilege.PRINCIPAL_WRITE_PROPERTIES), - () -> newTestAdminService(Set.of(PRINCIPAL_ROLE1)).createPrincipal(newPrincipal), - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testGetPrincipalSufficientPrivileges() { - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.SERVICE_MANAGE_ACCESS, - PolarisPrivilege.PRINCIPAL_READ_PROPERTIES, - PolarisPrivilege.PRINCIPAL_WRITE_PROPERTIES, - PolarisPrivilege.PRINCIPAL_FULL_METADATA), - () -> newTestAdminService().getPrincipal(PRINCIPAL_NAME), - null, // cleanupAction - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testGetPrincipalInsufficientPrivileges() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.CATALOG_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_ROLE_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_LIST, - PolarisPrivilege.PRINCIPAL_CREATE, - PolarisPrivilege.PRINCIPAL_DROP, - PolarisPrivilege.CATALOG_ROLE_FULL_METADATA), - () -> newTestAdminService().getPrincipal(PRINCIPAL_NAME), - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testUpdatePrincipalSufficientPrivileges() { - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.SERVICE_MANAGE_ACCESS, - PolarisPrivilege.PRINCIPAL_WRITE_PROPERTIES, - PolarisPrivilege.PRINCIPAL_FULL_METADATA), - () -> { - // Use the test-permission admin service instead of the root adminService to also - // perform the initial GET to illustrate that the actual user workflow for update - // *must* also encompass GET privileges to be able to set entityVersion properly. - UpdatePrincipalRequest updateRequest = - UpdatePrincipalRequest.builder() - .setCurrentEntityVersion( - newTestAdminService().getPrincipal(PRINCIPAL_NAME).getEntityVersion()) - .setProperties(Map.of("foo", Long.toString(System.currentTimeMillis()))) - .build(); - newTestAdminService().updatePrincipal(PRINCIPAL_NAME, updateRequest); - }, - null, // cleanupAction - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testUpdatePrincipalInsufficientPrivileges() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.CATALOG_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_ROLE_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_READ_PROPERTIES, - PolarisPrivilege.PRINCIPAL_LIST, - PolarisPrivilege.PRINCIPAL_CREATE, - PolarisPrivilege.PRINCIPAL_DROP, - PolarisPrivilege.CATALOG_ROLE_FULL_METADATA), - () -> { - UpdatePrincipalRequest updateRequest = - UpdatePrincipalRequest.builder() - .setCurrentEntityVersion( - newTestAdminService().getPrincipal(PRINCIPAL_NAME).getEntityVersion()) - .setProperties(Map.of("foo", Long.toString(System.currentTimeMillis()))) - .build(); - newTestAdminService().updatePrincipal(PRINCIPAL_NAME, updateRequest); - }, - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testDeletePrincipalSufficientPrivileges() { - // Cleanup with PRINCIPAL_ROLE2 - Assertions.assertThat( - adminService.grantPrivilegeOnRootContainerToPrincipalRole( - PRINCIPAL_ROLE2, PolarisPrivilege.PRINCIPAL_CREATE)) - .isTrue(); - final PrincipalEntity newPrincipal = - new PrincipalEntity.Builder().setName("new_principal").build(); - adminService.createPrincipal(newPrincipal); - - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.SERVICE_MANAGE_ACCESS, - PolarisPrivilege.PRINCIPAL_DROP, - PolarisPrivilege.PRINCIPAL_FULL_METADATA), - () -> newTestAdminService(Set.of(PRINCIPAL_ROLE1)).deletePrincipal(newPrincipal.getName()), - () -> newTestAdminService(Set.of(PRINCIPAL_ROLE2)).createPrincipal(newPrincipal), - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testDeletePrincipalInsufficientPrivileges() { - final PrincipalEntity newPrincipal = - new PrincipalEntity.Builder().setName("new_principal").build(); - adminService.createPrincipal(newPrincipal); - - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.CATALOG_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_ROLE_FULL_METADATA, - PolarisPrivilege.CATALOG_ROLE_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_CREATE, - PolarisPrivilege.PRINCIPAL_LIST, - PolarisPrivilege.PRINCIPAL_READ_PROPERTIES, - PolarisPrivilege.PRINCIPAL_WRITE_PROPERTIES), - () -> newTestAdminService(Set.of(PRINCIPAL_ROLE1)).deletePrincipal(newPrincipal.getName()), - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testListPrincipalRolesSufficientPrivileges() { - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.SERVICE_MANAGE_ACCESS, - PolarisPrivilege.PRINCIPAL_ROLE_LIST, - PolarisPrivilege.PRINCIPAL_ROLE_READ_PROPERTIES, - PolarisPrivilege.PRINCIPAL_ROLE_WRITE_PROPERTIES, - PolarisPrivilege.PRINCIPAL_ROLE_CREATE, - PolarisPrivilege.PRINCIPAL_ROLE_FULL_METADATA), - () -> newTestAdminService().listPrincipalRoles(), - null, // cleanupAction - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testListPrincipalRolesInsufficientPrivileges() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.CATALOG_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_ROLE_DROP, - PolarisPrivilege.CATALOG_ROLE_FULL_METADATA), - () -> newTestAdminService().listPrincipalRoles(), - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testCreatePrincipalRoleSufficientPrivileges() { - // Cleanup with PRINCIPAL_ROLE2 - Assertions.assertThat( - adminService.grantPrivilegeOnRootContainerToPrincipalRole( - PRINCIPAL_ROLE2, PolarisPrivilege.PRINCIPAL_ROLE_DROP)) - .isTrue(); - final PrincipalRoleEntity newPrincipalRole = - new PrincipalRoleEntity.Builder().setName("new_principal_role").build(); - - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.SERVICE_MANAGE_ACCESS, - PolarisPrivilege.PRINCIPAL_ROLE_CREATE, - PolarisPrivilege.PRINCIPAL_ROLE_FULL_METADATA), - () -> newTestAdminService(Set.of(PRINCIPAL_ROLE1)).createPrincipalRole(newPrincipalRole), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE2)) - .deletePrincipalRole(newPrincipalRole.getName()), - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testCreatePrincipalRoleInsufficientPrivileges() { - final PrincipalRoleEntity newPrincipalRole = - new PrincipalRoleEntity.Builder().setName("new_principal_role").build(); - - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.CATALOG_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_FULL_METADATA, - PolarisPrivilege.CATALOG_ROLE_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_ROLE_LIST, - PolarisPrivilege.PRINCIPAL_ROLE_DROP, - PolarisPrivilege.PRINCIPAL_ROLE_READ_PROPERTIES, - PolarisPrivilege.PRINCIPAL_ROLE_WRITE_PROPERTIES), - () -> newTestAdminService(Set.of(PRINCIPAL_ROLE1)).createPrincipalRole(newPrincipalRole), - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testGetPrincipalRoleSufficientPrivileges() { - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.SERVICE_MANAGE_ACCESS, - PolarisPrivilege.PRINCIPAL_ROLE_READ_PROPERTIES, - PolarisPrivilege.PRINCIPAL_ROLE_WRITE_PROPERTIES, - PolarisPrivilege.PRINCIPAL_ROLE_FULL_METADATA), - () -> newTestAdminService().getPrincipalRole(PRINCIPAL_ROLE2), - null, // cleanupAction - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testGetPrincipalRoleInsufficientPrivileges() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.CATALOG_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_ROLE_LIST, - PolarisPrivilege.PRINCIPAL_ROLE_CREATE, - PolarisPrivilege.PRINCIPAL_ROLE_DROP, - PolarisPrivilege.CATALOG_ROLE_FULL_METADATA), - () -> newTestAdminService().getPrincipalRole(PRINCIPAL_ROLE2), - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testUpdatePrincipalRoleSufficientPrivileges() { - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.SERVICE_MANAGE_ACCESS, - PolarisPrivilege.PRINCIPAL_ROLE_WRITE_PROPERTIES, - PolarisPrivilege.PRINCIPAL_ROLE_FULL_METADATA), - () -> { - // Use the test-permission admin service instead of the root adminService to also - // perform the initial GET to illustrate that the actual user workflow for update - // *must* also encompass GET privileges to be able to set entityVersion properly. - UpdatePrincipalRoleRequest updateRequest = - UpdatePrincipalRoleRequest.builder() - .setCurrentEntityVersion( - newTestAdminService().getPrincipalRole(PRINCIPAL_ROLE2).getEntityVersion()) - .setProperties(Map.of("foo", Long.toString(System.currentTimeMillis()))) - .build(); - newTestAdminService().updatePrincipalRole(PRINCIPAL_ROLE2, updateRequest); - }, - null, // cleanupAction - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testUpdatePrincipalRoleInsufficientPrivileges() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.CATALOG_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_ROLE_READ_PROPERTIES, - PolarisPrivilege.PRINCIPAL_ROLE_LIST, - PolarisPrivilege.PRINCIPAL_ROLE_CREATE, - PolarisPrivilege.PRINCIPAL_ROLE_DROP, - PolarisPrivilege.CATALOG_ROLE_FULL_METADATA), - () -> { - UpdatePrincipalRoleRequest updateRequest = - UpdatePrincipalRoleRequest.builder() - .setCurrentEntityVersion( - newTestAdminService().getPrincipalRole(PRINCIPAL_ROLE2).getEntityVersion()) - .setProperties(Map.of("foo", Long.toString(System.currentTimeMillis()))) - .build(); - newTestAdminService().updatePrincipalRole(PRINCIPAL_ROLE2, updateRequest); - }, - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testDeletePrincipalRoleSufficientPrivileges() { - // Cleanup with PRINCIPAL_ROLE2 - Assertions.assertThat( - adminService.grantPrivilegeOnRootContainerToPrincipalRole( - PRINCIPAL_ROLE2, PolarisPrivilege.PRINCIPAL_ROLE_CREATE)) - .isTrue(); - final PrincipalRoleEntity newPrincipalRole = - new PrincipalRoleEntity.Builder().setName("new_principal_role").build(); - adminService.createPrincipalRole(newPrincipalRole); - - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.SERVICE_MANAGE_ACCESS, - PolarisPrivilege.PRINCIPAL_ROLE_DROP, - PolarisPrivilege.PRINCIPAL_ROLE_FULL_METADATA), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE1)) - .deletePrincipalRole(newPrincipalRole.getName()), - () -> newTestAdminService(Set.of(PRINCIPAL_ROLE2)).createPrincipalRole(newPrincipalRole), - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testDeletePrincipalRoleInsufficientPrivileges() { - final PrincipalRoleEntity newPrincipalRole = - new PrincipalRoleEntity.Builder().setName("new_principal_role").build(); - adminService.createPrincipalRole(newPrincipalRole); - - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.CATALOG_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_FULL_METADATA, - PolarisPrivilege.CATALOG_ROLE_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_ROLE_CREATE, - PolarisPrivilege.PRINCIPAL_ROLE_LIST, - PolarisPrivilege.PRINCIPAL_ROLE_READ_PROPERTIES, - PolarisPrivilege.PRINCIPAL_ROLE_WRITE_PROPERTIES), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE1)) - .deletePrincipalRole(newPrincipalRole.getName()), - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testListCatalogRolesSufficientPrivileges() { - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.CATALOG_MANAGE_ACCESS, - PolarisPrivilege.CATALOG_ROLE_LIST, - PolarisPrivilege.CATALOG_ROLE_READ_PROPERTIES, - PolarisPrivilege.CATALOG_ROLE_WRITE_PROPERTIES, - PolarisPrivilege.CATALOG_ROLE_CREATE, - PolarisPrivilege.CATALOG_ROLE_FULL_METADATA), - () -> newTestAdminService().listCatalogRoles(CATALOG_NAME), - null, // cleanupAction - (privilege) -> - adminService.grantPrivilegeOnCatalogToRole(CATALOG_NAME, CATALOG_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnCatalogFromRole(CATALOG_NAME, CATALOG_ROLE1, privilege)); - } - - @Test - public void testListCatalogRolesInsufficientPrivileges() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.SERVICE_MANAGE_ACCESS, - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.CATALOG_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_FULL_METADATA, - PolarisPrivilege.CATALOG_ROLE_DROP, - PolarisPrivilege.PRINCIPAL_ROLE_FULL_METADATA), - () -> newTestAdminService().listCatalogRoles(CATALOG_NAME), - (privilege) -> - adminService.grantPrivilegeOnCatalogToRole(CATALOG_NAME, CATALOG_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnCatalogFromRole(CATALOG_NAME, CATALOG_ROLE1, privilege)); - } - - @Test - public void testCreateCatalogRoleSufficientPrivileges() { - // Cleanup with CATALOG_ROLE2 - Assertions.assertThat( - adminService.grantPrivilegeOnCatalogToRole( - CATALOG_NAME, CATALOG_ROLE2, PolarisPrivilege.CATALOG_ROLE_DROP)) - .isTrue(); - final CatalogRoleEntity newCatalogRole = - new CatalogRoleEntity.Builder().setName("new_catalog_role").build(); - - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.CATALOG_MANAGE_ACCESS, - PolarisPrivilege.CATALOG_ROLE_CREATE, - PolarisPrivilege.CATALOG_ROLE_FULL_METADATA), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE1)) - .createCatalogRole(CATALOG_NAME, newCatalogRole), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE2)) - .deleteCatalogRole(CATALOG_NAME, newCatalogRole.getName()), - (privilege) -> - adminService.grantPrivilegeOnCatalogToRole(CATALOG_NAME, CATALOG_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnCatalogFromRole(CATALOG_NAME, CATALOG_ROLE1, privilege)); - } - - @Test - public void testCreateCatalogRoleInsufficientPrivileges() { - final CatalogRoleEntity newCatalogRole = - new CatalogRoleEntity.Builder().setName("new_catalog_role").build(); - - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.SERVICE_MANAGE_ACCESS, - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.CATALOG_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_ROLE_FULL_METADATA, - PolarisPrivilege.CATALOG_ROLE_LIST, - PolarisPrivilege.CATALOG_ROLE_DROP, - PolarisPrivilege.CATALOG_ROLE_READ_PROPERTIES, - PolarisPrivilege.CATALOG_ROLE_WRITE_PROPERTIES), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE1)) - .createCatalogRole(CATALOG_NAME, newCatalogRole), - (privilege) -> - adminService.grantPrivilegeOnCatalogToRole(CATALOG_NAME, CATALOG_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnCatalogFromRole(CATALOG_NAME, CATALOG_ROLE1, privilege)); - } - - @Test - public void testGetCatalogRoleSufficientPrivileges() { - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.CATALOG_MANAGE_ACCESS, - PolarisPrivilege.CATALOG_ROLE_READ_PROPERTIES, - PolarisPrivilege.CATALOG_ROLE_WRITE_PROPERTIES, - PolarisPrivilege.CATALOG_ROLE_FULL_METADATA), - () -> newTestAdminService().getCatalogRole(CATALOG_NAME, CATALOG_ROLE2), - null, // cleanupAction - (privilege) -> - adminService.grantPrivilegeOnCatalogToRole(CATALOG_NAME, CATALOG_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnCatalogFromRole(CATALOG_NAME, CATALOG_ROLE1, privilege)); - } - - @Test - public void testGetCatalogRoleInsufficientPrivileges() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.SERVICE_MANAGE_ACCESS, - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.CATALOG_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_FULL_METADATA, - PolarisPrivilege.CATALOG_ROLE_LIST, - PolarisPrivilege.CATALOG_ROLE_CREATE, - PolarisPrivilege.CATALOG_ROLE_DROP, - PolarisPrivilege.PRINCIPAL_ROLE_FULL_METADATA), - () -> newTestAdminService().getCatalogRole(CATALOG_NAME, CATALOG_ROLE2), - (privilege) -> - adminService.grantPrivilegeOnCatalogToRole(CATALOG_NAME, CATALOG_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnCatalogFromRole(CATALOG_NAME, CATALOG_ROLE1, privilege)); - } - - @Test - public void testUpdateCatalogRoleSufficientPrivileges() { - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.CATALOG_MANAGE_ACCESS, - PolarisPrivilege.CATALOG_ROLE_WRITE_PROPERTIES, - PolarisPrivilege.CATALOG_ROLE_FULL_METADATA), - () -> { - // Use the test-permission admin service instead of the root adminService to also - // perform the initial GET to illustrate that the actual user workflow for update - // *must* also encompass GET privileges to be able to set entityVersion properly. - UpdateCatalogRoleRequest updateRequest = - UpdateCatalogRoleRequest.builder() - .setCurrentEntityVersion( - newTestAdminService() - .getCatalogRole(CATALOG_NAME, CATALOG_ROLE2) - .getEntityVersion()) - .setProperties(Map.of("foo", Long.toString(System.currentTimeMillis()))) - .build(); - newTestAdminService().updateCatalogRole(CATALOG_NAME, CATALOG_ROLE2, updateRequest); - }, - null, // cleanupAction - (privilege) -> - adminService.grantPrivilegeOnCatalogToRole(CATALOG_NAME, CATALOG_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnCatalogFromRole(CATALOG_NAME, CATALOG_ROLE1, privilege)); - } - - @Test - public void testUpdateCatalogRoleInsufficientPrivileges() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.SERVICE_MANAGE_ACCESS, - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.CATALOG_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_FULL_METADATA, - PolarisPrivilege.CATALOG_ROLE_READ_PROPERTIES, - PolarisPrivilege.CATALOG_ROLE_LIST, - PolarisPrivilege.CATALOG_ROLE_CREATE, - PolarisPrivilege.CATALOG_ROLE_DROP, - PolarisPrivilege.PRINCIPAL_ROLE_FULL_METADATA), - () -> { - UpdateCatalogRoleRequest updateRequest = - UpdateCatalogRoleRequest.builder() - .setCurrentEntityVersion( - newTestAdminService() - .getCatalogRole(CATALOG_NAME, CATALOG_ROLE2) - .getEntityVersion()) - .setProperties(Map.of("foo", Long.toString(System.currentTimeMillis()))) - .build(); - newTestAdminService().updateCatalogRole(CATALOG_NAME, CATALOG_ROLE2, updateRequest); - }, - (privilege) -> - adminService.grantPrivilegeOnCatalogToRole(CATALOG_NAME, CATALOG_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnCatalogFromRole(CATALOG_NAME, CATALOG_ROLE1, privilege)); - } - - @Test - public void testDeleteCatalogRoleSufficientPrivileges() { - // Cleanup with CATALOG_ROLE2 - Assertions.assertThat( - adminService.grantPrivilegeOnCatalogToRole( - CATALOG_NAME, CATALOG_ROLE2, PolarisPrivilege.CATALOG_ROLE_CREATE)) - .isTrue(); - final CatalogRoleEntity newCatalogRole = - new CatalogRoleEntity.Builder().setName("new_catalog_role").build(); - adminService.createCatalogRole(CATALOG_NAME, newCatalogRole); - - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.CATALOG_MANAGE_ACCESS, - PolarisPrivilege.CATALOG_ROLE_DROP, - PolarisPrivilege.CATALOG_ROLE_FULL_METADATA), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE1)) - .deleteCatalogRole(CATALOG_NAME, newCatalogRole.getName()), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE2)) - .createCatalogRole(CATALOG_NAME, newCatalogRole), - (privilege) -> - adminService.grantPrivilegeOnCatalogToRole(CATALOG_NAME, CATALOG_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnCatalogFromRole(CATALOG_NAME, CATALOG_ROLE1, privilege)); - } - - @Test - public void testDeleteCatalogRoleInsufficientPrivileges() { - final CatalogRoleEntity newCatalogRole = - new CatalogRoleEntity.Builder().setName("new_catalog_role").build(); - adminService.createCatalogRole(CATALOG_NAME, newCatalogRole); - - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.SERVICE_MANAGE_ACCESS, - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.CATALOG_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_ROLE_FULL_METADATA, - PolarisPrivilege.CATALOG_ROLE_CREATE, - PolarisPrivilege.CATALOG_ROLE_LIST, - PolarisPrivilege.CATALOG_ROLE_READ_PROPERTIES, - PolarisPrivilege.CATALOG_ROLE_WRITE_PROPERTIES), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE1)) - .deleteCatalogRole(CATALOG_NAME, newCatalogRole.getName()), - (privilege) -> - adminService.grantPrivilegeOnCatalogToRole(CATALOG_NAME, CATALOG_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnCatalogFromRole(CATALOG_NAME, CATALOG_ROLE1, privilege)); - } - - @Test - public void testAssignPrincipalRoleSufficientPrivileges() { - adminService.createPrincipal(new PrincipalEntity.Builder().setName("newprincipal").build()); - - // Assign only requires privileges on the securable. - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.SERVICE_MANAGE_ACCESS, - PolarisPrivilege.PRINCIPAL_ROLE_MANAGE_GRANTS_ON_SECURABLE), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE1)) - .assignPrincipalRole("newprincipal", PRINCIPAL_ROLE2), - null, // cleanupAction - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testAssignPrincipalRoleInsufficientPrivileges() { - adminService.createPrincipal(new PrincipalEntity.Builder().setName("newprincipal").build()); - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.PRINCIPAL_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.PRINCIPAL_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.PRINCIPAL_LIST_GRANTS, - PolarisPrivilege.PRINCIPAL_ROLE_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.PRINCIPAL_ROLE_LIST_GRANTS, - PolarisPrivilege.CATALOG_ROLE_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.CATALOG_ROLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.CATALOG_ROLE_LIST_GRANTS, - PolarisPrivilege.CATALOG_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.CATALOG_LIST_GRANTS, - PolarisPrivilege.NAMESPACE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.NAMESPACE_LIST_GRANTS, - PolarisPrivilege.TABLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.TABLE_LIST_GRANTS, - PolarisPrivilege.VIEW_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.VIEW_LIST_GRANTS, - PolarisPrivilege.PRINCIPAL_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_ROLE_FULL_METADATA, - PolarisPrivilege.CATALOG_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT, - PolarisPrivilege.CATALOG_MANAGE_ACCESS), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE1)) - .assignPrincipalRole("newprincipal", PRINCIPAL_ROLE2), - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testRevokePrincipalRoleSufficientPrivileges() { - adminService.createPrincipal(new PrincipalEntity.Builder().setName("newprincipal").build()); - - // Revoke requires privileges both on the "securable" (PrincipalRole) as well as the "grantee" - // (Principal). - doTestSufficientPrivilegeSets( - List.of( - Set.of(PolarisPrivilege.SERVICE_MANAGE_ACCESS), - Set.of( - PolarisPrivilege.PRINCIPAL_ROLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.PRINCIPAL_MANAGE_GRANTS_FOR_GRANTEE)), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE1)) - .revokePrincipalRole("newprincipal", PRINCIPAL_ROLE2), - null, // cleanupAction - PRINCIPAL_NAME, - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testRevokePrincipalRoleInsufficientPrivileges() { - adminService.createPrincipal(new PrincipalEntity.Builder().setName("newprincipal").build()); - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.PRINCIPAL_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.PRINCIPAL_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.PRINCIPAL_LIST_GRANTS, - PolarisPrivilege.PRINCIPAL_ROLE_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.PRINCIPAL_ROLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.PRINCIPAL_ROLE_LIST_GRANTS, - PolarisPrivilege.CATALOG_ROLE_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.CATALOG_ROLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.CATALOG_ROLE_LIST_GRANTS, - PolarisPrivilege.CATALOG_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.CATALOG_LIST_GRANTS, - PolarisPrivilege.NAMESPACE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.NAMESPACE_LIST_GRANTS, - PolarisPrivilege.TABLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.TABLE_LIST_GRANTS, - PolarisPrivilege.VIEW_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.VIEW_LIST_GRANTS, - PolarisPrivilege.PRINCIPAL_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_ROLE_FULL_METADATA, - PolarisPrivilege.CATALOG_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT, - PolarisPrivilege.CATALOG_MANAGE_ACCESS), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE1)) - .revokePrincipalRole("newprincipal", PRINCIPAL_ROLE2), - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testAssignCatalogRoleToPrincipalRoleSufficientPrivileges() { - // Assign only requires privileges on the securable. - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.CATALOG_MANAGE_ACCESS, - PolarisPrivilege.CATALOG_ROLE_MANAGE_GRANTS_ON_SECURABLE), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE1)) - .assignCatalogRoleToPrincipalRole(PRINCIPAL_ROLE2, CATALOG_NAME, CATALOG_ROLE1), - null, // cleanupAction - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testAssignCatalogRoleToPrincipalRoleInsufficientPrivileges() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.PRINCIPAL_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.PRINCIPAL_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.PRINCIPAL_LIST_GRANTS, - PolarisPrivilege.PRINCIPAL_ROLE_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.PRINCIPAL_ROLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.PRINCIPAL_ROLE_LIST_GRANTS, - PolarisPrivilege.CATALOG_ROLE_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.CATALOG_ROLE_LIST_GRANTS, - PolarisPrivilege.CATALOG_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.CATALOG_LIST_GRANTS, - PolarisPrivilege.NAMESPACE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.NAMESPACE_LIST_GRANTS, - PolarisPrivilege.TABLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.TABLE_LIST_GRANTS, - PolarisPrivilege.VIEW_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.VIEW_LIST_GRANTS, - PolarisPrivilege.PRINCIPAL_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_ROLE_FULL_METADATA, - PolarisPrivilege.CATALOG_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT, - PolarisPrivilege.SERVICE_MANAGE_ACCESS), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE1)) - .assignCatalogRoleToPrincipalRole(PRINCIPAL_ROLE2, CATALOG_NAME, CATALOG_ROLE1), - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testRevokeCatalogRoleFromPrincipalRoleSufficientPrivileges() { - // Revoke requires privileges both on the "securable" (CatalogRole) as well as the "grantee" - // (PrincipalRole); neither CATALOG_MANAGE_ACCESS nor SERVICE_MANAGE_ACCESS alone are - // sufficient. - doTestSufficientPrivilegeSets( - List.of( - Set.of(PolarisPrivilege.CATALOG_MANAGE_ACCESS, PolarisPrivilege.SERVICE_MANAGE_ACCESS), - Set.of( - PolarisPrivilege.CATALOG_MANAGE_ACCESS, - PolarisPrivilege.PRINCIPAL_ROLE_MANAGE_GRANTS_FOR_GRANTEE), - Set.of( - PolarisPrivilege.CATALOG_ROLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.PRINCIPAL_ROLE_MANAGE_GRANTS_FOR_GRANTEE)), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE1)) - .revokeCatalogRoleFromPrincipalRole(PRINCIPAL_ROLE2, CATALOG_NAME, CATALOG_ROLE1), - null, // cleanupAction - PRINCIPAL_NAME, - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testRevokeCatalogRoleFromPrincipalRoleInsufficientPrivileges() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.PRINCIPAL_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.PRINCIPAL_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.PRINCIPAL_LIST_GRANTS, - PolarisPrivilege.PRINCIPAL_ROLE_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.PRINCIPAL_ROLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.PRINCIPAL_ROLE_LIST_GRANTS, - PolarisPrivilege.CATALOG_ROLE_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.CATALOG_ROLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.CATALOG_ROLE_LIST_GRANTS, - PolarisPrivilege.CATALOG_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.CATALOG_LIST_GRANTS, - PolarisPrivilege.NAMESPACE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.NAMESPACE_LIST_GRANTS, - PolarisPrivilege.TABLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.TABLE_LIST_GRANTS, - PolarisPrivilege.VIEW_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.VIEW_LIST_GRANTS, - PolarisPrivilege.PRINCIPAL_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_ROLE_FULL_METADATA, - PolarisPrivilege.CATALOG_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT, - PolarisPrivilege.CATALOG_MANAGE_ACCESS, - PolarisPrivilege.SERVICE_MANAGE_ACCESS), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE1)) - .revokeCatalogRoleFromPrincipalRole(PRINCIPAL_ROLE2, CATALOG_NAME, CATALOG_ROLE1), - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testGrantPrivilegeOnRootContainerToPrincipalRoleSufficientPrivileges() { - doTestSufficientPrivileges( - List.of(PolarisPrivilege.SERVICE_MANAGE_ACCESS), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE1)) - .grantPrivilegeOnRootContainerToPrincipalRole( - PRINCIPAL_ROLE2, PolarisPrivilege.SERVICE_MANAGE_ACCESS), - null, // cleanupAction - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testGrantPrivilegeOnRootContainerToPrincipalRoleInsufficientPrivileges() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.PRINCIPAL_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.PRINCIPAL_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.PRINCIPAL_LIST_GRANTS, - PolarisPrivilege.PRINCIPAL_ROLE_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.PRINCIPAL_ROLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.PRINCIPAL_ROLE_LIST_GRANTS, - PolarisPrivilege.CATALOG_ROLE_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.CATALOG_ROLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.CATALOG_ROLE_LIST_GRANTS, - PolarisPrivilege.CATALOG_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.CATALOG_LIST_GRANTS, - PolarisPrivilege.NAMESPACE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.NAMESPACE_LIST_GRANTS, - PolarisPrivilege.TABLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.TABLE_LIST_GRANTS, - PolarisPrivilege.VIEW_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.VIEW_LIST_GRANTS, - PolarisPrivilege.PRINCIPAL_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_ROLE_FULL_METADATA, - PolarisPrivilege.CATALOG_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT, - PolarisPrivilege.CATALOG_MANAGE_ACCESS), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE1)) - .grantPrivilegeOnRootContainerToPrincipalRole( - PRINCIPAL_ROLE2, PolarisPrivilege.SERVICE_MANAGE_ACCESS), - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testRevokePrivilegeOnRootContainerFromPrincipalRoleSufficientPrivileges() { - doTestSufficientPrivileges( - List.of(PolarisPrivilege.SERVICE_MANAGE_ACCESS), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE1)) - .revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE2, PolarisPrivilege.SERVICE_MANAGE_ACCESS), - null, // cleanupAction - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testRevokePrivilegeOnRootContainerFromPrincipalRoleInsufficientPrivileges() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.PRINCIPAL_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.PRINCIPAL_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.PRINCIPAL_LIST_GRANTS, - PolarisPrivilege.PRINCIPAL_ROLE_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.PRINCIPAL_ROLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.PRINCIPAL_ROLE_LIST_GRANTS, - PolarisPrivilege.CATALOG_ROLE_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.CATALOG_ROLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.CATALOG_ROLE_LIST_GRANTS, - PolarisPrivilege.CATALOG_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.CATALOG_LIST_GRANTS, - PolarisPrivilege.NAMESPACE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.NAMESPACE_LIST_GRANTS, - PolarisPrivilege.TABLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.TABLE_LIST_GRANTS, - PolarisPrivilege.VIEW_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.VIEW_LIST_GRANTS, - PolarisPrivilege.PRINCIPAL_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_ROLE_FULL_METADATA, - PolarisPrivilege.CATALOG_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT, - PolarisPrivilege.CATALOG_MANAGE_ACCESS), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE1)) - .revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE2, PolarisPrivilege.SERVICE_MANAGE_ACCESS), - (privilege) -> - adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnRootContainerFromPrincipalRole( - PRINCIPAL_ROLE1, privilege)); - } - - @Test - public void testGrantPrivilegeOnCatalogToRoleSufficientPrivileges() { - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.CATALOG_MANAGE_ACCESS, - PolarisPrivilege.CATALOG_MANAGE_GRANTS_ON_SECURABLE), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE1)) - .grantPrivilegeOnCatalogToRole( - CATALOG_NAME, CATALOG_ROLE2, PolarisPrivilege.CATALOG_MANAGE_ACCESS), - null, // cleanupAction - (privilege) -> - adminService.grantPrivilegeOnCatalogToRole(CATALOG_NAME, CATALOG_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnCatalogFromRole(CATALOG_NAME, CATALOG_ROLE1, privilege)); - } - - @Test - public void testGrantPrivilegeOnCatalogToRoleInsufficientPrivileges() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.PRINCIPAL_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.PRINCIPAL_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.PRINCIPAL_LIST_GRANTS, - PolarisPrivilege.PRINCIPAL_ROLE_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.PRINCIPAL_ROLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.PRINCIPAL_ROLE_LIST_GRANTS, - PolarisPrivilege.CATALOG_ROLE_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.CATALOG_ROLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.CATALOG_ROLE_LIST_GRANTS, - PolarisPrivilege.CATALOG_LIST_GRANTS, - PolarisPrivilege.NAMESPACE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.NAMESPACE_LIST_GRANTS, - PolarisPrivilege.TABLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.TABLE_LIST_GRANTS, - PolarisPrivilege.VIEW_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.VIEW_LIST_GRANTS, - PolarisPrivilege.PRINCIPAL_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_ROLE_FULL_METADATA, - PolarisPrivilege.CATALOG_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT, - PolarisPrivilege.SERVICE_MANAGE_ACCESS), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE1)) - .grantPrivilegeOnCatalogToRole( - CATALOG_NAME, CATALOG_ROLE2, PolarisPrivilege.CATALOG_MANAGE_ACCESS), - (privilege) -> - adminService.grantPrivilegeOnCatalogToRole(CATALOG_NAME, CATALOG_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnCatalogFromRole(CATALOG_NAME, CATALOG_ROLE1, privilege)); - } - - @Test - public void testRevokePrivilegeOnCatalogFromRoleSufficientPrivileges() { - doTestSufficientPrivilegeSets( - List.of( - Set.of(PolarisPrivilege.CATALOG_MANAGE_ACCESS), - Set.of( - PolarisPrivilege.CATALOG_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.CATALOG_ROLE_MANAGE_GRANTS_FOR_GRANTEE)), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE1)) - .revokePrivilegeOnCatalogFromRole( - CATALOG_NAME, CATALOG_ROLE2, PolarisPrivilege.CATALOG_MANAGE_ACCESS), - null, // cleanupAction - PRINCIPAL_NAME, - (privilege) -> - adminService.grantPrivilegeOnCatalogToRole(CATALOG_NAME, CATALOG_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnCatalogFromRole(CATALOG_NAME, CATALOG_ROLE1, privilege)); - } - - @Test - public void testRevokePrivilegeOnCatalogFromRoleInsufficientPrivileges() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.PRINCIPAL_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.PRINCIPAL_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.PRINCIPAL_LIST_GRANTS, - PolarisPrivilege.PRINCIPAL_ROLE_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.PRINCIPAL_ROLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.PRINCIPAL_ROLE_LIST_GRANTS, - PolarisPrivilege.CATALOG_ROLE_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.CATALOG_ROLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.CATALOG_ROLE_LIST_GRANTS, - PolarisPrivilege.CATALOG_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.CATALOG_LIST_GRANTS, - PolarisPrivilege.NAMESPACE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.NAMESPACE_LIST_GRANTS, - PolarisPrivilege.TABLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.TABLE_LIST_GRANTS, - PolarisPrivilege.VIEW_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.VIEW_LIST_GRANTS, - PolarisPrivilege.PRINCIPAL_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_ROLE_FULL_METADATA, - PolarisPrivilege.CATALOG_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT, - PolarisPrivilege.SERVICE_MANAGE_ACCESS), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE1)) - .revokePrivilegeOnCatalogFromRole( - CATALOG_NAME, CATALOG_ROLE2, PolarisPrivilege.CATALOG_MANAGE_ACCESS), - (privilege) -> - adminService.grantPrivilegeOnCatalogToRole(CATALOG_NAME, CATALOG_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnCatalogFromRole(CATALOG_NAME, CATALOG_ROLE1, privilege)); - } - - @Test - public void testGrantPrivilegeOnNamespaceToRoleSufficientPrivileges() { - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.CATALOG_MANAGE_ACCESS, - PolarisPrivilege.NAMESPACE_MANAGE_GRANTS_ON_SECURABLE), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE1)) - .grantPrivilegeOnNamespaceToRole( - CATALOG_NAME, CATALOG_ROLE2, NS1, PolarisPrivilege.CATALOG_MANAGE_ACCESS), - null, // cleanupAction - (privilege) -> - adminService.grantPrivilegeOnCatalogToRole(CATALOG_NAME, CATALOG_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnCatalogFromRole(CATALOG_NAME, CATALOG_ROLE1, privilege)); - } - - @Test - public void testGrantPrivilegeOnNamespaceToRoleInsufficientPrivileges() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.PRINCIPAL_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.PRINCIPAL_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.PRINCIPAL_LIST_GRANTS, - PolarisPrivilege.PRINCIPAL_ROLE_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.PRINCIPAL_ROLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.PRINCIPAL_ROLE_LIST_GRANTS, - PolarisPrivilege.CATALOG_ROLE_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.CATALOG_ROLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.CATALOG_ROLE_LIST_GRANTS, - PolarisPrivilege.CATALOG_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.CATALOG_LIST_GRANTS, - PolarisPrivilege.NAMESPACE_LIST_GRANTS, - PolarisPrivilege.TABLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.TABLE_LIST_GRANTS, - PolarisPrivilege.VIEW_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.VIEW_LIST_GRANTS, - PolarisPrivilege.PRINCIPAL_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_ROLE_FULL_METADATA, - PolarisPrivilege.CATALOG_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT, - PolarisPrivilege.SERVICE_MANAGE_ACCESS), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE1)) - .grantPrivilegeOnNamespaceToRole( - CATALOG_NAME, CATALOG_ROLE2, NS1, PolarisPrivilege.CATALOG_MANAGE_ACCESS), - (privilege) -> - adminService.grantPrivilegeOnCatalogToRole(CATALOG_NAME, CATALOG_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnCatalogFromRole(CATALOG_NAME, CATALOG_ROLE1, privilege)); - } - - @Test - public void testRevokePrivilegeOnNamespaceFromRoleSufficientPrivileges() { - doTestSufficientPrivilegeSets( - List.of( - Set.of(PolarisPrivilege.CATALOG_MANAGE_ACCESS), - Set.of( - PolarisPrivilege.NAMESPACE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.CATALOG_ROLE_MANAGE_GRANTS_FOR_GRANTEE)), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE1)) - .revokePrivilegeOnNamespaceFromRole( - CATALOG_NAME, CATALOG_ROLE2, NS1, PolarisPrivilege.CATALOG_MANAGE_ACCESS), - null, // cleanupAction - PRINCIPAL_NAME, - (privilege) -> - adminService.grantPrivilegeOnCatalogToRole(CATALOG_NAME, CATALOG_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnCatalogFromRole(CATALOG_NAME, CATALOG_ROLE1, privilege)); - } - - @Test - public void testRevokePrivilegeOnNamespaceFromRoleInsufficientPrivileges() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.PRINCIPAL_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.PRINCIPAL_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.PRINCIPAL_LIST_GRANTS, - PolarisPrivilege.PRINCIPAL_ROLE_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.PRINCIPAL_ROLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.PRINCIPAL_ROLE_LIST_GRANTS, - PolarisPrivilege.CATALOG_ROLE_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.CATALOG_ROLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.CATALOG_ROLE_LIST_GRANTS, - PolarisPrivilege.CATALOG_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.CATALOG_LIST_GRANTS, - PolarisPrivilege.NAMESPACE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.NAMESPACE_LIST_GRANTS, - PolarisPrivilege.TABLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.TABLE_LIST_GRANTS, - PolarisPrivilege.VIEW_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.VIEW_LIST_GRANTS, - PolarisPrivilege.PRINCIPAL_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_ROLE_FULL_METADATA, - PolarisPrivilege.CATALOG_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT, - PolarisPrivilege.SERVICE_MANAGE_ACCESS), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE1)) - .revokePrivilegeOnNamespaceFromRole( - CATALOG_NAME, CATALOG_ROLE2, NS1, PolarisPrivilege.CATALOG_MANAGE_ACCESS), - (privilege) -> - adminService.grantPrivilegeOnCatalogToRole(CATALOG_NAME, CATALOG_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnCatalogFromRole(CATALOG_NAME, CATALOG_ROLE1, privilege)); - } - - @Test - public void testGrantPrivilegeOnTableToRoleSufficientPrivileges() { - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.CATALOG_MANAGE_ACCESS, - PolarisPrivilege.TABLE_MANAGE_GRANTS_ON_SECURABLE), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE1)) - .grantPrivilegeOnTableToRole( - CATALOG_NAME, - CATALOG_ROLE2, - TABLE_NS1_1, - PolarisPrivilege.CATALOG_MANAGE_ACCESS), - null, // cleanupAction - (privilege) -> - adminService.grantPrivilegeOnCatalogToRole(CATALOG_NAME, CATALOG_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnCatalogFromRole(CATALOG_NAME, CATALOG_ROLE1, privilege)); - } - - @Test - public void testGrantPrivilegeOnTableToRoleInsufficientPrivileges() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.PRINCIPAL_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.PRINCIPAL_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.PRINCIPAL_LIST_GRANTS, - PolarisPrivilege.PRINCIPAL_ROLE_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.PRINCIPAL_ROLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.PRINCIPAL_ROLE_LIST_GRANTS, - PolarisPrivilege.CATALOG_ROLE_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.CATALOG_ROLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.CATALOG_ROLE_LIST_GRANTS, - PolarisPrivilege.CATALOG_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.CATALOG_LIST_GRANTS, - PolarisPrivilege.NAMESPACE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.NAMESPACE_LIST_GRANTS, - PolarisPrivilege.TABLE_LIST_GRANTS, - PolarisPrivilege.VIEW_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.VIEW_LIST_GRANTS, - PolarisPrivilege.PRINCIPAL_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_ROLE_FULL_METADATA, - PolarisPrivilege.CATALOG_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT, - PolarisPrivilege.SERVICE_MANAGE_ACCESS), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE1)) - .grantPrivilegeOnTableToRole( - CATALOG_NAME, - CATALOG_ROLE2, - TABLE_NS1_1, - PolarisPrivilege.CATALOG_MANAGE_ACCESS), - (privilege) -> - adminService.grantPrivilegeOnCatalogToRole(CATALOG_NAME, CATALOG_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnCatalogFromRole(CATALOG_NAME, CATALOG_ROLE1, privilege)); - } - - @Test - public void testRevokePrivilegeOnTableFromRoleSufficientPrivileges() { - doTestSufficientPrivilegeSets( - List.of( - Set.of(PolarisPrivilege.CATALOG_MANAGE_ACCESS), - Set.of( - PolarisPrivilege.TABLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.CATALOG_ROLE_MANAGE_GRANTS_FOR_GRANTEE)), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE1)) - .revokePrivilegeOnTableFromRole( - CATALOG_NAME, - CATALOG_ROLE2, - TABLE_NS1_1, - PolarisPrivilege.CATALOG_MANAGE_ACCESS), - null, // cleanupAction - PRINCIPAL_NAME, - (privilege) -> - adminService.grantPrivilegeOnCatalogToRole(CATALOG_NAME, CATALOG_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnCatalogFromRole(CATALOG_NAME, CATALOG_ROLE1, privilege)); - } - - @Test - public void testRevokePrivilegeOnTableFromRoleInsufficientPrivileges() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.PRINCIPAL_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.PRINCIPAL_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.PRINCIPAL_LIST_GRANTS, - PolarisPrivilege.PRINCIPAL_ROLE_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.PRINCIPAL_ROLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.PRINCIPAL_ROLE_LIST_GRANTS, - PolarisPrivilege.CATALOG_ROLE_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.CATALOG_ROLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.CATALOG_ROLE_LIST_GRANTS, - PolarisPrivilege.CATALOG_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.CATALOG_LIST_GRANTS, - PolarisPrivilege.NAMESPACE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.NAMESPACE_LIST_GRANTS, - PolarisPrivilege.TABLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.TABLE_LIST_GRANTS, - PolarisPrivilege.VIEW_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.VIEW_LIST_GRANTS, - PolarisPrivilege.PRINCIPAL_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_ROLE_FULL_METADATA, - PolarisPrivilege.CATALOG_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT, - PolarisPrivilege.SERVICE_MANAGE_ACCESS), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE1)) - .revokePrivilegeOnTableFromRole( - CATALOG_NAME, - CATALOG_ROLE2, - TABLE_NS1_1, - PolarisPrivilege.CATALOG_MANAGE_ACCESS), - (privilege) -> - adminService.grantPrivilegeOnCatalogToRole(CATALOG_NAME, CATALOG_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnCatalogFromRole(CATALOG_NAME, CATALOG_ROLE1, privilege)); - } - - @Test - public void testGrantPrivilegeOnViewToRoleSufficientPrivileges() { - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.CATALOG_MANAGE_ACCESS, - PolarisPrivilege.VIEW_MANAGE_GRANTS_ON_SECURABLE), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE1)) - .grantPrivilegeOnViewToRole( - CATALOG_NAME, - CATALOG_ROLE2, - VIEW_NS1_1, - PolarisPrivilege.CATALOG_MANAGE_ACCESS), - null, // cleanupAction - (privilege) -> - adminService.grantPrivilegeOnCatalogToRole(CATALOG_NAME, CATALOG_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnCatalogFromRole(CATALOG_NAME, CATALOG_ROLE1, privilege)); - } - - @Test - public void testGrantPrivilegeOnViewToRoleInsufficientPrivileges() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.PRINCIPAL_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.PRINCIPAL_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.PRINCIPAL_LIST_GRANTS, - PolarisPrivilege.PRINCIPAL_ROLE_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.PRINCIPAL_ROLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.PRINCIPAL_ROLE_LIST_GRANTS, - PolarisPrivilege.CATALOG_ROLE_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.CATALOG_ROLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.CATALOG_ROLE_LIST_GRANTS, - PolarisPrivilege.CATALOG_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.CATALOG_LIST_GRANTS, - PolarisPrivilege.NAMESPACE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.NAMESPACE_LIST_GRANTS, - PolarisPrivilege.TABLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.TABLE_LIST_GRANTS, - PolarisPrivilege.VIEW_LIST_GRANTS, - PolarisPrivilege.PRINCIPAL_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_ROLE_FULL_METADATA, - PolarisPrivilege.CATALOG_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT, - PolarisPrivilege.SERVICE_MANAGE_ACCESS), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE1)) - .grantPrivilegeOnViewToRole( - CATALOG_NAME, - CATALOG_ROLE2, - VIEW_NS1_1, - PolarisPrivilege.CATALOG_MANAGE_ACCESS), - (privilege) -> - adminService.grantPrivilegeOnCatalogToRole(CATALOG_NAME, CATALOG_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnCatalogFromRole(CATALOG_NAME, CATALOG_ROLE1, privilege)); - } - - @Test - public void testRevokePrivilegeOnViewFromRoleSufficientPrivileges() { - doTestSufficientPrivilegeSets( - List.of( - Set.of(PolarisPrivilege.CATALOG_MANAGE_ACCESS), - Set.of( - PolarisPrivilege.VIEW_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.CATALOG_ROLE_MANAGE_GRANTS_FOR_GRANTEE)), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE1)) - .revokePrivilegeOnViewFromRole( - CATALOG_NAME, - CATALOG_ROLE2, - VIEW_NS1_1, - PolarisPrivilege.CATALOG_MANAGE_ACCESS), - null, // cleanupAction - PRINCIPAL_NAME, - (privilege) -> - adminService.grantPrivilegeOnCatalogToRole(CATALOG_NAME, CATALOG_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnCatalogFromRole(CATALOG_NAME, CATALOG_ROLE1, privilege)); - } - - @Test - public void testRevokePrivilegeOnViewFromRoleInsufficientPrivileges() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.PRINCIPAL_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.PRINCIPAL_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.PRINCIPAL_LIST_GRANTS, - PolarisPrivilege.PRINCIPAL_ROLE_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.PRINCIPAL_ROLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.PRINCIPAL_ROLE_LIST_GRANTS, - PolarisPrivilege.CATALOG_ROLE_MANAGE_GRANTS_FOR_GRANTEE, - PolarisPrivilege.CATALOG_ROLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.CATALOG_ROLE_LIST_GRANTS, - PolarisPrivilege.CATALOG_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.CATALOG_LIST_GRANTS, - PolarisPrivilege.NAMESPACE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.NAMESPACE_LIST_GRANTS, - PolarisPrivilege.TABLE_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.TABLE_LIST_GRANTS, - PolarisPrivilege.VIEW_MANAGE_GRANTS_ON_SECURABLE, - PolarisPrivilege.VIEW_LIST_GRANTS, - PolarisPrivilege.PRINCIPAL_FULL_METADATA, - PolarisPrivilege.PRINCIPAL_ROLE_FULL_METADATA, - PolarisPrivilege.CATALOG_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT, - PolarisPrivilege.SERVICE_MANAGE_ACCESS), - () -> - newTestAdminService(Set.of(PRINCIPAL_ROLE1)) - .revokePrivilegeOnViewFromRole( - CATALOG_NAME, - CATALOG_ROLE2, - VIEW_NS1_1, - PolarisPrivilege.CATALOG_MANAGE_ACCESS), - (privilege) -> - adminService.grantPrivilegeOnCatalogToRole(CATALOG_NAME, CATALOG_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnCatalogFromRole(CATALOG_NAME, CATALOG_ROLE1, privilege)); - } -} diff --git a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java b/polaris-service-quarkus/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java deleted file mode 100644 index a66ed10da..000000000 --- a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java +++ /dev/null @@ -1,548 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.admin; - -import static org.apache.iceberg.types.Types.NestedField.required; - -import com.google.auth.oauth2.AccessToken; -import com.google.auth.oauth2.GoogleCredentials; -import com.google.common.collect.ImmutableMap; -import io.quarkus.test.junit.QuarkusMock; -import jakarta.annotation.Nonnull; -import jakarta.enterprise.inject.Vetoed; -import jakarta.enterprise.inject.spi.CDI; -import jakarta.inject.Inject; -import java.io.IOException; -import java.time.Clock; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Function; -import org.apache.iceberg.CatalogProperties; -import org.apache.iceberg.Schema; -import org.apache.iceberg.catalog.Catalog; -import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.exceptions.ForbiddenException; -import org.apache.iceberg.types.Types; -import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.PolarisConfiguration; -import org.apache.polaris.core.PolarisConfigurationStore; -import org.apache.polaris.core.PolarisDiagnostics; -import org.apache.polaris.core.admin.model.FileStorageConfigInfo; -import org.apache.polaris.core.admin.model.PrincipalWithCredentials; -import org.apache.polaris.core.admin.model.PrincipalWithCredentialsCredentials; -import org.apache.polaris.core.admin.model.StorageConfigInfo; -import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; -import org.apache.polaris.core.auth.PolarisAuthorizer; -import org.apache.polaris.core.auth.PolarisAuthorizerImpl; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.core.entity.CatalogEntity; -import org.apache.polaris.core.entity.CatalogRoleEntity; -import org.apache.polaris.core.entity.PolarisBaseEntity; -import org.apache.polaris.core.entity.PolarisEntity; -import org.apache.polaris.core.entity.PolarisEntitySubType; -import org.apache.polaris.core.entity.PolarisEntityType; -import org.apache.polaris.core.entity.PolarisPrivilege; -import org.apache.polaris.core.entity.PrincipalEntity; -import org.apache.polaris.core.entity.PrincipalRoleEntity; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.core.persistence.PolarisEntityManager; -import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest; -import org.apache.polaris.service.catalog.BasePolarisCatalog; -import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView; -import org.apache.polaris.service.catalog.io.DefaultFileIOFactory; -import org.apache.polaris.service.catalog.io.FileIOFactory; -import org.apache.polaris.service.config.DefaultConfigurationStore; -import org.apache.polaris.service.config.RealmEntityManagerFactory; -import org.apache.polaris.service.context.CallContextCatalogFactory; -import org.apache.polaris.service.context.PolarisCallContextCatalogFactory; -import org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl; -import org.apache.polaris.service.task.TaskExecutor; -import org.assertj.core.api.Assertions; -import org.jetbrains.annotations.Nullable; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.TestInfo; -import org.mockito.Mockito; - -/** Base class for shared test setup logic used by various Polaris authz-related tests. */ -public abstract class PolarisAuthzTestBase { - protected static final String CATALOG_NAME = "polaris-catalog"; - protected static final String PRINCIPAL_NAME = "snowman"; - - // catalog_role1 will be assigned only to principal_role1 and - // catalog_role2 will be assigned only to principal_role2 - protected static final String PRINCIPAL_ROLE1 = "principal_role1"; - protected static final String PRINCIPAL_ROLE2 = "principal_role2"; - protected static final String CATALOG_ROLE1 = "catalog_role1"; - protected static final String CATALOG_ROLE2 = "catalog_role2"; - protected static final String CATALOG_ROLE_SHARED = "catalog_role_shared"; - - protected static final Namespace NS1 = Namespace.of("ns1"); - protected static final Namespace NS2 = Namespace.of("ns2"); - protected static final Namespace NS1A = Namespace.of("ns1", "ns1a"); - protected static final Namespace NS1AA = Namespace.of("ns1", "ns1a", "ns1aa"); - protected static final Namespace NS1B = Namespace.of("ns1", "ns1b"); - - // One table directly under ns1 - protected static final TableIdentifier TABLE_NS1_1 = TableIdentifier.of(NS1, "layer1_table"); - - // Two tables under ns1a - protected static final TableIdentifier TABLE_NS1A_1 = TableIdentifier.of(NS1A, "table1"); - protected static final TableIdentifier TABLE_NS1A_2 = TableIdentifier.of(NS1A, "table2"); - - // One table under ns1b with same name as one under ns1a - protected static final TableIdentifier TABLE_NS1B_1 = TableIdentifier.of(NS1B, "table1"); - - // One table directly under ns2 - protected static final TableIdentifier TABLE_NS2_1 = TableIdentifier.of(NS2, "table1"); - - // One view directly under ns1 - protected static final TableIdentifier VIEW_NS1_1 = TableIdentifier.of(NS1, "layer1_view"); - - // Two views under ns1a - protected static final TableIdentifier VIEW_NS1A_1 = TableIdentifier.of(NS1A, "view1"); - protected static final TableIdentifier VIEW_NS1A_2 = TableIdentifier.of(NS1A, "view2"); - - // One view under ns1b with same name as one under ns1a - protected static final TableIdentifier VIEW_NS1B_1 = TableIdentifier.of(NS1B, "view1"); - - // One view directly under ns2 - protected static final TableIdentifier VIEW_NS2_1 = TableIdentifier.of(NS2, "view1"); - - protected static final String VIEW_QUERY = "select * from ns1.layer1_table"; - - public static final Schema SCHEMA = - new Schema( - required(3, "id", Types.IntegerType.get(), "unique ID 🤪"), - required(4, "data", Types.StringType.get())); - protected final PolarisAuthorizer polarisAuthorizer = - new PolarisAuthorizerImpl( - new DefaultConfigurationStore( - Map.of( - PolarisConfiguration.ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING.key, - true))); - - @Inject protected MetaStoreManagerFactory managerFactory; - @Inject protected RealmEntityManagerFactory realmEntityManagerFactory; - @Inject protected CallContextCatalogFactory callContextCatalogFactory; - @Inject protected PolarisDiagnostics diagServices; - - protected BasePolarisCatalog baseCatalog; - protected PolarisAdminService adminService; - protected PolarisEntityManager entityManager; - protected PolarisMetaStoreManager metaStoreManager; - protected PolarisBaseEntity catalogEntity; - protected PrincipalEntity principalEntity; - protected CallContext callContext; - protected AuthenticatedPolarisPrincipal authenticatedRoot; - - private PolarisCallContext polarisContext; - - @BeforeAll - public static void setUpMocks() { - PolarisStorageIntegrationProviderImpl mock = - new PolarisStorageIntegrationProviderImpl( - Mockito::mock, () -> GoogleCredentials.create(new AccessToken("abc", new Date()))); - QuarkusMock.installMockForType(mock, PolarisStorageIntegrationProviderImpl.class); - RealmEntityManagerFactory realmEntityManagerFactory = - CDI.current().select(RealmEntityManagerFactory.class).get(); - TaskExecutor taskExecutor = CDI.current().select(TaskExecutor.class).get(); - FileIOFactory fileIOFactory = CDI.current().select(FileIOFactory.class).get(); - MetaStoreManagerFactory metaStoreManagerFactory = - CDI.current().select(MetaStoreManagerFactory.class).get(); - TestPolarisCallContextCatalogFactory m = - new TestPolarisCallContextCatalogFactory( - realmEntityManagerFactory, metaStoreManagerFactory, taskExecutor, fileIOFactory); - QuarkusMock.installMockForType(m, PolarisCallContextCatalogFactory.class); - } - - @BeforeEach - public void before(TestInfo testInfo) { - RealmContext realmContext = testInfo::getDisplayName; - metaStoreManager = managerFactory.getOrCreateMetaStoreManager(realmContext); - - Map configMap = - Map.of( - "ALLOW_SPECIFYING_FILE_IO_IMPL", true, "ALLOW_EXTERNAL_METADATA_FILE_LOCATION", true); - polarisContext = - new PolarisCallContext( - managerFactory.getOrCreateSessionSupplier(realmContext).get(), - diagServices, - new PolarisConfigurationStore() { - @Override - public @Nullable T getConfiguration(PolarisCallContext ctx, String configName) { - return (T) configMap.get(configName); - } - }, - Clock.systemDefaultZone()); - this.entityManager = realmEntityManagerFactory.getOrCreateEntityManager(realmContext); - - callContext = CallContext.of(realmContext, polarisContext); - CallContext.setCurrentContext(callContext); - - PrincipalEntity rootEntity = - new PrincipalEntity( - PolarisEntity.of( - metaStoreManager - .readEntityByName( - polarisContext, - null, - PolarisEntityType.PRINCIPAL, - PolarisEntitySubType.NULL_SUBTYPE, - "root") - .getEntity())); - - this.authenticatedRoot = new AuthenticatedPolarisPrincipal(rootEntity, Set.of()); - - this.adminService = - new PolarisAdminService( - callContext, entityManager, metaStoreManager, authenticatedRoot, polarisAuthorizer); - - String storageLocation = "file:///tmp/authz"; - FileStorageConfigInfo storageConfigModel = - FileStorageConfigInfo.builder() - .setStorageType(StorageConfigInfo.StorageTypeEnum.FILE) - .setAllowedLocations(List.of(storageLocation, "file:///tmp/authz")) - .build(); - catalogEntity = - adminService.createCatalog( - new CatalogEntity.Builder() - .setName(CATALOG_NAME) - .setCatalogType("INTERNAL") - .setDefaultBaseLocation(storageLocation) - .setStorageConfigurationInfo(storageConfigModel, storageLocation) - .build()); - - initBaseCatalog(); - - PrincipalWithCredentials principal = - adminService.createPrincipal(new PrincipalEntity.Builder().setName(PRINCIPAL_NAME).build()); - principalEntity = - rotateAndRefreshPrincipal( - metaStoreManager, PRINCIPAL_NAME, principal.getCredentials(), polarisContext); - - // Pre-create the principal roles and catalog roles without any grants on securables, but - // assign both principal roles to the principal, then CATALOG_ROLE1 to PRINCIPAL_ROLE1, - // CATALOG_ROLE2 to PRINCIPAL_ROLE2, and CATALOG_ROLE_SHARED to both. - adminService.createPrincipalRole( - new PrincipalRoleEntity.Builder().setName(PRINCIPAL_ROLE1).build()); - adminService.createPrincipalRole( - new PrincipalRoleEntity.Builder().setName(PRINCIPAL_ROLE2).build()); - adminService.createCatalogRole( - CATALOG_NAME, new CatalogRoleEntity.Builder().setName(CATALOG_ROLE1).build()); - adminService.createCatalogRole( - CATALOG_NAME, new CatalogRoleEntity.Builder().setName(CATALOG_ROLE2).build()); - adminService.createCatalogRole( - CATALOG_NAME, new CatalogRoleEntity.Builder().setName(CATALOG_ROLE_SHARED).build()); - - adminService.assignPrincipalRole(PRINCIPAL_NAME, PRINCIPAL_ROLE1); - adminService.assignPrincipalRole(PRINCIPAL_NAME, PRINCIPAL_ROLE2); - - adminService.assignCatalogRoleToPrincipalRole(PRINCIPAL_ROLE1, CATALOG_NAME, CATALOG_ROLE1); - adminService.assignCatalogRoleToPrincipalRole(PRINCIPAL_ROLE2, CATALOG_NAME, CATALOG_ROLE2); - adminService.assignCatalogRoleToPrincipalRole( - PRINCIPAL_ROLE1, CATALOG_NAME, CATALOG_ROLE_SHARED); - adminService.assignCatalogRoleToPrincipalRole( - PRINCIPAL_ROLE2, CATALOG_NAME, CATALOG_ROLE_SHARED); - - // Do some shared setup with non-authz-aware baseCatalog. - baseCatalog.createNamespace(NS1); - baseCatalog.createNamespace(NS2); - baseCatalog.createNamespace(NS1A); - baseCatalog.createNamespace(NS1AA); - baseCatalog.createNamespace(NS1B); - - baseCatalog.buildTable(TABLE_NS1_1, SCHEMA).create(); - baseCatalog.buildTable(TABLE_NS1A_1, SCHEMA).create(); - baseCatalog.buildTable(TABLE_NS1A_2, SCHEMA).create(); - baseCatalog.buildTable(TABLE_NS1B_1, SCHEMA).create(); - baseCatalog.buildTable(TABLE_NS2_1, SCHEMA).create(); - - baseCatalog - .buildView(VIEW_NS1_1) - .withSchema(SCHEMA) - .withDefaultNamespace(NS1) - .withQuery("spark", VIEW_QUERY) - .create(); - baseCatalog - .buildView(VIEW_NS1A_1) - .withSchema(SCHEMA) - .withDefaultNamespace(NS1) - .withQuery("spark", VIEW_QUERY) - .create(); - baseCatalog - .buildView(VIEW_NS1A_2) - .withSchema(SCHEMA) - .withDefaultNamespace(NS1) - .withQuery("spark", VIEW_QUERY) - .create(); - baseCatalog - .buildView(VIEW_NS1B_1) - .withSchema(SCHEMA) - .withDefaultNamespace(NS1) - .withQuery("spark", VIEW_QUERY) - .create(); - baseCatalog - .buildView(VIEW_NS2_1) - .withSchema(SCHEMA) - .withDefaultNamespace(NS1) - .withQuery("spark", VIEW_QUERY) - .create(); - } - - @AfterEach - public void after() { - try { - if (this.baseCatalog != null) { - try { - this.baseCatalog.close(); - this.baseCatalog = null; - } catch (IOException e) { - throw new RuntimeException(e); - } - } - } finally { - metaStoreManager.purge(polarisContext); - } - } - - protected @Nonnull PrincipalEntity rotateAndRefreshPrincipal( - PolarisMetaStoreManager metaStoreManager, - String principalName, - PrincipalWithCredentialsCredentials credentials, - PolarisCallContext polarisContext) { - PolarisMetaStoreManager.EntityResult lookupEntity = - metaStoreManager.readEntityByName( - callContext.getPolarisCallContext(), - null, - PolarisEntityType.PRINCIPAL, - PolarisEntitySubType.NULL_SUBTYPE, - principalName); - metaStoreManager.rotatePrincipalSecrets( - callContext.getPolarisCallContext(), - credentials.getClientId(), - lookupEntity.getEntity().getId(), - false, - credentials.getClientSecret()); // This should actually be the secret's hash - - return new PrincipalEntity( - PolarisEntity.of( - metaStoreManager - .readEntityByName( - polarisContext, - null, - PolarisEntityType.PRINCIPAL, - PolarisEntitySubType.NULL_SUBTYPE, - principalName) - .getEntity())); - } - - /** - * This baseCatalog is used for setup rather than being the test target under a wrapper instance; - * we set up this baseCatalog with a PolarisPassthroughResolutionView to allow it to circumvent - * the "authorized" resolution set of entities used by wrapper instances, allowing it to resolve - * all entities in the underlying metaStoreManager at once. - */ - private void initBaseCatalog() { - if (this.baseCatalog != null) { - try { - this.baseCatalog.close(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - PolarisPassthroughResolutionView passthroughView = - new PolarisPassthroughResolutionView( - callContext, entityManager, authenticatedRoot, CATALOG_NAME); - this.baseCatalog = - new BasePolarisCatalog( - entityManager, - metaStoreManager, - callContext, - passthroughView, - authenticatedRoot, - Mockito.mock(), - new DefaultFileIOFactory()); - this.baseCatalog.initialize( - CATALOG_NAME, - ImmutableMap.of( - CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO")); - } - - @Vetoed - public static class TestPolarisCallContextCatalogFactory - extends PolarisCallContextCatalogFactory { - - public TestPolarisCallContextCatalogFactory( - RealmEntityManagerFactory entityManagerFactory, - MetaStoreManagerFactory metaStoreManagerFactory, - TaskExecutor taskExecutor, - FileIOFactory fileIOFactory) { - super(entityManagerFactory, metaStoreManagerFactory, taskExecutor, fileIOFactory); - } - - @Override - public Catalog createCallContextCatalog( - CallContext context, - AuthenticatedPolarisPrincipal authenticatedPolarisPrincipal, - final PolarisResolutionManifest resolvedManifest) { - // This depends on the BasePolarisCatalog allowing calling initialize multiple times - // to override the previous config. - Catalog catalog = - super.createCallContextCatalog(context, authenticatedPolarisPrincipal, resolvedManifest); - catalog.initialize( - CATALOG_NAME, - ImmutableMap.of( - CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO")); - return catalog; - } - } - - /** - * Tests each "sufficient" privilege individually by invoking {@code grantAction} for each set of - * privileges, running the action being tested, revoking after each test set, and also ensuring - * that the request fails after each revocation. - * - * @param sufficientPrivileges each set of concurrent privileges expected to be sufficient - * together. - * @param action The operation being tested; could also be multiple operations that should all - * succeed with the sufficient privilege - * @param cleanupAction If non-null, additional action to run to "undo" a previous success action - * in case the action has side effects. Called before revoking the sufficient privilege; - * either the cleanup privileges must be latent, or the cleanup action could be run with - * PRINCIPAL_ROLE2 while runnint {@code action} with PRINCIPAL_ROLE1. - * @param principalName the name expected to appear in forbidden errors - * @param grantAction the grantPrivilege action to use for each test privilege that will apply the - * privilege to whatever context is used in the {@code action} - * @param revokeAction the revokePrivilege action to clean up after each granted test privilege - */ - protected void doTestSufficientPrivilegeSets( - List> sufficientPrivileges, - Runnable action, - Runnable cleanupAction, - String principalName, - Function grantAction, - Function revokeAction) { - for (Set privilegeSet : sufficientPrivileges) { - for (PolarisPrivilege privilege : privilegeSet) { - // Grant the single privilege at a catalog level to cascade to all objects. - Assertions.assertThat(grantAction.apply(privilege)).isTrue(); - } - - // Should run without issues. - try { - action.run(); - } catch (Throwable t) { - Assertions.fail( - String.format( - "Expected success with sufficientPrivileges '%s', got throwable instead.", - privilegeSet), - t); - } - if (cleanupAction != null) { - try { - cleanupAction.run(); - } catch (Throwable t) { - Assertions.fail( - String.format( - "Running cleanupAction with sufficientPrivileges '%s', got throwable.", - privilegeSet), - t); - } - } - - if (privilegeSet.size() > 1) { - // Knockout testing - Revoke single privileges and the same action should throw - // NotAuthorizedException. - for (PolarisPrivilege privilege : privilegeSet) { - Assertions.assertThat(revokeAction.apply(privilege)).isTrue(); - - try { - Assertions.assertThatThrownBy(() -> action.run()) - .isInstanceOf(ForbiddenException.class) - .hasMessageContaining(principalName) - .hasMessageContaining("is not authorized"); - } catch (Throwable t) { - Assertions.fail( - String.format( - "Expected failure after revoking sufficientPrivilege '%s' from set '%s'", - privilege, privilegeSet), - t); - } - - // Grant the single privilege at a catalog level to cascade to all objects. - Assertions.assertThat(grantAction.apply(privilege)).isTrue(); - } - } - - // Now remove all the privileges - for (PolarisPrivilege privilege : privilegeSet) { - Assertions.assertThat(revokeAction.apply(privilege)).isTrue(); - } - try { - Assertions.assertThatThrownBy(() -> action.run()) - .isInstanceOf(ForbiddenException.class) - .hasMessageContaining(principalName) - .hasMessageContaining("is not authorized"); - } catch (Throwable t) { - Assertions.fail( - String.format( - "Expected failure after revoking all sufficientPrivileges '%s'", privilegeSet), - t); - } - } - } - - /** - * Tests each "insufficient" privilege individually using CATALOG_ROLE1 by granting at the - * CATALOG_NAME level, ensuring the action fails, then revoking after each test case. - */ - protected void doTestInsufficientPrivileges( - List insufficientPrivileges, - String principalName, - Runnable action, - Function grantAction, - Function revokeAction) { - for (PolarisPrivilege privilege : insufficientPrivileges) { - // Grant the single privilege at a catalog level to cascade to all objects. - Assertions.assertThat(grantAction.apply(privilege)).isTrue(); - - // Should be insufficient - try { - Assertions.assertThatThrownBy(() -> action.run()) - .isInstanceOf(ForbiddenException.class) - .hasMessageContaining(principalName) - .hasMessageContaining("is not authorized"); - } catch (Throwable t) { - Assertions.fail( - String.format("Expected failure with insufficientPrivilege '%s'", privilege), t); - } - - // Revoking only matters in case there are some multi-privilege actions being tested with - // only granting individual privileges in isolation. - Assertions.assertThat(revokeAction.apply(privilege)).isTrue(); - } - } -} diff --git a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingCatalogTest.java b/polaris-service-quarkus/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingCatalogTest.java deleted file mode 100644 index 8548b8a7e..000000000 --- a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingCatalogTest.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.admin; - -import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; -import static org.assertj.core.api.Assertions.assertThat; - -import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.QuarkusTestProfile; -import io.quarkus.test.junit.TestProfile; -import jakarta.inject.Inject; -import jakarta.ws.rs.client.Entity; -import jakarta.ws.rs.client.Invocation; -import jakarta.ws.rs.core.Response; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; -import org.apache.polaris.core.admin.model.Catalog; -import org.apache.polaris.core.admin.model.CatalogProperties; -import org.apache.polaris.core.admin.model.CreateCatalogRequest; -import org.apache.polaris.core.admin.model.StorageConfigInfo; -import org.apache.polaris.service.test.PolarisIntegrationTestHelper; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.TestInfo; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; - -@QuarkusTest -@TestInstance(TestInstance.Lifecycle.PER_CLASS) -@TestProfile(PolarisOverlappingCatalogTest.Profile.class) -public class PolarisOverlappingCatalogTest { - - @Inject PolarisIntegrationTestHelper testHelper; - - @BeforeAll - public void setUp(TestInfo testInfo) { - testHelper.setUp(testInfo); - } - - @AfterAll - public void tearDown() { - testHelper.tearDown(); - } - - private Response createCatalog(String prefix, String defaultBaseLocation, boolean isExternal) { - return createCatalog(prefix, defaultBaseLocation, isExternal, new ArrayList()); - } - - private Invocation.Builder request() { - return testHelper - .client - .target( - String.format("http://localhost:%d/api/management/v1/catalogs", testHelper.localPort)) - .request("application/json") - .header("Authorization", "Bearer " + testHelper.adminToken) - .header(REALM_PROPERTY_KEY, testHelper.realm); - } - - private Response createCatalog( - String prefix, - String defaultBaseLocation, - boolean isExternal, - List allowedLocations) { - String uuid = UUID.randomUUID().toString(); - StorageConfigInfo config = - AwsStorageConfigInfo.builder() - .setRoleArn("arn:aws:iam::123456789012:role/my-role") - .setExternalId("externalId") - .setUserArn("userArn") - .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) - .setAllowedLocations( - allowedLocations.stream() - .map( - l -> { - return String.format("s3://bucket/%s/%s", prefix, l); - }) - .toList()) - .build(); - Catalog catalog = - new Catalog( - isExternal ? Catalog.TypeEnum.EXTERNAL : Catalog.TypeEnum.INTERNAL, - String.format("overlap_catalog_%s", uuid), - new CatalogProperties(String.format("s3://bucket/%s/%s", prefix, defaultBaseLocation)), - System.currentTimeMillis(), - System.currentTimeMillis(), - 1, - config); - try (Response response = request().post(Entity.json(new CreateCatalogRequest(catalog)))) { - return response; - } - } - - @ParameterizedTest - @CsvSource({"true, true", "true, false", "false, true", "false, false"}) - public void testBasicOverlappingCatalogs(boolean initiallyExternal, boolean laterExternal) { - String prefix = UUID.randomUUID().toString(); - - assertThat(createCatalog(prefix, "root", initiallyExternal)) - .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - - // OK, non-overlapping - assertThat(createCatalog(prefix, "boot", laterExternal)) - .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - - // OK, non-overlapping due to no `/` - assertThat(createCatalog(prefix, "roo", laterExternal)) - .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - - // Also OK due to no `/` - assertThat(createCatalog(prefix, "root.child", laterExternal)) - .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - - // inside `root` - assertThat(createCatalog(prefix, "root/child", laterExternal)) - .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); - - // `root` is inside this - assertThat(createCatalog(prefix, "", laterExternal)) - .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); - } - - @ParameterizedTest - @CsvSource({"true, true", "true, false", "false, true", "false, false"}) - public void testAllowedLocationOverlappingCatalogs( - boolean initiallyExternal, boolean laterExternal) { - String prefix = UUID.randomUUID().toString(); - - assertThat(createCatalog(prefix, "animals", initiallyExternal, Arrays.asList("dogs", "cats"))) - .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - - // OK, non-overlapping - assertThat(createCatalog(prefix, "danimals", laterExternal, Arrays.asList("dan", "daniel"))) - .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - - // This DBL overlaps with initial AL - assertThat(createCatalog(prefix, "dogs", initiallyExternal, Arrays.asList("huskies", "labs"))) - .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); - - // This AL overlaps with initial DBL - assertThat( - createCatalog( - prefix, "kingdoms", initiallyExternal, Arrays.asList("plants", "animals"))) - .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); - - // This AL overlaps with an initial AL - assertThat(createCatalog(prefix, "plays", initiallyExternal, Arrays.asList("rent", "cats"))) - .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); - } - - public static class Profile implements QuarkusTestProfile { - - @Override - public Map getConfigOverrides() { - return Map.of( - "polaris.config.feature-configurations.ALLOW_OVERLAPPING_CATALOG_URLS", "false"); - } - } -} diff --git a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplIntegrationTest.java b/polaris-service-quarkus/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplIntegrationTest.java deleted file mode 100644 index 9a3d0e0b7..000000000 --- a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplIntegrationTest.java +++ /dev/null @@ -1,2340 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.admin; - -import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; -import static org.assertj.core.api.Assertions.assertThat; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.QuarkusTestProfile; -import io.quarkus.test.junit.TestProfile; -import jakarta.inject.Inject; -import jakarta.ws.rs.client.Entity; -import jakarta.ws.rs.client.Invocation; -import jakarta.ws.rs.core.Response; -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import org.apache.commons.lang3.RandomStringUtils; -import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.rest.RESTUtil; -import org.apache.iceberg.rest.requests.CreateNamespaceRequest; -import org.apache.iceberg.rest.responses.ErrorResponse; -import org.apache.iceberg.rest.responses.ListNamespacesResponse; -import org.apache.polaris.core.admin.model.AddGrantRequest; -import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; -import org.apache.polaris.core.admin.model.AzureStorageConfigInfo; -import org.apache.polaris.core.admin.model.Catalog; -import org.apache.polaris.core.admin.model.CatalogGrant; -import org.apache.polaris.core.admin.model.CatalogPrivilege; -import org.apache.polaris.core.admin.model.CatalogProperties; -import org.apache.polaris.core.admin.model.CatalogRole; -import org.apache.polaris.core.admin.model.CatalogRoles; -import org.apache.polaris.core.admin.model.Catalogs; -import org.apache.polaris.core.admin.model.CreateCatalogRequest; -import org.apache.polaris.core.admin.model.CreateCatalogRoleRequest; -import org.apache.polaris.core.admin.model.CreatePrincipalRequest; -import org.apache.polaris.core.admin.model.CreatePrincipalRoleRequest; -import org.apache.polaris.core.admin.model.ExternalCatalog; -import org.apache.polaris.core.admin.model.FileStorageConfigInfo; -import org.apache.polaris.core.admin.model.GcpStorageConfigInfo; -import org.apache.polaris.core.admin.model.GrantCatalogRoleRequest; -import org.apache.polaris.core.admin.model.GrantPrincipalRoleRequest; -import org.apache.polaris.core.admin.model.GrantResource; -import org.apache.polaris.core.admin.model.NamespaceGrant; -import org.apache.polaris.core.admin.model.NamespacePrivilege; -import org.apache.polaris.core.admin.model.PolarisCatalog; -import org.apache.polaris.core.admin.model.Principal; -import org.apache.polaris.core.admin.model.PrincipalRole; -import org.apache.polaris.core.admin.model.PrincipalRoles; -import org.apache.polaris.core.admin.model.PrincipalWithCredentials; -import org.apache.polaris.core.admin.model.PrincipalWithCredentialsCredentials; -import org.apache.polaris.core.admin.model.Principals; -import org.apache.polaris.core.admin.model.StorageConfigInfo; -import org.apache.polaris.core.admin.model.UpdateCatalogRequest; -import org.apache.polaris.core.admin.model.UpdateCatalogRoleRequest; -import org.apache.polaris.core.admin.model.UpdatePrincipalRequest; -import org.apache.polaris.core.admin.model.UpdatePrincipalRoleRequest; -import org.apache.polaris.core.entity.PolarisEntityConstants; -import org.apache.polaris.service.auth.TokenUtils; -import org.apache.polaris.service.test.PolarisIntegrationTestHelper; -import org.assertj.core.api.InstanceOfAssertFactories; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInfo; -import org.junit.jupiter.api.TestInstance; -import org.slf4j.LoggerFactory; - -@QuarkusTest -@TestInstance(TestInstance.Lifecycle.PER_CLASS) -@TestProfile(PolarisServiceImplIntegrationTest.Profile.class) -public class PolarisServiceImplIntegrationTest { - private static final int MAX_IDENTIFIER_LENGTH = 256; - - // TODO: Add a test-only hook that fully clobbers all persistence state so we can have a fresh - // slate on every test case; otherwise, leftover state from one test from failures will interfere - // with other test cases. - - @Inject PolarisIntegrationTestHelper testHelper; - - @BeforeAll - public void setUp(TestInfo testInfo) { - testHelper.setUp(testInfo); - } - - @AfterAll - public void tearDown() { - testHelper.tearDown(); - } - - @AfterEach - public void after() { - try (Response response = newRequest("http://localhost:%d/api/management/v1/catalogs").get()) { - response - .readEntity(Catalogs.class) - .getCatalogs() - .forEach( - catalog -> { - // clean up the catalog before we try to drop it - - // delete all the namespaces - try (Response res = - newRequest( - "http://localhost:%d/api/catalog/v1/" - + catalog.getName() - + "/namespaces") - .get()) { - if (res.getStatus() != Response.Status.OK.getStatusCode()) { - LoggerFactory.getLogger(getClass()) - .warn( - "Unable to list namespaces in catalog {}: {}", - catalog.getName(), - res.readEntity(String.class)); - } else { - res.readEntity(ListNamespacesResponse.class) - .namespaces() - .forEach( - namespace -> { - newRequest( - "http://localhost:%d/api/catalog/v1/" - + catalog.getName() - + "/namespaces/" - + RESTUtil.encodeNamespace(namespace)) - .delete() - .close(); - }); - } - } - - // delete all the catalog roles except catalog_admin - try (Response res = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/" - + catalog.getName() - + "/catalog-roles") - .get()) { - if (res.getStatus() != Response.Status.OK.getStatusCode()) { - LoggerFactory.getLogger(getClass()) - .warn( - "Unable to list catalog roles for catalog {}: {}", - catalog.getName(), - res.readEntity(String.class)); - return; - } - res.readEntity(CatalogRoles.class).getRoles().stream() - .filter(cr -> !cr.getName().equals("catalog_admin")) - .forEach( - cr -> - newRequest( - "http://localhost:%d/api/management/v1/catalogs/" - + catalog.getName() - + "/catalog-roles/" - + cr.getName()) - .delete() - .close()); - } - - Response deleteResponse = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/" + catalog.getName()) - .delete(); - if (deleteResponse.getStatus() != Response.Status.NO_CONTENT.getStatusCode()) { - LoggerFactory.getLogger(getClass()) - .warn( - "Unable to delete catalog {}: {}", - catalog.getName(), - deleteResponse.readEntity(String.class)); - } - deleteResponse.close(); - }); - } - try (Response response = newRequest("http://localhost:%d/api/management/v1/principals").get()) { - response.readEntity(Principals.class).getPrincipals().stream() - .filter( - principal -> - !principal.getName().equals(PolarisEntityConstants.getRootPrincipalName())) - .forEach( - principal -> { - newRequest( - "http://localhost:%d/api/management/v1/principals/" + principal.getName()) - .delete() - .close(); - }); - } - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles").get()) { - response.readEntity(PrincipalRoles.class).getRoles().stream() - .filter( - principalRole -> - !principalRole - .getName() - .equals(PolarisEntityConstants.getNameOfPrincipalServiceAdminRole())) - .forEach( - principalRole -> { - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" - + principalRole.getName()) - .delete() - .close(); - }); - } - } - - @Test - public void testCatalogSerializing() throws IOException { - CatalogProperties props = new CatalogProperties("s3://my-old-bucket/path/to/data"); - props.put("prop1", "propval"); - PolarisCatalog catalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName("my_catalog") - .setProperties(props) - .setStorageConfigInfo( - AwsStorageConfigInfo.builder() - .setRoleArn("arn:aws:iam::123456789012:role/my-role") - .setExternalId("externalId") - .setUserArn("userArn") - .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) - .setAllowedLocations(List.of("s3://my-old-bucket/path/to/data")) - .build()) - .build(); - - ObjectMapper mapper = new ObjectMapper(); - String json = mapper.writeValueAsString(catalog); - System.out.println(json); - Catalog deserialized = mapper.readValue(json, Catalog.class); - assertThat(deserialized).isInstanceOf(PolarisCatalog.class); - } - - @Test - public void testListCatalogs() { - try (Response response = newRequest("http://localhost:%d/api/management/v1/catalogs").get()) { - assertThat(response) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus) - .extracting(r -> r.readEntity(Catalogs.class)) - .returns( - List.of(), - l -> - l.getCatalogs().stream() - .filter(c -> !c.getName().equalsIgnoreCase("ROOT")) - .toList()); - } - } - - @Test - public void testListCatalogsUnauthorized() { - Principal principal = new Principal("a_new_user"); - String newToken = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") - .post(Entity.json(principal))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - PrincipalWithCredentials creds = response.readEntity(PrincipalWithCredentials.class); - newToken = - TokenUtils.getTokenFromSecrets( - testHelper.client, - testHelper.localPort, - creds.getCredentials().getClientId(), - creds.getCredentials().getClientSecret(), - testHelper.realm); - } - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", newToken).get()) { - assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); - } - } - - @Test - public void testCreateCatalog() { - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post( - Entity.json( - "{\"catalog\":{\"type\":\"INTERNAL\",\"name\":\"my-catalog\",\"properties\":{\"default-base-location\":\"s3://my-bucket/path/to/data\"},\"storageConfigInfo\":{\"storageType\":\"S3\",\"roleArn\":\"arn:aws:iam::123456789012:role/my-role\",\"externalId\":\"externalId\",\"userArn\":\"userArn\",\"allowedLocations\":[\"s3://my-old-bucket/path/to/data\"]}}}"))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - - // 204 Successful delete - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/my-catalog").delete()) { - assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } - } - - @Test - public void testCreateCatalogWithInvalidName() { - AwsStorageConfigInfo awsConfigModel = - AwsStorageConfigInfo.builder() - .setRoleArn("arn:aws:iam::123456789012:role/my-role") - .setExternalId("externalId") - .setUserArn("userArn") - .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) - .setAllowedLocations(List.of("s3://my-old-bucket/path/to/data")) - .build(); - - String goodName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH, true, true); - - Catalog catalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName(goodName) - .setProperties(new CatalogProperties("s3://my-bucket/path/to/data")) - .setStorageConfigInfo(awsConfigModel) - .build(); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(testHelper.objectMapper.writeValueAsString(catalog)))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - - String longInvalidName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH + 1, true, true); - List invalidCatalogNames = - Arrays.asList( - longInvalidName, - "", - "system$catalog1", - "SYSTEM$TestCatalog", - "System$test_catalog", - " SysTeM$ test catalog"); - - for (String invalidCatalogName : invalidCatalogNames) { - catalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName(invalidCatalogName) - .setProperties(new CatalogProperties("s3://my-bucket/path/to/data")) - .setStorageConfigInfo(awsConfigModel) - .build(); - - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(testHelper.objectMapper.writeValueAsString(catalog)))) { - assertThat(response) - .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); - assertThat(response.hasEntity()).isTrue(); - ErrorResponse errorResponse = response.readEntity(ErrorResponse.class); - assertThat(errorResponse.message()).contains("Invalid value:"); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } - } - - @Test - public void testCreateCatalogWithAzureStorageConfig() { - AzureStorageConfigInfo azureConfigInfo = - AzureStorageConfigInfo.builder() - .setConsentUrl("https://consent.url") - .setMultiTenantAppName("myappname") - .setTenantId("tenantId") - .setStorageType(StorageConfigInfo.StorageTypeEnum.AZURE) - .build(); - Catalog catalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName("my-catalog") - .setProperties( - new CatalogProperties( - "abfss://polaris@polarisdev.dfs.core.windows.net/path/to/my/data/")) - .setStorageConfigInfo(azureConfigInfo) - .build(); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(new CreateCatalogRequest(catalog)))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/my-catalog").get()) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - Catalog catResponse = response.readEntity(Catalog.class); - assertThat(catResponse.getStorageConfigInfo()) - .isInstanceOf(AzureStorageConfigInfo.class) - .hasFieldOrPropertyWithValue("consentUrl", "https://consent.url") - .hasFieldOrPropertyWithValue("multiTenantAppName", "myappname") - .hasFieldOrPropertyWithValue( - "allowedLocations", - List.of("abfss://polaris@polarisdev.dfs.core.windows.net/path/to/my/data/")); - } - } - - @Test - public void testCreateCatalogWithGcpStorageConfig() { - GcpStorageConfigInfo gcpConfigModel = - GcpStorageConfigInfo.builder() - .setGcsServiceAccount("my-sa") - .setStorageType(StorageConfigInfo.StorageTypeEnum.GCS) - .build(); - Catalog catalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName("my-catalog") - .setProperties(new CatalogProperties("gs://my-bucket/path/to/data")) - .setStorageConfigInfo(gcpConfigModel) - .build(); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(new CreateCatalogRequest(catalog)))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/my-catalog").get()) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - Catalog catResponse = response.readEntity(Catalog.class); - assertThat(catResponse.getStorageConfigInfo()) - .isInstanceOf(GcpStorageConfigInfo.class) - .hasFieldOrPropertyWithValue("gcsServiceAccount", "my-sa") - .hasFieldOrPropertyWithValue("allowedLocations", List.of("gs://my-bucket/path/to/data")); - } - } - - @Test - public void testCreateCatalogWithNullBaseLocation() { - AwsStorageConfigInfo awsConfigModel = - AwsStorageConfigInfo.builder() - .setRoleArn("arn:aws:iam::123456789012:role/my-role") - .setExternalId("externalId") - .setUserArn("userArn") - .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) - .setAllowedLocations(List.of("s3://my-old-bucket/path/to/data")) - .build(); - ObjectMapper mapper = new ObjectMapper(); - JsonNode storageConfig = mapper.valueToTree(awsConfigModel); - ObjectNode catalogNode = mapper.createObjectNode(); - catalogNode.set("storageConfigInfo", storageConfig); - catalogNode.put("name", "my-catalog"); - catalogNode.put("type", "INTERNAL"); - catalogNode.set("properties", mapper.createObjectNode()); - ObjectNode requestNode = mapper.createObjectNode(); - requestNode.set("catalog", catalogNode); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(requestNode))) { - assertThat(response) - .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); - } - } - - @Test - public void testCreateCatalogWithoutProperties() { - AwsStorageConfigInfo awsConfigModel = - AwsStorageConfigInfo.builder() - .setRoleArn("arn:aws:iam::123456789012:role/my-role") - .setExternalId("externalId") - .setUserArn("userArn") - .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) - .setAllowedLocations(List.of("s3://my-old-bucket/path/to/data")) - .build(); - ObjectMapper mapper = new ObjectMapper(); - JsonNode storageConfig = mapper.valueToTree(awsConfigModel); - ObjectNode catalogNode = mapper.createObjectNode(); - catalogNode.set("storageConfigInfo", storageConfig); - catalogNode.put("name", "my-catalog"); - catalogNode.put("type", "INTERNAL"); - ObjectNode requestNode = mapper.createObjectNode(); - requestNode.set("catalog", catalogNode); - - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", testHelper.adminToken) - .post(Entity.json(requestNode))) { - assertThat(response) - .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); - ErrorResponse error = response.readEntity(ErrorResponse.class); - assertThat(error) - .isNotNull() - .returns( - "Invalid value: createCatalog.arg0.catalog.properties: must not be null", - ErrorResponse::message); - } - } - - @Test - public void testCreateCatalogWithoutStorageConfig() throws JsonProcessingException { - String catalogString = - "{\"catalog\": {\"type\":\"INTERNAL\",\"name\":\"my-catalog\",\"properties\":{\"default-base-location\":\"s3://my-bucket/path/to/data\"}}}"; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", testHelper.adminToken) - .post(Entity.json(catalogString))) { - assertThat(response) - .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); - ErrorResponse error = response.readEntity(ErrorResponse.class); - assertThat(error) - .isNotNull() - .returns( - "Invalid value: createCatalog.arg0.catalog.storageConfigInfo: must not be null", - ErrorResponse::message); - } - } - - @Test - public void testCreateCatalogWithUnparsableJson() throws JsonProcessingException { - String catalogString = "{\"catalog\": {{\"bad data}"; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", testHelper.adminToken) - .post(Entity.json(catalogString))) { - assertThat(response) - .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); - ErrorResponse error = response.readEntity(ErrorResponse.class); - assertThat(error) - .isNotNull() - .extracting(ErrorResponse::message) - .isEqualTo("HTTP 400 Bad Request"); - } - } - - @Test - public void testCreateCatalogWithDisallowedStorageConfig() throws JsonProcessingException { - FileStorageConfigInfo fileStorage = - FileStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.FILE) - .setAllowedLocations(List.of("file://")) - .build(); - String catalogName = "my-external-catalog"; - Catalog catalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName(catalogName) - .setProperties(new CatalogProperties("file:///tmp/path/to/data")) - .setStorageConfigInfo(fileStorage) - .build(); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", testHelper.adminToken) - .post(Entity.json(new CreateCatalogRequest(catalog)))) { - assertThat(response) - .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); - ErrorResponse error = response.readEntity(ErrorResponse.class); - assertThat(error) - .isNotNull() - .returns("Unsupported storage type: FILE", ErrorResponse::message); - } - } - - @Test - public void testUpdateCatalogWithDisallowedStorageConfig() throws JsonProcessingException { - AwsStorageConfigInfo awsConfigModel = - AwsStorageConfigInfo.builder() - .setRoleArn("arn:aws:iam::123456789012:role/my-role") - .setExternalId("externalId") - .setUserArn("userArn") - .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) - .setAllowedLocations(List.of("s3://my-old-bucket/path/to/data")) - .build(); - String catalogName = "mycatalog"; - Catalog catalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName(catalogName) - .setProperties(new CatalogProperties("s3://bucket/path/to/data")) - .setStorageConfigInfo(awsConfigModel) - .build(); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", testHelper.adminToken) - .post(Entity.json(new CreateCatalogRequest(catalog)))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - - // 200 successful GET after creation - Catalog fetchedCatalog = null; - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/" + catalogName, - testHelper.adminToken) - .get()) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - fetchedCatalog = response.readEntity(Catalog.class); - - assertThat(fetchedCatalog.getName()).isEqualTo(catalogName); - assertThat(fetchedCatalog.getProperties().toMap()) - .isEqualTo(Map.of("default-base-location", "s3://bucket/path/to/data")); - assertThat(fetchedCatalog.getEntityVersion()).isGreaterThan(0); - } - - FileStorageConfigInfo fileStorage = - FileStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.FILE) - .setAllowedLocations(List.of("file://")) - .build(); - UpdateCatalogRequest updateRequest = - new UpdateCatalogRequest( - fetchedCatalog.getEntityVersion(), - Map.of("default-base-location", "file:///tmp/path/to/data/"), - fileStorage); - - // failure to update - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/" + catalogName, - testHelper.adminToken) - .put(Entity.json(updateRequest))) { - assertThat(response) - .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); - ErrorResponse error = response.readEntity(ErrorResponse.class); - - assertThat(error).returns("Unsupported storage type: FILE", ErrorResponse::message); - } - } - - @Test - public void testCreateExternalCatalog() { - AwsStorageConfigInfo awsConfigModel = - AwsStorageConfigInfo.builder() - .setRoleArn("arn:aws:iam::123456789012:role/my-role") - .setExternalId("externalId") - .setUserArn("userArn") - .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) - .setAllowedLocations(List.of("s3://my-old-bucket/path/to/data")) - .build(); - String catalogName = "my-external-catalog"; - String remoteUrl = "http://localhost:8080"; - Catalog catalog = - ExternalCatalog.builder() - .setType(Catalog.TypeEnum.EXTERNAL) - .setName(catalogName) - .setRemoteUrl(remoteUrl) - .setProperties(new CatalogProperties("s3://my-bucket/path/to/data")) - .setStorageConfigInfo(awsConfigModel) - .build(); - createCatalog(catalog); - - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + catalogName).get()) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - Catalog fetchedCatalog = response.readEntity(Catalog.class); - assertThat(fetchedCatalog) - .isNotNull() - .isInstanceOf(ExternalCatalog.class) - .asInstanceOf(InstanceOfAssertFactories.type(ExternalCatalog.class)) - .returns(remoteUrl, ExternalCatalog::getRemoteUrl) - .extracting(ExternalCatalog::getStorageConfigInfo) - .isNotNull() - .isInstanceOf(AwsStorageConfigInfo.class) - .asInstanceOf(InstanceOfAssertFactories.type(AwsStorageConfigInfo.class)) - .returns("arn:aws:iam::123456789012:role/my-role", AwsStorageConfigInfo::getRoleArn); - } - - // 204 Successful delete - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + catalogName).delete()) { - assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } - } - - @Test - public void testCreateCatalogWithoutDefaultLocation() { - AwsStorageConfigInfo awsConfigModel = - AwsStorageConfigInfo.builder() - .setRoleArn("arn:aws:iam::123456789012:role/my-role") - .setExternalId("externalId") - .setUserArn("userArn") - .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) - .setAllowedLocations(List.of("s3://my-old-bucket/path/to/data")) - .build(); - ObjectMapper mapper = new ObjectMapper(); - JsonNode storageConfig = mapper.valueToTree(awsConfigModel); - ObjectNode catalogNode = mapper.createObjectNode(); - catalogNode.set("storageConfigInfo", storageConfig); - catalogNode.put("name", "my-catalog"); - catalogNode.put("type", "INTERNAL"); - ObjectNode properties = mapper.createObjectNode(); - properties.set("default-base-location", mapper.nullNode()); - catalogNode.set("properties", properties); - ObjectNode requestNode = mapper.createObjectNode(); - requestNode.set("catalog", catalogNode); - - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(requestNode))) { - assertThat(response) - .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); - } - } - - @Test - public void serialization() throws JsonProcessingException { - CatalogProperties properties = new CatalogProperties("s3://my-bucket/path/to/data"); - ObjectMapper mapper = new ObjectMapper(); - CatalogProperties translated = mapper.convertValue(properties, CatalogProperties.class); - assertThat(translated.toMap()) - .containsEntry("default-base-location", "s3://my-bucket/path/to/data"); - } - - @Test - public void testCreateAndUpdateAzureCatalog() { - StorageConfigInfo storageConfig = - new AzureStorageConfigInfo("azure:tenantid:12345", StorageConfigInfo.StorageTypeEnum.AZURE); - Catalog catalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName("myazurecatalog") - .setStorageConfigInfo(storageConfig) - .setProperties(new CatalogProperties("abfss://container1@acct1.dfs.core.windows.net/")) - .build(); - - // 200 Successful create - createCatalog(catalog); - - // 200 successful GET after creation - Catalog fetchedCatalog = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/myazurecatalog").get()) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - fetchedCatalog = response.readEntity(Catalog.class); - - assertThat(fetchedCatalog.getName()).isEqualTo("myazurecatalog"); - assertThat(fetchedCatalog.getProperties().toMap()) - .isEqualTo( - Map.of("default-base-location", "abfss://container1@acct1.dfs.core.windows.net/")); - assertThat(fetchedCatalog.getEntityVersion()).isGreaterThan(0); - } - - StorageConfigInfo modifiedStorageConfig = - new AzureStorageConfigInfo("azure:tenantid:22222", StorageConfigInfo.StorageTypeEnum.AZURE); - UpdateCatalogRequest badUpdateRequest = - new UpdateCatalogRequest( - fetchedCatalog.getEntityVersion(), - Map.of("default-base-location", "abfss://newcontainer@acct1.dfs.core.windows.net/"), - modifiedStorageConfig); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/myazurecatalog") - .put(Entity.json(badUpdateRequest))) { - assertThat(response) - .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); - ErrorResponse error = response.readEntity(ErrorResponse.class); - assertThat(error) - .isNotNull() - .extracting(ErrorResponse::message) - .asString() - .startsWith("Cannot modify"); - } - - UpdateCatalogRequest updateRequest = - new UpdateCatalogRequest( - fetchedCatalog.getEntityVersion(), - Map.of("default-base-location", "abfss://newcontainer@acct1.dfs.core.windows.net/"), - storageConfig); - - // 200 successful update - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/myazurecatalog") - .put(Entity.json(updateRequest))) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - fetchedCatalog = response.readEntity(Catalog.class); - - assertThat(fetchedCatalog.getProperties().toMap()) - .isEqualTo( - Map.of("default-base-location", "abfss://newcontainer@acct1.dfs.core.windows.net/")); - } - - // 204 Successful delete - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/myazurecatalog").delete()) { - assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } - } - - @Test - public void testCreateListUpdateAndDeleteCatalog() { - StorageConfigInfo storageConfig = - new AwsStorageConfigInfo( - "arn:aws:iam::123456789011:role/role1", StorageConfigInfo.StorageTypeEnum.S3); - Catalog catalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName("mycatalog") - .setStorageConfigInfo(storageConfig) - .setProperties(new CatalogProperties("s3://bucket1/")) - .build(); - - createCatalog(catalog); - - // Second attempt to create the same entity should fail with CONFLICT. - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(new CreateCatalogRequest(catalog)))) { - assertThat(response).returns(Response.Status.CONFLICT.getStatusCode(), Response::getStatus); - } - - // 200 successful GET after creation - Catalog fetchedCatalog = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").get()) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - fetchedCatalog = response.readEntity(Catalog.class); - - assertThat(fetchedCatalog.getName()).isEqualTo("mycatalog"); - assertThat(fetchedCatalog.getProperties().toMap()) - .isEqualTo(Map.of("default-base-location", "s3://bucket1/")); - assertThat(fetchedCatalog.getEntityVersion()).isGreaterThan(0); - } - - // Should list the catalog. - try (Response response = newRequest("http://localhost:%d/api/management/v1/catalogs").get()) { - assertThat(response) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus) - .extracting(r -> r.readEntity(Catalogs.class)) - .extracting(Catalogs::getCatalogs) - .asInstanceOf(InstanceOfAssertFactories.list(Catalog.class)) - .filteredOn(cat -> !cat.getName().equalsIgnoreCase("ROOT")) - .satisfiesExactly(cat -> assertThat(cat).returns("mycatalog", Catalog::getName)); - } - - // Reject update of fields that can't be currently updated - StorageConfigInfo modifiedStorageConfig = - new AwsStorageConfigInfo( - "arn:aws:iam::123456789011:role/newrole", StorageConfigInfo.StorageTypeEnum.S3); - UpdateCatalogRequest badUpdateRequest = - new UpdateCatalogRequest( - fetchedCatalog.getEntityVersion(), - Map.of("default-base-location", "s3://newbucket/"), - modifiedStorageConfig); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog") - .put(Entity.json(badUpdateRequest))) { - assertThat(response) - .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); - ErrorResponse error = response.readEntity(ErrorResponse.class); - assertThat(error) - .isNotNull() - .extracting(ErrorResponse::message) - .asString() - .startsWith("Cannot modify"); - } - - UpdateCatalogRequest updateRequest = - new UpdateCatalogRequest( - fetchedCatalog.getEntityVersion(), - Map.of("default-base-location", "s3://newbucket/"), - storageConfig); - - // 200 successful update - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog") - .put(Entity.json(updateRequest))) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - fetchedCatalog = response.readEntity(Catalog.class); - - assertThat(fetchedCatalog.getProperties().toMap()) - .isEqualTo(Map.of("default-base-location", "s3://newbucket/")); - } - - // 200 GET after update should show new properties - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").get()) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - fetchedCatalog = response.readEntity(Catalog.class); - - assertThat(fetchedCatalog.getProperties().toMap()) - .isEqualTo(Map.of("default-base-location", "s3://newbucket/")); - } - - // 204 Successful delete - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").delete()) { - assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } - - // NOT_FOUND after deletion - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").get()) { - assertThat(response).returns(Response.Status.NOT_FOUND.getStatusCode(), Response::getStatus); - } - - // Empty list - try (Response response = newRequest("http://localhost:%d/api/management/v1/catalogs").get()) { - assertThat(response) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus) - .extracting(r -> r.readEntity(Catalogs.class)) - .returns( - List.of(), - c -> - c.getCatalogs().stream() - .filter(cat -> !cat.getName().equalsIgnoreCase("ROOT")) - .toList()); - } - } - - private Invocation.Builder newRequest(String url, String token) { - return testHelper - .client - .target(String.format(url, testHelper.localPort)) - .request("application/json") - .header("Authorization", "Bearer " + token) - .header(REALM_PROPERTY_KEY, testHelper.realm); - } - - private Invocation.Builder newRequest(String url) { - return newRequest(url, testHelper.adminToken); - } - - @Test - public void testGetCatalogNotFound() { - // there's no catalog yet. Expect 404 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").get()) { - assertThat(response).returns(Response.Status.NOT_FOUND.getStatusCode(), Response::getStatus); - } - } - - @Test - public void testGetCatalogInvalidName() { - String longInvalidName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH + 1, true, true); - List invalidCatalogNames = - Arrays.asList( - longInvalidName, - "system$catalog1", - "SYSTEM$TestCatalog", - "System$test_catalog", - " SysTeM$ test catalog"); - - for (String invalidCatalogName : invalidCatalogNames) { - // there's no catalog yet. Expect 404 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + invalidCatalogName) - .get()) { - assertThat(response) - .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); - assertThat(response.hasEntity()).isTrue(); - ErrorResponse errorResponse = response.readEntity(ErrorResponse.class); - assertThat(errorResponse.message()).contains("Invalid value:"); - } - } - } - - @Test - public void testCatalogRoleInvalidName() { - Catalog catalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName("mycatalog1") - .setProperties(new CatalogProperties("s3://required/base/location")) - .setStorageConfigInfo( - new AwsStorageConfigInfo( - "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) - .build(); - createCatalog(catalog); - - String longInvalidName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH + 1, true, true); - List invalidCatalogRoleNames = - Arrays.asList( - longInvalidName, - "system$catalog1", - "SYSTEM$TestCatalog", - "System$test_catalog", - " SysTeM$ test catalog"); - - for (String invalidCatalogRoleName : invalidCatalogRoleNames) { - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/" - + invalidCatalogRoleName) - .get()) { - - assertThat(response) - .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); - assertThat(response.hasEntity()).isTrue(); - ErrorResponse errorResponse = response.readEntity(ErrorResponse.class); - assertThat(errorResponse.message()).contains("Invalid value:"); - } - } - } - - @Test - public void testListPrincipalsUnauthorized() { - Principal principal = new Principal("new_admin"); - String newToken = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") - .post(Entity.json(principal))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - PrincipalWithCredentials creds = response.readEntity(PrincipalWithCredentials.class); - newToken = - TokenUtils.getTokenFromSecrets( - testHelper.client, - testHelper.localPort, - creds.getCredentials().getClientId(), - creds.getCredentials().getClientSecret(), - testHelper.realm); - } - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals", newToken).get()) { - assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); - } - } - - @Test - public void testCreatePrincipalAndRotateCredentials() { - Principal principal = - Principal.builder() - .setName("myprincipal") - .setProperties(Map.of("custom-tag", "foo")) - .build(); - - PrincipalWithCredentialsCredentials creds = null; - Principal returnedPrincipal = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") - .post(Entity.json(new CreatePrincipalRequest(principal, true)))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - PrincipalWithCredentials parsed = response.readEntity(PrincipalWithCredentials.class); - creds = parsed.getCredentials(); - returnedPrincipal = parsed.getPrincipal(); - } - assertThat(creds.getClientId()).isEqualTo(returnedPrincipal.getClientId()); - - String oldClientId = creds.getClientId(); - String oldSecret = creds.getClientSecret(); - - // Now rotate the credentials. First, if we try to just use the adminToken to rotate the - // newly created principal's credentials, we should fail; rotateCredentials is only - // a "self" privilege that even admins can't inherit. - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal/rotate") - .post(Entity.json(""))) { - assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); - } - - // Get a fresh token associate with the principal itself. - String newPrincipalToken = - TokenUtils.getTokenFromSecrets( - testHelper.client, testHelper.localPort, oldClientId, oldSecret, testHelper.realm); - - // Any call should initially fail with error indicating that rotation is needed. - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principals/myprincipal", newPrincipalToken) - .get()) { - assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); - ErrorResponse error = response.readEntity(ErrorResponse.class); - assertThat(error) - .isNotNull() - .extracting(ErrorResponse::message) - .asString() - .contains("PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_STATE"); - } - - // Now try to rotate using the principal's token. - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principals/myprincipal/rotate", - newPrincipalToken) - .post(Entity.json(""))) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - PrincipalWithCredentials parsed = response.readEntity(PrincipalWithCredentials.class); - creds = parsed.getCredentials(); - returnedPrincipal = parsed.getPrincipal(); - } - assertThat(creds.getClientId()).isEqualTo(returnedPrincipal.getClientId()); - - // ClientId shouldn't change - assertThat(creds.getClientId()).isEqualTo(oldClientId); - assertThat(creds.getClientSecret()).isNotEqualTo(oldSecret); - - // TODO: Test the validity of the old secret for getting tokens, here and then after a second - // rotation that makes the old secret fall off retention. - } - - @Test - public void testCreateListUpdateAndDeletePrincipal() { - Principal principal = - Principal.builder() - .setName("myprincipal") - .setProperties(Map.of("custom-tag", "foo")) - .build(); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") - .post(Entity.json(new CreatePrincipalRequest(principal, null)))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - - // Second attempt to create the same entity should fail with CONFLICT. - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") - .post(Entity.json(new CreatePrincipalRequest(principal, false)))) { - assertThat(response).returns(Response.Status.CONFLICT.getStatusCode(), Response::getStatus); - } - - // 200 successful GET after creation - Principal fetchedPrincipal = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal").get()) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - fetchedPrincipal = response.readEntity(Principal.class); - - assertThat(fetchedPrincipal.getName()).isEqualTo("myprincipal"); - assertThat(fetchedPrincipal.getProperties()).isEqualTo(Map.of("custom-tag", "foo")); - assertThat(fetchedPrincipal.getEntityVersion()).isGreaterThan(0); - } - - // Should list the principal. - try (Response response = newRequest("http://localhost:%d/api/management/v1/principals").get()) { - assertThat(response) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus) - .extracting(r -> r.readEntity(Principals.class)) - .extracting(Principals::getPrincipals) - .asInstanceOf(InstanceOfAssertFactories.list(Principal.class)) - .anySatisfy(pr -> assertThat(pr).returns("myprincipal", Principal::getName)); - } - - UpdatePrincipalRequest updateRequest = - new UpdatePrincipalRequest( - fetchedPrincipal.getEntityVersion(), Map.of("custom-tag", "updated")); - - // 200 successful update - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal") - .put(Entity.json(updateRequest))) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - fetchedPrincipal = response.readEntity(Principal.class); - - assertThat(fetchedPrincipal.getProperties()).isEqualTo(Map.of("custom-tag", "updated")); - } - - // 200 GET after update should show new properties - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal").get()) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - fetchedPrincipal = response.readEntity(Principal.class); - - assertThat(fetchedPrincipal.getProperties()).isEqualTo(Map.of("custom-tag", "updated")); - } - - // 204 Successful delete - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal").delete()) { - assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } - - // NOT_FOUND after deletion - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal").get()) { - assertThat(response).returns(Response.Status.NOT_FOUND.getStatusCode(), Response::getStatus); - } - - // Empty list - try (Response response = newRequest("http://localhost:%d/api/management/v1/principals").get()) { - assertThat(response) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus) - .extracting(r -> r.readEntity(Principals.class)) - .extracting(Principals::getPrincipals) - .asInstanceOf(InstanceOfAssertFactories.list(Principal.class)) - .noneSatisfy(pr -> assertThat(pr).returns("myprincipal", Principal::getName)); - } - } - - @Test - public void testCreatePrincipalWithInvalidName() { - String goodName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH, true, true); - Principal principal = - Principal.builder() - .setName(goodName) - .setProperties(Map.of("custom-tag", "good_principal")) - .build(); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") - .post(Entity.json(new CreatePrincipalRequest(principal, null)))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - - String longInvalidName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH + 1, true, true); - List invalidPrincipalNames = - Arrays.asList( - longInvalidName, - "", - "system$principal1", - "SYSTEM$TestPrincipal", - "System$test_principal", - " SysTeM$ principal"); - - for (String invalidPrincipalName : invalidPrincipalNames) { - principal = - Principal.builder() - .setName(invalidPrincipalName) - .setProperties(Map.of("custom-tag", "bad_principal")) - .build(); - - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") - .post(Entity.json(new CreatePrincipalRequest(principal, false)))) { - assertThat(response) - .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); - assertThat(response.hasEntity()).isTrue(); - ErrorResponse errorResponse = response.readEntity(ErrorResponse.class); - assertThat(errorResponse.message()).contains("Invalid value:"); - } - } - } - - @Test - public void testGetPrincipalWithInvalidName() { - String longInvalidName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH + 1, true, true); - List invalidPrincipalNames = - Arrays.asList( - longInvalidName, - "system$principal1", - "SYSTEM$TestPrincipal", - "System$test_principal", - " SysTeM$ principal"); - - for (String invalidPrincipalName : invalidPrincipalNames) { - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/" + invalidPrincipalName) - .get()) { - assertThat(response) - .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); - assertThat(response.hasEntity()).isTrue(); - ErrorResponse errorResponse = response.readEntity(ErrorResponse.class); - assertThat(errorResponse.message()).contains("Invalid value:"); - } - } - } - - @Test - public void testCreateListUpdateAndDeletePrincipalRole() { - PrincipalRole principalRole = - new PrincipalRole("myprincipalrole", Map.of("custom-tag", "foo"), 0L, 0L, 1); - createPrincipalRole(principalRole); - - // Second attempt to create the same entity should fail with CONFLICT. - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles") - .post(Entity.json(new CreatePrincipalRoleRequest(principalRole)))) { - - assertThat(response).returns(Response.Status.CONFLICT.getStatusCode(), Response::getStatus); - } - - // 200 successful GET after creation - PrincipalRole fetchedPrincipalRole = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole").get()) { - - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - fetchedPrincipalRole = response.readEntity(PrincipalRole.class); - - assertThat(fetchedPrincipalRole.getName()).isEqualTo("myprincipalrole"); - assertThat(fetchedPrincipalRole.getProperties()).isEqualTo(Map.of("custom-tag", "foo")); - assertThat(fetchedPrincipalRole.getEntityVersion()).isGreaterThan(0); - } - - // Should list the principalRole. - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles").get()) { - - assertThat(response) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus) - .extracting(r -> r.readEntity(PrincipalRoles.class)) - .extracting(PrincipalRoles::getRoles) - .asInstanceOf(InstanceOfAssertFactories.list(PrincipalRole.class)) - .anySatisfy(pr -> assertThat(pr).returns("myprincipalrole", PrincipalRole::getName)); - } - - UpdatePrincipalRoleRequest updateRequest = - new UpdatePrincipalRoleRequest( - fetchedPrincipalRole.getEntityVersion(), Map.of("custom-tag", "updated")); - - // 200 successful update - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole") - .put(Entity.json(updateRequest))) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - fetchedPrincipalRole = response.readEntity(PrincipalRole.class); - - assertThat(fetchedPrincipalRole.getProperties()).isEqualTo(Map.of("custom-tag", "updated")); - } - - // 200 GET after update should show new properties - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole").get()) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - fetchedPrincipalRole = response.readEntity(PrincipalRole.class); - - assertThat(fetchedPrincipalRole.getProperties()).isEqualTo(Map.of("custom-tag", "updated")); - } - - // 204 Successful delete - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole") - .delete()) { - - assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } - - // NOT_FOUND after deletion - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole").get()) { - - assertThat(response).returns(Response.Status.NOT_FOUND.getStatusCode(), Response::getStatus); - } - - // Empty list - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles").get()) { - - assertThat(response) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus) - .extracting(r -> r.readEntity(PrincipalRoles.class)) - .extracting(PrincipalRoles::getRoles) - .asInstanceOf(InstanceOfAssertFactories.list(PrincipalRole.class)) - .noneSatisfy(pr -> assertThat(pr).returns("myprincipalrole", PrincipalRole::getName)); - } - } - - @Test - public void testCreatePrincipalRoleInvalidName() { - String goodName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH, true, true); - PrincipalRole principalRole = - new PrincipalRole(goodName, Map.of("custom-tag", "good_principal_role"), 0L, 0L, 1); - createPrincipalRole(principalRole); - - String longInvalidName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH + 1, true, true); - List invalidPrincipalRoleNames = - Arrays.asList( - longInvalidName, - "", - "system$principalrole1", - "SYSTEM$TestPrincipalRole", - "System$test_principal_role", - " SysTeM$ principal role"); - - for (String invalidPrincipalRoleName : invalidPrincipalRoleNames) { - principalRole = - new PrincipalRole( - invalidPrincipalRoleName, Map.of("custom-tag", "bad_principal_role"), 0L, 0L, 1); - - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles") - .post(Entity.json(new CreatePrincipalRoleRequest(principalRole)))) { - assertThat(response) - .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); - assertThat(response.hasEntity()).isTrue(); - ErrorResponse errorResponse = response.readEntity(ErrorResponse.class); - assertThat(errorResponse.message()).contains("Invalid value:"); - } - } - } - - @Test - public void testGetPrincipalRoleInvalidName() { - String longInvalidName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH + 1, true, true); - List invalidPrincipalRoleNames = - Arrays.asList( - longInvalidName, - "system$principalrole1", - "SYSTEM$TestPrincipalRole", - "System$test_principal_role", - " SysTeM$ principal role"); - - for (String invalidPrincipalRoleName : invalidPrincipalRoleNames) { - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" - + invalidPrincipalRoleName) - .get()) { - assertThat(response) - .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); - assertThat(response.hasEntity()).isTrue(); - ErrorResponse errorResponse = response.readEntity(ErrorResponse.class); - assertThat(errorResponse.message()).contains("Invalid value:"); - } - } - } - - @Test - public void testCreateListUpdateAndDeleteCatalogRole() { - Catalog catalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName("mycatalog1") - .setProperties(new CatalogProperties("s3://required/base/location")) - .setStorageConfigInfo( - new AwsStorageConfigInfo( - "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) - .build(); - createCatalog(catalog); - - Catalog catalog2 = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName("mycatalog2") - .setStorageConfigInfo( - new AwsStorageConfigInfo( - "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) - .setProperties(new CatalogProperties("s3://required/base/other_location")) - .build(); - createCatalog(catalog2); - - CatalogRole catalogRole = - new CatalogRole("mycatalogrole", Map.of("custom-tag", "foo"), 0L, 0L, 1); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles") - .post(Entity.json(new CreateCatalogRoleRequest(catalogRole)))) { - - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - - // Second attempt to create the same entity should fail with CONFLICT. - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles") - .post(Entity.json(new CreateCatalogRoleRequest(catalogRole)))) { - - assertThat(response).returns(Response.Status.CONFLICT.getStatusCode(), Response::getStatus); - } - - // 200 successful GET after creation - CatalogRole fetchedCatalogRole = null; - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") - .get()) { - - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - fetchedCatalogRole = response.readEntity(CatalogRole.class); - - assertThat(fetchedCatalogRole.getName()).isEqualTo("mycatalogrole"); - assertThat(fetchedCatalogRole.getProperties()).isEqualTo(Map.of("custom-tag", "foo")); - assertThat(fetchedCatalogRole.getEntityVersion()).isGreaterThan(0); - } - - // Should list the catalogRole. - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles") - .get()) { - - assertThat(response) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus) - .extracting(r -> r.readEntity(CatalogRoles.class)) - .extracting(CatalogRoles::getRoles) - .asInstanceOf(InstanceOfAssertFactories.list(CatalogRole.class)) - .anySatisfy(cr -> assertThat(cr).returns("mycatalogrole", CatalogRole::getName)); - } - - // Empty list if listing in catalog2 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog2/catalog-roles") - .get()) { - - assertThat(response) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus) - .extracting(r -> r.readEntity(CatalogRoles.class)) - .extracting(CatalogRoles::getRoles) - .asInstanceOf(InstanceOfAssertFactories.list(CatalogRole.class)) - .satisfiesExactly( - cr -> - assertThat(cr) - .returns( - PolarisEntityConstants.getNameOfCatalogAdminRole(), - CatalogRole::getName)); - } - - UpdateCatalogRoleRequest updateRequest = - new UpdateCatalogRoleRequest( - fetchedCatalogRole.getEntityVersion(), Map.of("custom-tag", "updated")); - - // 200 successful update - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") - .put(Entity.json(updateRequest))) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - fetchedCatalogRole = response.readEntity(CatalogRole.class); - - assertThat(fetchedCatalogRole.getProperties()).isEqualTo(Map.of("custom-tag", "updated")); - } - - // 200 GET after update should show new properties - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") - .get()) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - fetchedCatalogRole = response.readEntity(CatalogRole.class); - - assertThat(fetchedCatalogRole.getProperties()).isEqualTo(Map.of("custom-tag", "updated")); - } - - // 204 Successful delete - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") - .delete()) { - - assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } - - // NOT_FOUND after deletion - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") - .get()) { - - assertThat(response).returns(Response.Status.NOT_FOUND.getStatusCode(), Response::getStatus); - } - - // Empty list - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles") - .get()) { - - assertThat(response) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus) - .extracting(r -> r.readEntity(CatalogRoles.class)) - .extracting(CatalogRoles::getRoles) - .asInstanceOf(InstanceOfAssertFactories.list(CatalogRole.class)) - .noneSatisfy(cr -> assertThat(cr).returns("mycatalogrole", CatalogRole::getName)); - } - - // 204 Successful delete mycatalog - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog1").delete()) { - - assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } - - // 204 Successful delete mycatalog2 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog2").delete()) { - - assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } - } - - @Test - public void testAssignListAndRevokePrincipalRoles() { - // Create two Principals - Principal principal1 = new Principal("myprincipal1"); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") - .post(Entity.json(new CreatePrincipalRequest(principal1, false)))) { - - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - - Principal principal2 = new Principal("myprincipal2"); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") - .post(Entity.json(new CreatePrincipalRequest(principal2, false)))) { - - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - - // One PrincipalRole - PrincipalRole principalRole = new PrincipalRole("myprincipalrole"); - createPrincipalRole(principalRole); - - // Assign the role to myprincipal1 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal1/principal-roles") - .put(Entity.json(principalRole))) { - - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - - // Should list myprincipalrole - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal1/principal-roles") - .get()) { - - assertThat(response) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus) - .extracting(r -> r.readEntity(PrincipalRoles.class)) - .extracting(PrincipalRoles::getRoles) - .asInstanceOf(InstanceOfAssertFactories.list(PrincipalRole.class)) - .hasSize(1) - .satisfiesExactly( - pr -> assertThat(pr).returns("myprincipalrole", PrincipalRole::getName)); - } - - // Should list myprincipal1 if listing assignees of myprincipalrole - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/myprincipalrole/principals") - .get()) { - - assertThat(response) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus) - .extracting(r -> r.readEntity(Principals.class)) - .extracting(Principals::getPrincipals) - .asInstanceOf(InstanceOfAssertFactories.list(Principal.class)) - .hasSize(1) - .satisfiesExactly(pr -> assertThat(pr).returns("myprincipal1", Principal::getName)); - } - - // Empty list if listing in principal2 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal2/principal-roles") - .get()) { - - assertThat(response) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus) - .extracting(r -> r.readEntity(PrincipalRoles.class)) - .returns(List.of(), PrincipalRoles::getRoles); - } - - // 204 Successful revoke - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principals/myprincipal1/principal-roles/myprincipalrole") - .delete()) { - - assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } - - // Empty list - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal1/principal-roles") - .get()) { - - assertThat(response) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus) - .extracting(r -> r.readEntity(PrincipalRoles.class)) - .returns(List.of(), PrincipalRoles::getRoles); - } - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/myprincipalrole/principals") - .get()) { - - assertThat(response) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus) - .extracting(r -> r.readEntity(Principals.class)) - .returns(List.of(), Principals::getPrincipals); - } - - // 204 Successful delete myprincipal1 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal1").delete()) { - - assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } - - // 204 Successful delete myprincipal2 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal2").delete()) { - - assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } - - // 204 Successful delete myprincipalrole - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole") - .delete()) { - - assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } - } - - @Test - public void testAssignListAndRevokeCatalogRoles() { - // Create two PrincipalRoles - PrincipalRole principalRole1 = new PrincipalRole("mypr1"); - createPrincipalRole(principalRole1); - - PrincipalRole principalRole2 = new PrincipalRole("mypr2"); - createPrincipalRole(principalRole2); - - // One CatalogRole - Catalog catalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName("mycatalog") - .setStorageConfigInfo( - new AwsStorageConfigInfo( - "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) - .setProperties(new CatalogProperties("s3://bucket1/")) - .build(); - createCatalog(catalog); - - CatalogRole catalogRole = new CatalogRole("mycr"); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog/catalog-roles") - .post(Entity.json(new CreateCatalogRoleRequest(catalogRole)))) { - - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - - // Create another one in a different catalog. - Catalog otherCatalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName("othercatalog") - .setProperties(new CatalogProperties("s3://path/to/data")) - .setStorageConfigInfo( - new AwsStorageConfigInfo( - "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) - .build(); - createCatalog(otherCatalog); - - CatalogRole otherCatalogRole = new CatalogRole("myothercr"); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/othercatalog/catalog-roles") - .post(Entity.json(new CreateCatalogRoleRequest(otherCatalogRole)))) { - - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - - // Assign both the roles to mypr1 - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr1/catalog-roles/mycatalog") - .put(Entity.json(new GrantCatalogRoleRequest(catalogRole)))) { - - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr1/catalog-roles/othercatalog") - .put(Entity.json(new GrantCatalogRoleRequest(otherCatalogRole)))) { - - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - - // Should list only mycr - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr1/catalog-roles/mycatalog") - .get()) { - - assertThat(response) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus) - .extracting(r -> r.readEntity(CatalogRoles.class)) - .extracting(CatalogRoles::getRoles) - .asInstanceOf(InstanceOfAssertFactories.list(CatalogRole.class)) - .hasSize(1) - .satisfiesExactly(cr -> assertThat(cr).returns("mycr", CatalogRole::getName)); - } - - // Should list mypr1 if listing assignees of mycr - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog/catalog-roles/mycr/principal-roles") - .get()) { - - assertThat(response) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus) - .extracting(r -> r.readEntity(PrincipalRoles.class)) - .extracting(PrincipalRoles::getRoles) - .asInstanceOf(InstanceOfAssertFactories.list(PrincipalRole.class)) - .hasSize(1) - .satisfiesExactly(pr -> assertThat(pr).returns("mypr1", PrincipalRole::getName)); - } - - // Empty list if listing in principalRole2 - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr2/catalog-roles/mycatalog") - .get()) { - - assertThat(response) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus) - .extracting(r -> r.readEntity(CatalogRoles.class)) - .returns(List.of(), CatalogRoles::getRoles); - } - - // 204 Successful revoke - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr1/catalog-roles/mycatalog/mycr") - .delete()) { - - assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } - - // Empty list - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr1/catalog-roles/mycatalog") - .get()) { - - assertThat(response) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus) - .extracting(r -> r.readEntity(CatalogRoles.class)) - .returns(List.of(), CatalogRoles::getRoles); - } - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog/catalog-roles/mycr/principal-roles") - .get()) { - - assertThat(response) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus) - .extracting(r -> r.readEntity(PrincipalRoles.class)) - .returns(List.of(), PrincipalRoles::getRoles); - } - - // 204 Successful delete mypr1 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/mypr1").delete()) { - - assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } - - // 204 Successful delete mypr2 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/mypr2").delete()) { - - assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } - - // 204 Successful delete mycr - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog/catalog-roles/mycr") - .delete()) { - - assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } - - // 204 Successful delete mycatalog - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").delete()) { - - assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } - - // 204 Successful delete myothercr - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/othercatalog/catalog-roles/myothercr") - .delete()) { - - assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } - - // 204 Successful delete othercatalog - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/othercatalog").delete()) { - - assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } - } - - @Test - public void testCatalogAdminGrantAndRevokeCatalogRoles() { - // Create a PrincipalRole and a new catalog. Grant the catalog_admin role to the new principal - // role - String principalRoleName = "mypr33"; - PrincipalRole principalRole1 = new PrincipalRole(principalRoleName); - createPrincipalRole(principalRole1); - - String catalogName = "myuniquetestcatalog"; - Catalog catalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName(catalogName) - .setStorageConfigInfo( - new AwsStorageConfigInfo( - "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) - .setProperties(new CatalogProperties("s3://bucket1/")) - .build(); - createCatalog(catalog); - - CatalogRole catalogAdminRole = readCatalogRole(catalogName, "catalog_admin"); - grantCatalogRoleToPrincipalRole( - principalRoleName, catalogName, catalogAdminRole, testHelper.adminToken); - - PrincipalWithCredentials catalogAdminPrincipal = createPrincipal("principal1"); - - grantPrincipalRoleToPrincipal(catalogAdminPrincipal.getPrincipal().getName(), principalRole1); - - String catalogAdminToken = - TokenUtils.getTokenFromSecrets( - testHelper.client, - testHelper.localPort, - catalogAdminPrincipal.getCredentials().getClientId(), - catalogAdminPrincipal.getCredentials().getClientSecret(), - testHelper.realm); - - // Create a second principal role. Use the catalog admin principal to list principal roles and - // grant a catalog role to the new principal role - String principalRoleName2 = "mypr2"; - PrincipalRole principalRole2 = new PrincipalRole(principalRoleName2); - createPrincipalRole(principalRole2); - - // create a catalog role and grant it manage_content privilege - String catalogRoleName = "mycr1"; - createCatalogRole(catalogName, catalogRoleName, catalogAdminToken); - - CatalogPrivilege privilege = CatalogPrivilege.CATALOG_MANAGE_CONTENT; - grantPrivilegeToCatalogRole( - catalogName, - catalogRoleName, - new CatalogGrant(privilege, GrantResource.TypeEnum.CATALOG), - catalogAdminToken, - Response.Status.CREATED); - - // The catalog admin can grant the new catalog role to the mypr2 principal role - grantCatalogRoleToPrincipalRole( - principalRoleName2, catalogName, new CatalogRole(catalogRoleName), catalogAdminToken); - - // But the catalog admin cannot revoke the role because it requires - // PRINCIPAL_ROLE_MANAGE_GRANTS_FOR_GRANTEE - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" - + principalRoleName - + "/catalog-roles/" - + catalogName - + "/" - + catalogRoleName, - catalogAdminToken) - .delete()) { - assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); - } - - // The service admin can revoke the role because it has the - // PRINCIPAL_ROLE_MANAGE_GRANTS_FOR_GRANTEE privilege - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" - + principalRoleName - + "/catalog-roles/" - + catalogName - + "/" - + catalogRoleName, - testHelper.adminToken) - .delete()) { - assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } - } - - @Test - public void testServiceAdminCanTransferCatalogAdmin() { - // Create a PrincipalRole and a new catalog. Grant the catalog_admin role to the new principal - // role - String principalRoleName = "mypr33"; - PrincipalRole principalRole1 = new PrincipalRole(principalRoleName); - createPrincipalRole(principalRole1); - - String catalogName = "myothertestcatalog"; - Catalog catalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName(catalogName) - .setStorageConfigInfo( - new AwsStorageConfigInfo( - "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) - .setProperties(new CatalogProperties("s3://bucket1/")) - .build(); - createCatalog(catalog); - - CatalogRole catalogAdminRole = readCatalogRole(catalogName, "catalog_admin"); - grantCatalogRoleToPrincipalRole( - principalRoleName, catalogName, catalogAdminRole, testHelper.adminToken); - - PrincipalWithCredentials catalogAdminPrincipal = createPrincipal("principal1"); - - grantPrincipalRoleToPrincipal(catalogAdminPrincipal.getPrincipal().getName(), principalRole1); - - String catalogAdminToken = - TokenUtils.getTokenFromSecrets( - testHelper.client, - testHelper.localPort, - catalogAdminPrincipal.getCredentials().getClientId(), - catalogAdminPrincipal.getCredentials().getClientSecret(), - testHelper.realm); - - // service_admin revokes the catalog_admin privilege from its principal role - try { - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/service_admin/catalog-roles/" - + catalogName - + "/catalog_admin", - testHelper.adminToken) - .delete()) { - assertThat(response) - .returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } - - // the service_admin can not revoke the catalog_admin privilege from the new principal role - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" - + principalRoleName - + "/catalog-roles/" - + catalogName - + "/catalog_admin", - catalogAdminToken) - .delete()) { - assertThat(response) - .returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); - } - } finally { - // grant the admin role back to service_admin so that cleanup can happen - grantCatalogRoleToPrincipalRole( - "service_admin", catalogName, catalogAdminRole, catalogAdminToken); - } - } - - @Test - public void testCatalogAdminGrantAndRevokeCatalogRolesFromWrongCatalog() { - // Create a PrincipalRole and a new catalog. Grant the catalog_admin role to the new principal - // role - String principalRoleName = "mypr33"; - PrincipalRole principalRole1 = new PrincipalRole(principalRoleName); - createPrincipalRole(principalRole1); - - // create a catalog - String catalogName = "mytestcatalog"; - Catalog catalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName(catalogName) - .setStorageConfigInfo( - new AwsStorageConfigInfo( - "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) - .setProperties(new CatalogProperties("s3://bucket1/")) - .build(); - createCatalog(catalog); - - // create a second catalog - String catalogName2 = "anothercatalog"; - Catalog catalog2 = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName(catalogName2) - .setStorageConfigInfo( - new AwsStorageConfigInfo( - "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) - .setProperties(new CatalogProperties("s3://bucket1/")) - .build(); - createCatalog(catalog2); - - // create a catalog role *in the second catalog* and grant it manage_content privilege - String catalogRoleName = "mycr1"; - createCatalogRole(catalogName2, catalogRoleName, testHelper.adminToken); - - // Get the catalog admin role from the *first* catalog and grant that role to the principal role - CatalogRole catalogAdminRole = readCatalogRole(catalogName, "catalog_admin"); - grantCatalogRoleToPrincipalRole( - principalRoleName, catalogName, catalogAdminRole, testHelper.adminToken); - - // Create a principal and grant the principal role to it - PrincipalWithCredentials catalogAdminPrincipal = createPrincipal("principal1"); - grantPrincipalRoleToPrincipal(catalogAdminPrincipal.getPrincipal().getName(), principalRole1); - - String catalogAdminToken = - TokenUtils.getTokenFromSecrets( - testHelper.client, - testHelper.localPort, - catalogAdminPrincipal.getCredentials().getClientId(), - catalogAdminPrincipal.getCredentials().getClientSecret(), - testHelper.realm); - - // Create a second principal role. - String principalRoleName2 = "mypr2"; - PrincipalRole principalRole2 = new PrincipalRole(principalRoleName2); - createPrincipalRole(principalRole2); - - // The catalog admin cannot grant the new catalog role to the mypr2 principal role because the - // catalog role is in the wrong catalog - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" - + principalRoleName - + "/catalog-roles/" - + catalogName2, - catalogAdminToken) - .put(Entity.json(new GrantCatalogRoleRequest(new CatalogRole(catalogRoleName))))) { - assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); - } - } - - @Test - public void testTableManageAccessCanGrantAndRevokeFromCatalogRoles() { - // Create a PrincipalRole and a new catalog. - String principalRoleName = "mypr33"; - PrincipalRole principalRole1 = new PrincipalRole(principalRoleName); - createPrincipalRole(principalRole1); - - // create a catalog - String catalogName = "mytablemanagecatalog"; - Catalog catalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName(catalogName) - .setStorageConfigInfo( - new AwsStorageConfigInfo( - "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) - .setProperties(new CatalogProperties("s3://bucket1/")) - .build(); - createCatalog(catalog); - - // create a valid target CatalogRole in this catalog - createCatalogRole(catalogName, "target_catalog_role", testHelper.adminToken); - - // create a second catalog - String catalogName2 = "anothertablemanagecatalog"; - Catalog catalog2 = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName(catalogName2) - .setStorageConfigInfo( - new AwsStorageConfigInfo( - "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) - .setProperties(new CatalogProperties("s3://bucket1/")) - .build(); - createCatalog(catalog2); - - // create an *invalid* target CatalogRole in second catalog - createCatalogRole(catalogName2, "invalid_target_catalog_role", testHelper.adminToken); - - // create the namespace "c" in *both* namespaces - String namespaceName = "c"; - createNamespace(catalogName, namespaceName); - createNamespace(catalogName2, namespaceName); - - // create a catalog role *in the first catalog* and grant it manage_content privilege at the - // namespace level - // grant that role to the PrincipalRole - String catalogRoleName = "ns_manage_access_role"; - createCatalogRole(catalogName, catalogRoleName, testHelper.adminToken); - grantPrivilegeToCatalogRole( - catalogName, - catalogRoleName, - new NamespaceGrant( - List.of(namespaceName), - NamespacePrivilege.CATALOG_MANAGE_ACCESS, - GrantResource.TypeEnum.NAMESPACE), - testHelper.adminToken, - Response.Status.CREATED); - - grantCatalogRoleToPrincipalRole( - principalRoleName, catalogName, new CatalogRole(catalogRoleName), testHelper.adminToken); - - // Create a principal and grant the principal role to it - PrincipalWithCredentials catalogAdminPrincipal = createPrincipal("ns_manage_access_user"); - grantPrincipalRoleToPrincipal(catalogAdminPrincipal.getPrincipal().getName(), principalRole1); - - String manageAccessUserToken = - TokenUtils.getTokenFromSecrets( - testHelper.client, - testHelper.localPort, - catalogAdminPrincipal.getCredentials().getClientId(), - catalogAdminPrincipal.getCredentials().getClientSecret(), - testHelper.realm); - - // Use the ns_manage_access_user to grant TABLE_CREATE access to the target catalog role - // This works because the user has CATALOG_MANAGE_ACCESS within the namespace and the target - // catalog role is in - // the same catalog - grantPrivilegeToCatalogRole( - catalogName, - "target_catalog_role", - new NamespaceGrant( - List.of(namespaceName), - NamespacePrivilege.TABLE_CREATE, - GrantResource.TypeEnum.NAMESPACE), - manageAccessUserToken, - Response.Status.CREATED); - - // Even though the ns_manage_access_role can grant privileges to the catalog role, it cannot - // grant the target - // catalog role to the mypr2 principal role because it doesn't have privilege to manage grants - // on the catalog role - // as a securable - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" - + principalRoleName - + "/catalog-roles/" - + catalogName, - manageAccessUserToken) - .put( - Entity.json(new GrantCatalogRoleRequest(new CatalogRole("target_catalog_role"))))) { - assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); - } - - // The user cannot grant catalog-level privileges to the catalog role - grantPrivilegeToCatalogRole( - catalogName, - "target_catalog_role", - new CatalogGrant(CatalogPrivilege.TABLE_CREATE, GrantResource.TypeEnum.CATALOG), - manageAccessUserToken, - Response.Status.FORBIDDEN); - - // even though the namespace "c" exists in both catalogs, the ns_manage_access_role can only - // grant privileges for - // the namespace in its own catalog - grantPrivilegeToCatalogRole( - catalogName2, - "invalid_target_catalog_role", - new NamespaceGrant( - List.of(namespaceName), - NamespacePrivilege.TABLE_CREATE, - GrantResource.TypeEnum.NAMESPACE), - manageAccessUserToken, - Response.Status.FORBIDDEN); - - // nor can it grant privileges to the catalog role in the second catalog - grantPrivilegeToCatalogRole( - catalogName2, - "invalid_target_catalog_role", - new CatalogGrant(CatalogPrivilege.TABLE_CREATE, GrantResource.TypeEnum.CATALOG), - manageAccessUserToken, - Response.Status.FORBIDDEN); - } - - private void createNamespace(String catalogName, String namespaceName) { - try (Response response = - newRequest( - "http://localhost:%d/api/catalog/v1/" + catalogName + "/namespaces", - testHelper.adminToken) - .post( - Entity.json( - CreateNamespaceRequest.builder() - .withNamespace(Namespace.of(namespaceName)) - .build()))) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - } - } - - private void createCatalog(Catalog catalog) { - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(new CreateCatalogRequest(catalog)))) { - - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - } - - private void grantPrivilegeToCatalogRole( - String catalogName, - String catalogRoleName, - GrantResource grant, - String catalogAdminToken, - Response.Status expectedStatus) { - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/" - + catalogName - + "/catalog-roles/" - + catalogRoleName - + "/grants", - catalogAdminToken) - .put(Entity.json(new AddGrantRequest(grant)))) { - assertThat(response).returns(expectedStatus.getStatusCode(), Response::getStatus); - } - } - - private void createCatalogRole( - String catalogName, String catalogRoleName, String catalogAdminToken) { - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/" + catalogName + "/catalog-roles", - catalogAdminToken) - .post(Entity.json(new CreateCatalogRoleRequest(new CatalogRole(catalogRoleName))))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - } - - private void grantPrincipalRoleToPrincipal(String principalName, PrincipalRole principalRole) { - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principals/" - + principalName - + "/principal-roles") - .put(Entity.json(new GrantPrincipalRoleRequest(principalRole)))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - } - - private PrincipalWithCredentials createPrincipal(String principalName) { - PrincipalWithCredentials catalogAdminPrincipal; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") - .post(Entity.json(new CreatePrincipalRequest(new Principal(principalName), false)))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - catalogAdminPrincipal = response.readEntity(PrincipalWithCredentials.class); - } - return catalogAdminPrincipal; - } - - private void grantCatalogRoleToPrincipalRole( - String principalRoleName, String catalogName, CatalogRole catalogRole, String token) { - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" - + principalRoleName - + "/catalog-roles/" - + catalogName, - token) - .put(Entity.json(new GrantCatalogRoleRequest(catalogRole)))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - } - - private CatalogRole readCatalogRole(String catalogName, String roleName) { - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/" - + catalogName - + "/catalog-roles/" - + roleName) - .get()) { - - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - return response.readEntity(CatalogRole.class); - } - } - - private void createPrincipalRole(PrincipalRole principalRole1) { - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles") - .post(Entity.json(new CreatePrincipalRoleRequest(principalRole1)))) { - - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - } - - public static class Profile implements QuarkusTestProfile { - - @Override - public Map getConfigOverrides() { - // disallow FILE urls for the sake of tests below - return Map.of( - "polaris.config.feature-configurations.SUPPORTED_CATALOG_STORAGE_TYPES", - "[\"S3\",\"GCS\",\"AZURE\"]"); - } - } -} diff --git a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/auth/JWTRSAKeyPairTest.java b/polaris-service-quarkus/src/test/java/org/apache/polaris/service/auth/JWTRSAKeyPairTest.java deleted file mode 100644 index 3cd8b925d..000000000 --- a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/auth/JWTRSAKeyPairTest.java +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.auth; - -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.assertj.core.api.Assertions.assertThat; - -import com.auth0.jwt.JWT; -import com.auth0.jwt.JWTVerifier; -import com.auth0.jwt.algorithms.Algorithm; -import com.auth0.jwt.interfaces.DecodedJWT; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; -import java.security.KeyPair; -import java.security.KeyPairGenerator; -import java.security.interfaces.RSAPrivateKey; -import java.security.interfaces.RSAPublicKey; -import java.util.Base64; -import java.util.HashMap; -import java.util.Map; -import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.auth.PolarisSecretsManager.PrincipalSecretsResult; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.core.entity.PolarisBaseEntity; -import org.apache.polaris.core.entity.PolarisEntitySubType; -import org.apache.polaris.core.entity.PolarisEntityType; -import org.apache.polaris.core.entity.PolarisPrincipalSecrets; -import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.service.config.DefaultConfigurationStore; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -public class JWTRSAKeyPairTest { - - private void writePemToTmpFile(String privateFileLocation, String publicFileLocation) - throws Exception { - new File(privateFileLocation).delete(); - new File(publicFileLocation).delete(); - KeyPairGenerator kpg = KeyPairGenerator.getInstance("RSA"); - kpg.initialize(2048); - KeyPair kp = kpg.generateKeyPair(); - try (BufferedWriter writer = - new BufferedWriter(new FileWriter(privateFileLocation, UTF_8, true))) { - writer.write("-----BEGIN PRIVATE KEY-----"); // pragma: allowlist secret - writer.newLine(); - writer.write(Base64.getMimeEncoder().encodeToString(kp.getPrivate().getEncoded())); - writer.newLine(); - writer.write("-----END PRIVATE KEY-----"); - writer.newLine(); - } - try (BufferedWriter writer = - new BufferedWriter(new FileWriter(publicFileLocation, UTF_8, true))) { - writer.write("-----BEGIN PUBLIC KEY-----"); - writer.newLine(); - writer.write(Base64.getMimeEncoder().encodeToString(kp.getPublic().getEncoded())); - writer.newLine(); - writer.write("-----END PUBLIC KEY-----"); - writer.newLine(); - } - } - - public CallContext getTestCallContext(PolarisCallContext polarisCallContext) { - return CallContext.setCurrentContext( - new CallContext() { - @Override - public RealmContext getRealmContext() { - return () -> "realm"; - } - - @Override - public PolarisCallContext getPolarisCallContext() { - return polarisCallContext; - } - - @Override - public Map contextVariables() { - return Map.of("token", "me"); - } - }); - } - - @Test - public void testSuccessfulTokenGeneration() throws Exception { - String privateFileLocation = "/tmp/test-private.pem"; - String publicFileLocation = "/tmp/test-public.pem"; - writePemToTmpFile(privateFileLocation, publicFileLocation); - - final String clientId = "test-client-id"; - final String scope = "PRINCIPAL_ROLE:TEST"; - - Map config = new HashMap<>(); - - config.put("LOCAL_PRIVATE_KEY_LOCATION_KEY", privateFileLocation); - config.put("LOCAL_PUBLIC_LOCATION_KEY", publicFileLocation); - - DefaultConfigurationStore store = new DefaultConfigurationStore(config); - PolarisCallContext polarisCallContext = new PolarisCallContext(null, null, store, null); - CallContext.setCurrentContext(getTestCallContext(polarisCallContext)); - PolarisMetaStoreManager metastoreManager = Mockito.mock(PolarisMetaStoreManager.class); - String mainSecret = "client-secret"; - PolarisPrincipalSecrets principalSecrets = - new PolarisPrincipalSecrets(1L, clientId, mainSecret, "otherSecret"); - Mockito.when(metastoreManager.loadPrincipalSecrets(polarisCallContext, clientId)) - .thenReturn(new PrincipalSecretsResult(principalSecrets)); - PolarisBaseEntity principal = - new PolarisBaseEntity( - 0L, - 1L, - PolarisEntityType.PRINCIPAL, - PolarisEntitySubType.NULL_SUBTYPE, - 0L, - "principal"); - Mockito.when(metastoreManager.loadEntity(polarisCallContext, 0L, 1L)) - .thenReturn(new PolarisMetaStoreManager.EntityResult(principal)); - TokenBroker tokenBroker = new JWTRSAKeyPair(metastoreManager, 420); - TokenResponse token = - tokenBroker.generateFromClientSecrets( - clientId, mainSecret, TokenRequestValidator.CLIENT_CREDENTIALS, scope); - assertThat(token).isNotNull(); - assertThat(token.getExpiresIn()).isEqualTo(420); - - LocalRSAKeyProvider provider = new LocalRSAKeyProvider(); - assertThat(provider.getPrivateKey()).isNotNull(); - assertThat(provider.getPublicKey()).isNotNull(); - JWTVerifier verifier = - JWT.require( - Algorithm.RSA256( - (RSAPublicKey) provider.getPublicKey(), - (RSAPrivateKey) provider.getPrivateKey())) - .withIssuer("polaris") - .build(); - DecodedJWT decodedJWT = verifier.verify(token.getAccessToken()); - assertThat(decodedJWT).isNotNull(); - assertThat(decodedJWT.getClaim("scope").asString()).isEqualTo("PRINCIPAL_ROLE:TEST"); - assertThat(decodedJWT.getClaim("client_id").asString()).isEqualTo("test-client-id"); - } -} diff --git a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/auth/JWTSymmetricKeyGeneratorTest.java b/polaris-service-quarkus/src/test/java/org/apache/polaris/service/auth/JWTSymmetricKeyGeneratorTest.java deleted file mode 100644 index 110a99269..000000000 --- a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/auth/JWTSymmetricKeyGeneratorTest.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.auth; - -import static org.assertj.core.api.Assertions.assertThat; - -import com.auth0.jwt.JWT; -import com.auth0.jwt.JWTVerifier; -import com.auth0.jwt.algorithms.Algorithm; -import com.auth0.jwt.interfaces.DecodedJWT; -import java.util.Map; -import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.core.entity.PolarisBaseEntity; -import org.apache.polaris.core.entity.PolarisEntitySubType; -import org.apache.polaris.core.entity.PolarisEntityType; -import org.apache.polaris.core.entity.PolarisPrincipalSecrets; -import org.apache.polaris.core.persistence.PolarisEntityManager; -import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.storage.cache.StorageCredentialCache; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -public class JWTSymmetricKeyGeneratorTest { - - /** Sanity test to verify that we can generate a token */ - @Test - public void testJWTSymmetricKeyGenerator() { - PolarisCallContext polarisCallContext = new PolarisCallContext(null, null, null, null); - CallContext.setCurrentContext( - new CallContext() { - @Override - public RealmContext getRealmContext() { - return () -> "realm"; - } - - @Override - public PolarisCallContext getPolarisCallContext() { - return polarisCallContext; - } - - @Override - public Map contextVariables() { - return Map.of(); - } - }); - PolarisMetaStoreManager metastoreManager = Mockito.mock(PolarisMetaStoreManager.class); - String mainSecret = "test_secret"; - String clientId = "test_client_id"; - PolarisPrincipalSecrets principalSecrets = - new PolarisPrincipalSecrets(1L, clientId, mainSecret, "otherSecret"); - PolarisEntityManager entityManager = - new PolarisEntityManager(metastoreManager, new StorageCredentialCache()); - Mockito.when(metastoreManager.loadPrincipalSecrets(polarisCallContext, clientId)) - .thenReturn(new PolarisMetaStoreManager.PrincipalSecretsResult(principalSecrets)); - PolarisBaseEntity principal = - new PolarisBaseEntity( - 0L, - 1L, - PolarisEntityType.PRINCIPAL, - PolarisEntitySubType.NULL_SUBTYPE, - 0L, - "principal"); - Mockito.when(metastoreManager.loadEntity(polarisCallContext, 0L, 1L)) - .thenReturn(new PolarisMetaStoreManager.EntityResult(principal)); - TokenBroker generator = new JWTSymmetricKeyBroker(metastoreManager, 666, () -> "polaris"); - TokenResponse token = - generator.generateFromClientSecrets( - clientId, mainSecret, TokenRequestValidator.CLIENT_CREDENTIALS, "PRINCIPAL_ROLE:TEST"); - assertThat(token).isNotNull(); - - JWTVerifier verifier = JWT.require(Algorithm.HMAC256("polaris")).withIssuer("polaris").build(); - DecodedJWT decodedJWT = verifier.verify(token.getAccessToken()); - assertThat(decodedJWT).isNotNull(); - assertThat(token.getExpiresIn()).isEqualTo(666); - assertThat(decodedJWT.getClaim("scope").asString()).isEqualTo("PRINCIPAL_ROLE:TEST"); - assertThat(decodedJWT.getClaim("client_id").asString()).isEqualTo(clientId); - } -} diff --git a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/auth/TokenRequestValidatorTest.java b/polaris-service-quarkus/src/test/java/org/apache/polaris/service/auth/TokenRequestValidatorTest.java deleted file mode 100644 index bd8dd9a3a..000000000 --- a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/auth/TokenRequestValidatorTest.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.auth; - -import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; - -public class TokenRequestValidatorTest { - @Test - public void testValidateForClientCredentialsFlowNullClientId() { - Assertions.assertThat( - new TokenRequestValidator() - .validateForClientCredentialsFlow(null, "notnull", "notnull", "nontnull") - .get()) - .isEqualTo(OAuthTokenErrorResponse.Error.invalid_client); - Assertions.assertThat( - new TokenRequestValidator() - .validateForClientCredentialsFlow("", "notnull", "notnull", "nonnull") - .get()) - .isEqualTo(OAuthTokenErrorResponse.Error.invalid_client); - } - - @Test - public void testValidateForClientCredentialsFlowNullClientSecret() { - Assertions.assertThat( - new TokenRequestValidator() - .validateForClientCredentialsFlow("client-id", null, "notnull", "nontnull") - .get()) - .isEqualTo(OAuthTokenErrorResponse.Error.invalid_client); - Assertions.assertThat( - new TokenRequestValidator() - .validateForClientCredentialsFlow("client-id", "", "notnull", "notnull") - .get()) - .isEqualTo(OAuthTokenErrorResponse.Error.invalid_client); - } - - @Test - public void testValidateForClientCredentialsFlowInvalidGrantType() { - Assertions.assertThat( - new TokenRequestValidator() - .validateForClientCredentialsFlow( - "client-id", "client-secret", "not-client-credentials", "notnull") - .get()) - .isEqualTo(OAuthTokenErrorResponse.Error.invalid_grant); - Assertions.assertThat( - new TokenRequestValidator() - .validateForClientCredentialsFlow("client-id", "client-secret", "grant", "notnull") - .get()) - .isEqualTo(OAuthTokenErrorResponse.Error.invalid_grant); - } - - @ParameterizedTest - @ValueSource(strings = {"null", "", ",", "ALL", "PRINCIPAL_ROLE:", "PRINCIPAL_ROLE"}) - public void testValidateForClientCredentialsFlowInvalidScope(String scope) { - Assertions.assertThat( - new TokenRequestValidator() - .validateForClientCredentialsFlow( - "client-id", "client-secret", "client_credentials", scope) - .get()) - .isEqualTo(OAuthTokenErrorResponse.Error.invalid_scope); - } - - @Test - public void testValidateForClientCredentialsFlowAllValid() { - Assertions.assertThat( - new TokenRequestValidator() - .validateForClientCredentialsFlow( - "client-id", "client-secret", "client_credentials", "PRINCIPAL_ROLE:ALL")) - .isEmpty(); - } -} diff --git a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/auth/TokenUtils.java b/polaris-service-quarkus/src/test/java/org/apache/polaris/service/auth/TokenUtils.java deleted file mode 100644 index a4cc29803..000000000 --- a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/auth/TokenUtils.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.auth; - -import static org.apache.polaris.service.auth.BasePolarisAuthenticator.PRINCIPAL_ROLE_ALL; -import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; -import static org.assertj.core.api.Assertions.assertThat; - -import jakarta.ws.rs.client.Client; -import jakarta.ws.rs.client.Entity; -import jakarta.ws.rs.client.Invocation; -import jakarta.ws.rs.core.MultivaluedHashMap; -import jakarta.ws.rs.core.Response; -import java.util.Map; -import org.apache.iceberg.rest.responses.OAuthTokenResponse; - -public class TokenUtils { - - /** Get token against specified realm */ - public static String getTokenFromSecrets( - Client client, int port, String clientId, String clientSecret, String realm) { - String token; - - Invocation.Builder builder = - client - .target(String.format("http://localhost:%d/api/catalog/v1/oauth/tokens", port)) - .request("application/json"); - if (realm != null) { - builder = builder.header(REALM_PROPERTY_KEY, realm); - } - - try (Response response = - builder.post( - Entity.form( - new MultivaluedHashMap<>( - Map.of( - "grant_type", - "client_credentials", - "scope", - PRINCIPAL_ROLE_ALL, - "client_id", - clientId, - "client_secret", - clientSecret))))) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - token = response.readEntity(OAuthTokenResponse.class).token(); - } - return token; - } -} diff --git a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/AccessDelegationModeTest.java b/polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/AccessDelegationModeTest.java deleted file mode 100644 index 367840116..000000000 --- a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/AccessDelegationModeTest.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.catalog; - -import static org.apache.polaris.service.catalog.AccessDelegationMode.*; -import static org.apache.polaris.service.catalog.AccessDelegationMode.REMOTE_SIGNING; -import static org.apache.polaris.service.catalog.AccessDelegationMode.VENDED_CREDENTIALS; -import static org.apache.polaris.service.catalog.AccessDelegationMode.fromProtocolValuesList; -import static org.assertj.core.api.Assertions.assertThat; - -import java.util.EnumSet; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.EnumSource; - -class AccessDelegationModeTest { - - @ParameterizedTest - @EnumSource(AccessDelegationMode.class) - void testSingle(AccessDelegationMode mode) { - assertThat(fromProtocolValuesList(mode.protocolValue())).isEqualTo(EnumSet.of(mode)); - } - - @Test - void testSeveral() { - assertThat(fromProtocolValuesList("vended-credentials, remote-signing")) - .isEqualTo(EnumSet.of(VENDED_CREDENTIALS, REMOTE_SIGNING)); - } - - @Test - void testEmpty() { - assertThat(fromProtocolValuesList(null)).isEqualTo(EnumSet.noneOf(AccessDelegationMode.class)); - assertThat(fromProtocolValuesList("")).isEqualTo(EnumSet.noneOf(AccessDelegationMode.class)); - } - - @Test - void testUnknown() { - assertThat(fromProtocolValuesList("abc")).isEqualTo(EnumSet.of(UNKNOWN)); - assertThat(fromProtocolValuesList("abc,def")).isEqualTo(EnumSet.of(UNKNOWN)); - assertThat(fromProtocolValuesList("abc,remote-signing")) - .isEqualTo(EnumSet.of(REMOTE_SIGNING, UNKNOWN)); - } - - @Test - void testLegacy() { - assertThat(fromProtocolValuesList("true")).isEqualTo(EnumSet.of(VENDED_CREDENTIALS)); - assertThat(fromProtocolValuesList("true, vended-credentials")) - .isEqualTo(EnumSet.of(UNKNOWN, VENDED_CREDENTIALS)); - } -} diff --git a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java b/polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java deleted file mode 100644 index 09291b3d8..000000000 --- a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java +++ /dev/null @@ -1,1530 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.catalog; - -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.iceberg.types.Types.NestedField.required; -import static org.mockito.ArgumentMatchers.isA; -import static org.mockito.Mockito.when; - -import com.google.common.collect.ImmutableMap; -import io.quarkus.test.junit.QuarkusMock; -import io.quarkus.test.junit.QuarkusTest; -import jakarta.inject.Inject; -import java.io.IOException; -import java.lang.reflect.Method; -import java.time.Clock; -import java.util.Arrays; -import java.util.EnumMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.function.Supplier; -import org.apache.commons.lang3.NotImplementedException; -import org.apache.iceberg.BaseTable; -import org.apache.iceberg.CatalogProperties; -import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.Schema; -import org.apache.iceberg.SortOrder; -import org.apache.iceberg.Table; -import org.apache.iceberg.TableMetadata; -import org.apache.iceberg.TableMetadataParser; -import org.apache.iceberg.catalog.CatalogTests; -import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.catalog.SupportsNamespaces; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.exceptions.AlreadyExistsException; -import org.apache.iceberg.exceptions.BadRequestException; -import org.apache.iceberg.exceptions.ForbiddenException; -import org.apache.iceberg.exceptions.NoSuchNamespaceException; -import org.apache.iceberg.inmemory.InMemoryFileIO; -import org.apache.iceberg.io.FileIO; -import org.apache.iceberg.types.Types; -import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.PolarisConfiguration; -import org.apache.polaris.core.PolarisConfigurationStore; -import org.apache.polaris.core.PolarisDiagnostics; -import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; -import org.apache.polaris.core.admin.model.StorageConfigInfo; -import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; -import org.apache.polaris.core.auth.PolarisAuthorizerImpl; -import org.apache.polaris.core.auth.PolarisSecretsManager.PrincipalSecretsResult; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.core.entity.CatalogEntity; -import org.apache.polaris.core.entity.PolarisBaseEntity; -import org.apache.polaris.core.entity.PolarisEntity; -import org.apache.polaris.core.entity.PolarisEntitySubType; -import org.apache.polaris.core.entity.PolarisEntityType; -import org.apache.polaris.core.entity.PrincipalEntity; -import org.apache.polaris.core.entity.TaskEntity; -import org.apache.polaris.core.monitor.PolarisMetricRegistry; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.core.persistence.PolarisEntityManager; -import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.PolarisMetaStoreSession; -import org.apache.polaris.core.storage.PolarisCredentialProperty; -import org.apache.polaris.core.storage.PolarisStorageIntegration; -import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; -import org.apache.polaris.core.storage.aws.AwsCredentialsStorageIntegration; -import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo; -import org.apache.polaris.core.storage.cache.StorageCredentialCache; -import org.apache.polaris.service.admin.PolarisAdminService; -import org.apache.polaris.service.catalog.io.DefaultFileIOFactory; -import org.apache.polaris.service.catalog.io.FileIOFactory; -import org.apache.polaris.service.catalog.io.MeasuredFileIOFactory; -import org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl; -import org.apache.polaris.service.task.TableCleanupTaskHandler; -import org.apache.polaris.service.task.TaskExecutor; -import org.apache.polaris.service.task.TaskFileIOSupplier; -import org.apache.polaris.service.types.NotificationRequest; -import org.apache.polaris.service.types.NotificationType; -import org.apache.polaris.service.types.TableUpdateNotification; -import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInfo; -import org.mockito.Mockito; -import software.amazon.awssdk.services.sts.StsClient; -import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; -import software.amazon.awssdk.services.sts.model.AssumeRoleResponse; -import software.amazon.awssdk.services.sts.model.Credentials; - -@QuarkusTest -public class BasePolarisCatalogTest extends CatalogTests { - protected static final Namespace NS = Namespace.of("newdb"); - protected static final TableIdentifier TABLE = TableIdentifier.of(NS, "table"); - protected static final Schema SCHEMA = - new Schema( - required(3, "id", Types.IntegerType.get(), "unique ID 🤪"), - required(4, "data", Types.StringType.get())); - public static final String CATALOG_NAME = "polaris-catalog"; - public static final String TEST_ACCESS_KEY = "test_access_key"; - public static final String SECRET_ACCESS_KEY = "secret_access_key"; - public static final String SESSION_TOKEN = "session_token"; - - @Inject MetaStoreManagerFactory managerFactory; - @Inject PolarisConfigurationStore configurationStore; - @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; - @Inject PolarisDiagnostics diagServices; - - private BasePolarisCatalog catalog; - - private String realmName; - private PolarisMetaStoreManager metaStoreManager; - private PolarisCallContext polarisContext; - private PolarisAdminService adminService; - private PolarisEntityManager entityManager; - private AuthenticatedPolarisPrincipal authenticatedRoot; - private PolarisEntity catalogEntity; - - @BeforeAll - public static void setUpMocks() { - PolarisStorageIntegrationProviderImpl mock = - Mockito.mock(PolarisStorageIntegrationProviderImpl.class); - QuarkusMock.installMockForType(mock, PolarisStorageIntegrationProviderImpl.class); - } - - @BeforeEach - @SuppressWarnings("unchecked") - public void before(TestInfo testInfo) { - realmName = - "realm_%s_%s" - .formatted( - testInfo.getTestMethod().map(Method::getName).orElse("test"), System.nanoTime()); - RealmContext realmContext = () -> realmName; - metaStoreManager = managerFactory.getOrCreateMetaStoreManager(realmContext); - polarisContext = - new PolarisCallContext( - managerFactory.getOrCreateSessionSupplier(realmContext).get(), - diagServices, - configurationStore, - Clock.systemDefaultZone()); - entityManager = new PolarisEntityManager(metaStoreManager, new StorageCredentialCache()); - - CallContext callContext = CallContext.of(realmContext, polarisContext); - CallContext.setCurrentContext(callContext); - - PrincipalEntity rootEntity = - new PrincipalEntity( - PolarisEntity.of( - metaStoreManager - .readEntityByName( - polarisContext, - null, - PolarisEntityType.PRINCIPAL, - PolarisEntitySubType.NULL_SUBTYPE, - "root") - .getEntity())); - - authenticatedRoot = new AuthenticatedPolarisPrincipal(rootEntity, Set.of()); - - adminService = - new PolarisAdminService( - callContext, - entityManager, - metaStoreManager, - authenticatedRoot, - new PolarisAuthorizerImpl(new PolarisConfigurationStore() {})); - String storageLocation = "s3://my-bucket/path/to/data"; - AwsStorageConfigInfo storageConfigModel = - AwsStorageConfigInfo.builder() - .setRoleArn("arn:aws:iam::012345678901:role/jdoe") - .setExternalId("externalId") - .setUserArn("aws::a:user:arn") - .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) - .setAllowedLocations(List.of(storageLocation, "s3://externally-owned-bucket")) - .build(); - catalogEntity = - adminService.createCatalog( - new CatalogEntity.Builder() - .setName(CATALOG_NAME) - .setDefaultBaseLocation(storageLocation) - .setReplaceNewLocationPrefixWithCatalogDefault("file:") - .addProperty( - PolarisConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") - .addProperty( - PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") - .setStorageConfigurationInfo(storageConfigModel, storageLocation) - .build()); - - PolarisPassthroughResolutionView passthroughView = - new PolarisPassthroughResolutionView( - callContext, entityManager, authenticatedRoot, CATALOG_NAME); - TaskExecutor taskExecutor = Mockito.mock(); - this.catalog = - new BasePolarisCatalog( - entityManager, - metaStoreManager, - callContext, - passthroughView, - authenticatedRoot, - taskExecutor, - new DefaultFileIOFactory()); - this.catalog.initialize( - CATALOG_NAME, - ImmutableMap.of( - CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO")); - StsClient stsClient = Mockito.mock(StsClient.class); - when(stsClient.assumeRole(isA(AssumeRoleRequest.class))) - .thenReturn( - AssumeRoleResponse.builder() - .credentials( - Credentials.builder() - .accessKeyId(TEST_ACCESS_KEY) - .secretAccessKey(SECRET_ACCESS_KEY) - .sessionToken(SESSION_TOKEN) - .build()) - .build()); - PolarisStorageIntegration storageIntegration = - new AwsCredentialsStorageIntegration(stsClient); - when(storageIntegrationProvider.getStorageIntegrationForConfig( - isA(AwsStorageConfigurationInfo.class))) - .thenReturn((PolarisStorageIntegration) storageIntegration); - } - - @AfterEach - public void after() throws IOException { - catalog().close(); - metaStoreManager.purge(polarisContext); - } - - @Override - protected BasePolarisCatalog catalog() { - return catalog; - } - - @Override - protected boolean requiresNamespaceCreate() { - return true; - } - - @Override - protected boolean supportsNestedNamespaces() { - return true; - } - - @Override - protected boolean overridesRequestedLocation() { - return true; - } - - protected boolean supportsNotifications() { - return true; - } - - private MetaStoreManagerFactory createMockMetaStoreManagerFactory() { - return new MetaStoreManagerFactory() { - @Override - public PolarisMetaStoreManager getOrCreateMetaStoreManager(RealmContext realmContext) { - return metaStoreManager; - } - - @Override - public Supplier getOrCreateSessionSupplier( - RealmContext realmContext) { - return () -> polarisContext.getMetaStore(); - } - - @Override - public StorageCredentialCache getOrCreateStorageCredentialCache(RealmContext realmContext) { - return new StorageCredentialCache(); - } - - @Override - public void setMetricRegistry(PolarisMetricRegistry metricRegistry) {} - - @Override - public Map bootstrapRealms(List realms) { - throw new NotImplementedException("Bootstrapping realms is not supported"); - } - - @Override - public void purgeRealms(List realms) { - throw new NotImplementedException("Purging realms is not supported"); - } - - @Override - public void setStorageIntegrationProvider( - PolarisStorageIntegrationProvider storageIntegrationProvider) {} - }; - } - - @Test - public void testRenameTableMissingDestinationNamespace() { - Assumptions.assumeTrue( - requiresNamespaceCreate(), - "Only applicable if namespaces must be created before adding children"); - - BasePolarisCatalog catalog = catalog(); - catalog.createNamespace(NS); - - Assertions.assertThat(catalog.tableExists(TABLE)) - .as("Source table should not exist before create") - .isFalse(); - - catalog.buildTable(TABLE, SCHEMA).create(); - Assertions.assertThat(catalog.tableExists(TABLE)) - .as("Table should exist after create") - .isTrue(); - - Namespace newNamespace = Namespace.of("nonexistent_namespace"); - TableIdentifier renamedTable = TableIdentifier.of(newNamespace, "table_renamed"); - - Assertions.assertThat(catalog.namespaceExists(newNamespace)) - .as("Destination namespace should not exist before rename") - .isFalse(); - - Assertions.assertThat(catalog.tableExists(renamedTable)) - .as("Destination table should not exist before rename") - .isFalse(); - - Assertions.assertThatThrownBy(() -> catalog.renameTable(TABLE, renamedTable)) - .isInstanceOf(NoSuchNamespaceException.class) - .hasMessageContaining("Namespace does not exist"); - - Assertions.assertThat(catalog.namespaceExists(newNamespace)) - .as("Destination namespace should not exist after failed rename") - .isFalse(); - - Assertions.assertThat(catalog.tableExists(renamedTable)) - .as("Table should not exist after failed rename") - .isFalse(); - } - - @Test - public void testCreateNestedNamespaceUnderMissingParent() { - Assumptions.assumeTrue( - requiresNamespaceCreate(), - "Only applicable if namespaces must be created before adding children"); - Assumptions.assumeTrue( - supportsNestedNamespaces(), "Only applicable if nested namespaces are supoprted"); - - BasePolarisCatalog catalog = catalog(); - - Namespace child1 = Namespace.of("parent", "child1"); - - Assertions.assertThatThrownBy(() -> catalog.createNamespace(child1)) - .isInstanceOf(NoSuchNamespaceException.class) - .hasMessageContaining("Parent"); - } - - @Test - public void testValidateNotificationWhenTableAndNamespacesDontExist() { - Assumptions.assumeTrue( - requiresNamespaceCreate(), - "Only applicable if namespaces must be created before adding children"); - Assumptions.assumeTrue( - supportsNestedNamespaces(), "Only applicable if nested namespaces are supported"); - Assumptions.assumeTrue( - supportsNotifications(), "Only applicable if notifications are supported"); - - final String tableLocation = "s3://externally-owned-bucket/validate_table/"; - final String tableMetadataLocation = tableLocation + "metadata/v1.metadata.json"; - BasePolarisCatalog catalog = catalog(); - - Namespace namespace = Namespace.of("parent", "child1"); - TableIdentifier table = TableIdentifier.of(namespace, "table"); - - // For a VALIDATE request we can pass in a full metadata JSON filename or just the table's - // metadata directory; either way the path will be validated to be under the allowed locations, - // but any actual metadata JSON file will not be accessed. - NotificationRequest request = new NotificationRequest(); - request.setNotificationType(NotificationType.VALIDATE); - TableUpdateNotification update = new TableUpdateNotification(); - update.setMetadataLocation(tableMetadataLocation); - update.setTableName(table.name()); - update.setTableUuid(UUID.randomUUID().toString()); - update.setTimestamp(230950845L); - request.setPayload(update); - - // We should be able to send the notification without creating the metadata file since it's - // only validating the ability to send the CREATE/UPDATE notification possibly before actually - // creating the table at all on the remote catalog. - Assertions.assertThat(catalog.sendNotification(table, request)) - .as("Notification should be sent successfully") - .isTrue(); - Assertions.assertThat(catalog.namespaceExists(namespace)) - .as("Intermediate namespaces should not be created") - .isFalse(); - Assertions.assertThat(catalog.tableExists(table)) - .as("Table should not be created for a VALIDATE notification") - .isFalse(); - - // Now also check that despite creating the metadata file, the validation call still doesn't - // create any namespaces or tables. - InMemoryFileIO fileIO = (InMemoryFileIO) catalog.getIo(); - fileIO.addFile( - tableMetadataLocation, - TableMetadataParser.toJson(createSampleTableMetadata(tableLocation)).getBytes(UTF_8)); - - Assertions.assertThat(catalog.sendNotification(table, request)) - .as("Notification should be sent successfully") - .isTrue(); - Assertions.assertThat(catalog.namespaceExists(namespace)) - .as("Intermediate namespaces should not be created") - .isFalse(); - Assertions.assertThat(catalog.tableExists(table)) - .as("Table should not be created for a VALIDATE notification") - .isFalse(); - } - - @Test - public void testValidateNotificationInDisallowedLocation() { - Assumptions.assumeTrue( - requiresNamespaceCreate(), - "Only applicable if namespaces must be created before adding children"); - Assumptions.assumeTrue( - supportsNestedNamespaces(), "Only applicable if nested namespaces are supported"); - Assumptions.assumeTrue( - supportsNotifications(), "Only applicable if notifications are supported"); - - // The location of the metadata JSON file specified in the create will be forbidden. - // For a VALIDATE call we can pass in the metadata/ prefix itself instead of a metadata JSON - // filename. - final String tableLocation = "s3://forbidden-table-location/table/"; - final String tableMetadataLocation = tableLocation + "metadata/"; - BasePolarisCatalog catalog = catalog(); - - Namespace namespace = Namespace.of("parent", "child1"); - TableIdentifier table = TableIdentifier.of(namespace, "table"); - - NotificationRequest request = new NotificationRequest(); - request.setNotificationType(NotificationType.VALIDATE); - TableUpdateNotification update = new TableUpdateNotification(); - update.setMetadataLocation(tableMetadataLocation); - update.setTableName(table.name()); - update.setTableUuid(UUID.randomUUID().toString()); - update.setTimestamp(230950845L); - request.setPayload(update); - - Assertions.assertThatThrownBy(() -> catalog.sendNotification(table, request)) - .isInstanceOf(ForbiddenException.class) - .hasMessageContaining("Invalid location"); - } - - @Test - public void testValidateNotificationFailToCreateFileIO() { - Assumptions.assumeTrue( - requiresNamespaceCreate(), - "Only applicable if namespaces must be created before adding children"); - Assumptions.assumeTrue( - supportsNestedNamespaces(), "Only applicable if nested namespaces are supported"); - Assumptions.assumeTrue( - supportsNotifications(), "Only applicable if notifications are supported"); - - // The location of the metadata JSON file specified in the create will be allowed, but - // we'll inject a separate ForbiddenException during FileIO instantiation. - // For a VALIDATE call we can pass in the metadata/ prefix itself instead of a metadata JSON - // filename. - final String tableLocation = "s3://externally-owned-bucket/validate_table/"; - final String tableMetadataLocation = tableLocation + "metadata/"; - BasePolarisCatalog catalog = catalog(); - - Namespace namespace = Namespace.of("parent", "child1"); - TableIdentifier table = TableIdentifier.of(namespace, "table"); - - NotificationRequest request = new NotificationRequest(); - request.setNotificationType(NotificationType.VALIDATE); - TableUpdateNotification update = new TableUpdateNotification(); - update.setMetadataLocation(tableMetadataLocation); - update.setTableName(table.name()); - update.setTableUuid(UUID.randomUUID().toString()); - update.setTimestamp(230950845L); - request.setPayload(update); - - catalog.setFileIOFactory( - new FileIOFactory() { - @Override - public FileIO loadFileIO(String impl, Map properties) { - throw new ForbiddenException("Fake failure applying downscoped credentials"); - } - }); - Assertions.assertThatThrownBy(() -> catalog.sendNotification(table, request)) - .isInstanceOf(ForbiddenException.class) - .hasMessageContaining("Fake failure applying downscoped credentials"); - } - - @Test - public void testUpdateNotificationWhenTableAndNamespacesDontExist() { - Assumptions.assumeTrue( - requiresNamespaceCreate(), - "Only applicable if namespaces must be created before adding children"); - Assumptions.assumeTrue( - supportsNestedNamespaces(), "Only applicable if nested namespaces are supported"); - Assumptions.assumeTrue( - supportsNotifications(), "Only applicable if notifications are supported"); - - final String tableLocation = "s3://externally-owned-bucket/table/"; - final String tableMetadataLocation = tableLocation + "metadata/v1.metadata.json"; - BasePolarisCatalog catalog = catalog(); - - Namespace namespace = Namespace.of("parent", "child1"); - TableIdentifier table = TableIdentifier.of(namespace, "table"); - - NotificationRequest request = new NotificationRequest(); - request.setNotificationType(NotificationType.UPDATE); - TableUpdateNotification update = new TableUpdateNotification(); - update.setMetadataLocation(tableMetadataLocation); - update.setTableName(table.name()); - update.setTableUuid(UUID.randomUUID().toString()); - update.setTimestamp(230950845L); - request.setPayload(update); - - InMemoryFileIO fileIO = (InMemoryFileIO) catalog.getIo(); - - fileIO.addFile( - tableMetadataLocation, - TableMetadataParser.toJson(createSampleTableMetadata(tableLocation)).getBytes(UTF_8)); - - Assertions.assertThat(catalog.sendNotification(table, request)) - .as("Notification should be sent successfully") - .isTrue(); - Assertions.assertThat(catalog.namespaceExists(namespace)) - .as("Intermediate namespaces should be created") - .isTrue(); - Assertions.assertThat(catalog.tableExists(table)) - .as("Table should be created on receiving notification") - .isTrue(); - } - - @Test - public void testUpdateNotificationCreateTableInDisallowedLocation() { - Assumptions.assumeTrue( - requiresNamespaceCreate(), - "Only applicable if namespaces must be created before adding children"); - Assumptions.assumeTrue( - supportsNestedNamespaces(), "Only applicable if nested namespaces are supported"); - Assumptions.assumeTrue( - supportsNotifications(), "Only applicable if notifications are supported"); - - // The location of the metadata JSON file specified in the create will be forbidden. - final String tableLocation = "s3://forbidden-table-location/table/"; - final String tableMetadataLocation = tableLocation + "metadata/v1.metadata.json"; - BasePolarisCatalog catalog = catalog(); - - Namespace namespace = Namespace.of("parent", "child1"); - TableIdentifier table = TableIdentifier.of(namespace, "table"); - - NotificationRequest request = new NotificationRequest(); - request.setNotificationType(NotificationType.UPDATE); - TableUpdateNotification update = new TableUpdateNotification(); - update.setMetadataLocation(tableMetadataLocation); - update.setTableName(table.name()); - update.setTableUuid(UUID.randomUUID().toString()); - update.setTimestamp(230950845L); - request.setPayload(update); - - InMemoryFileIO fileIO = (InMemoryFileIO) catalog.getIo(); - - fileIO.addFile( - tableMetadataLocation, - TableMetadataParser.toJson(createSampleTableMetadata(tableLocation)).getBytes(UTF_8)); - - Assertions.assertThatThrownBy(() -> catalog.sendNotification(table, request)) - .isInstanceOf(ForbiddenException.class) - .hasMessageContaining("Invalid location"); - } - - @Test - public void testCreateNotificationCreateTableInExternalLocation() { - Assumptions.assumeTrue( - requiresNamespaceCreate(), - "Only applicable if namespaces must be created before adding children"); - Assumptions.assumeTrue( - supportsNestedNamespaces(), "Only applicable if nested namespaces are supported"); - Assumptions.assumeTrue( - supportsNotifications(), "Only applicable if notifications are supported"); - - // The location of the metadata JSON file specified is outside of the table's base location - // according to the - // metadata. We assume this is fraudulent and disallowed - final String tableLocation = "s3://my-bucket/path/to/data/my_table/"; - final String tableMetadataLocation = tableLocation + "metadata/v1.metadata.json"; - final String anotherTableLocation = "s3://my-bucket/path/to/data/another_table/"; - - metaStoreManager.updateEntityPropertiesIfNotChanged( - polarisContext, - List.of(PolarisEntity.toCore(catalogEntity)), - new CatalogEntity.Builder(CatalogEntity.of(catalogEntity)) - .addProperty( - PolarisConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "false") - .addProperty( - PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") - .build()); - BasePolarisCatalog catalog = catalog(); - TableMetadata tableMetadata = - TableMetadata.buildFromEmpty() - .assignUUID() - .setLocation(anotherTableLocation) - .addSchema(SCHEMA, 4) - .addPartitionSpec(PartitionSpec.unpartitioned()) - .addSortOrder(SortOrder.unsorted()) - .build(); - TableMetadataParser.write(tableMetadata, catalog.getIo().newOutputFile(tableMetadataLocation)); - - Namespace namespace = Namespace.of("parent", "child1"); - TableIdentifier table = TableIdentifier.of(namespace, "my_table"); - - NotificationRequest request = new NotificationRequest(); - request.setNotificationType(NotificationType.CREATE); - TableUpdateNotification update = new TableUpdateNotification(); - update.setMetadataLocation(tableMetadataLocation); - update.setTableName(table.name()); - update.setTableUuid(UUID.randomUUID().toString()); - update.setTimestamp(230950845L); - request.setPayload(update); - - Assertions.assertThatThrownBy(() -> catalog.sendNotification(table, request)) - .isInstanceOf(BadRequestException.class) - .hasMessageContaining("is not allowed outside of table location"); - } - - @Test - public void testCreateNotificationCreateTableOutsideOfMetadataLocation() { - Assumptions.assumeTrue( - requiresNamespaceCreate(), - "Only applicable if namespaces must be created before adding children"); - Assumptions.assumeTrue( - supportsNestedNamespaces(), "Only applicable if nested namespaces are supported"); - Assumptions.assumeTrue( - supportsNotifications(), "Only applicable if notifications are supported"); - - // The location of the metadata JSON file specified is outside of the table's metadata directory - // according to the - // metadata. We assume this is fraudulent and disallowed - final String tableLocation = "s3://my-bucket/path/to/data/my_table/"; - final String tableMetadataLocation = tableLocation + "metadata/v3.metadata.json"; - - // this passes the first validation, since it's within the namespace subdirectory, but - // the location is in another table's subdirectory - final String anotherTableLocation = "s3://my-bucket/path/to/data/another_table"; - - metaStoreManager.updateEntityPropertiesIfNotChanged( - polarisContext, - List.of(PolarisEntity.toCore(catalogEntity)), - new CatalogEntity.Builder(CatalogEntity.of(catalogEntity)) - .addProperty( - PolarisConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "false") - .addProperty( - PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") - .build()); - BasePolarisCatalog catalog = catalog(); - TableMetadata tableMetadata = - TableMetadata.buildFromEmpty() - .assignUUID() - .setLocation(anotherTableLocation) - .addSchema(SCHEMA, 4) - .addPartitionSpec(PartitionSpec.unpartitioned()) - .addSortOrder(SortOrder.unsorted()) - .build(); - TableMetadataParser.write(tableMetadata, catalog.getIo().newOutputFile(tableMetadataLocation)); - - Namespace namespace = Namespace.of("parent", "child1"); - TableIdentifier table = TableIdentifier.of(namespace, "my_table"); - - NotificationRequest request = new NotificationRequest(); - request.setNotificationType(NotificationType.CREATE); - TableUpdateNotification update = new TableUpdateNotification(); - update.setMetadataLocation(tableMetadataLocation); - update.setTableName(table.name()); - update.setTableUuid(UUID.randomUUID().toString()); - update.setTimestamp(230950845L); - request.setPayload(update); - - Assertions.assertThatThrownBy(() -> catalog.sendNotification(table, request)) - .isInstanceOf(BadRequestException.class) - .hasMessageContaining("is not allowed outside of table location"); - } - - @Test - public void testUpdateNotificationCreateTableInExternalLocation() { - Assumptions.assumeTrue( - requiresNamespaceCreate(), - "Only applicable if namespaces must be created before adding children"); - Assumptions.assumeTrue( - supportsNestedNamespaces(), "Only applicable if nested namespaces are supported"); - Assumptions.assumeTrue( - supportsNotifications(), "Only applicable if notifications are supported"); - - // The location of the metadata JSON file specified is outside of the table's base location - // according to the - // metadata. We assume this is fraudulent and disallowed - final String tableLocation = "s3://my-bucket/path/to/data/my_table/"; - final String tableMetadataLocation = tableLocation + "metadata/v1.metadata.json"; - final String anotherTableLocation = "s3://my-bucket/path/to/data/another_table/"; - - metaStoreManager.updateEntityPropertiesIfNotChanged( - polarisContext, - List.of(PolarisEntity.toCore(catalogEntity)), - new CatalogEntity.Builder(CatalogEntity.of(catalogEntity)) - .addProperty( - PolarisConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "false") - .addProperty( - PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") - .build()); - BasePolarisCatalog catalog = catalog(); - InMemoryFileIO fileIO = (InMemoryFileIO) catalog.getIo(); - - fileIO.addFile( - tableMetadataLocation, - TableMetadataParser.toJson(createSampleTableMetadata(tableLocation)).getBytes(UTF_8)); - - Namespace namespace = Namespace.of("parent", "child1"); - TableIdentifier table = TableIdentifier.of(namespace, "my_table"); - - NotificationRequest createRequest = new NotificationRequest(); - createRequest.setNotificationType(NotificationType.CREATE); - TableUpdateNotification create = new TableUpdateNotification(); - create.setMetadataLocation(tableMetadataLocation); - create.setTableName(table.name()); - create.setTableUuid(UUID.randomUUID().toString()); - create.setTimestamp(230950845L); - createRequest.setPayload(create); - - // the create should succeed - catalog.sendNotification(table, createRequest); - - // now craft the malicious metadata file - final String maliciousMetadataFile = tableLocation + "metadata/v2.metadata.json"; - TableMetadata tableMetadata = - TableMetadata.buildFromEmpty() - .assignUUID() - .setLocation(anotherTableLocation) - .addSchema(SCHEMA, 4) - .addPartitionSpec(PartitionSpec.unpartitioned()) - .addSortOrder(SortOrder.unsorted()) - .build(); - TableMetadataParser.write(tableMetadata, catalog.getIo().newOutputFile(maliciousMetadataFile)); - - NotificationRequest updateRequest = new NotificationRequest(); - updateRequest.setNotificationType(NotificationType.UPDATE); - TableUpdateNotification update = new TableUpdateNotification(); - update.setMetadataLocation(maliciousMetadataFile); - update.setTableName(table.name()); - update.setTableUuid(UUID.randomUUID().toString()); - update.setTimestamp(230950849L); - updateRequest.setPayload(update); - - Assertions.assertThatThrownBy(() -> catalog.sendNotification(table, updateRequest)) - .isInstanceOf(BadRequestException.class) - .hasMessageContaining("is not allowed outside of table location"); - } - - @Test - public void testUpdateNotificationCreateTableWithLocalFilePrefix() { - Assumptions.assumeTrue( - requiresNamespaceCreate(), - "Only applicable if namespaces must be created before adding children"); - Assumptions.assumeTrue( - supportsNestedNamespaces(), "Only applicable if nested namespaces are supported"); - Assumptions.assumeTrue( - supportsNotifications(), "Only applicable if notifications are supported"); - - // The location of the metadata JSON file specified in the create will be forbidden. - final String metadataLocation = "file:///etc/metadata.json/../passwd"; - String catalogWithoutStorage = "catalogWithoutStorage"; - PolarisEntity catalogEntity = - adminService.createCatalog( - new CatalogEntity.Builder() - .setDefaultBaseLocation("file://") - .setName(catalogWithoutStorage) - .build()); - - CallContext callContext = CallContext.getCurrentContext(); - PolarisPassthroughResolutionView passthroughView = - new PolarisPassthroughResolutionView( - callContext, entityManager, authenticatedRoot, catalogWithoutStorage); - TaskExecutor taskExecutor = Mockito.mock(); - BasePolarisCatalog catalog = - new BasePolarisCatalog( - entityManager, - metaStoreManager, - callContext, - passthroughView, - authenticatedRoot, - taskExecutor, - new DefaultFileIOFactory()); - catalog.initialize( - catalogWithoutStorage, - ImmutableMap.of( - CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO")); - - Namespace namespace = Namespace.of("parent", "child1"); - TableIdentifier table = TableIdentifier.of(namespace, "table"); - - NotificationRequest request = new NotificationRequest(); - request.setNotificationType(NotificationType.UPDATE); - TableUpdateNotification update = new TableUpdateNotification(); - update.setMetadataLocation(metadataLocation); - update.setTableName(table.name()); - update.setTableUuid(UUID.randomUUID().toString()); - update.setTimestamp(230950845L); - request.setPayload(update); - - InMemoryFileIO fileIO = (InMemoryFileIO) catalog.getIo(); - - fileIO.addFile( - metadataLocation, - TableMetadataParser.toJson(createSampleTableMetadata(metadataLocation)).getBytes(UTF_8)); - - PolarisCallContext polarisCallContext = callContext.getPolarisCallContext(); - if (!polarisCallContext - .getConfigurationStore() - .getConfiguration(polarisCallContext, PolarisConfiguration.SUPPORTED_CATALOG_STORAGE_TYPES) - .contains("FILE")) { - Assertions.assertThatThrownBy(() -> catalog.sendNotification(table, request)) - .isInstanceOf(ForbiddenException.class) - .hasMessageContaining("Invalid location"); - } - } - - @Test - public void testUpdateNotificationCreateTableWithHttpPrefix() { - Assumptions.assumeTrue( - requiresNamespaceCreate(), - "Only applicable if namespaces must be created before adding children"); - Assumptions.assumeTrue( - supportsNestedNamespaces(), "Only applicable if nested namespaces are supported"); - Assumptions.assumeTrue( - supportsNotifications(), "Only applicable if notifications are supported"); - - String catalogName = "catalogForMaliciousDomain"; - adminService.createCatalog( - new CatalogEntity.Builder() - .setDefaultBaseLocation("http://maliciousdomain.com") - .setName(catalogName) - .build()); - - CallContext callContext = CallContext.getCurrentContext(); - PolarisPassthroughResolutionView passthroughView = - new PolarisPassthroughResolutionView( - callContext, entityManager, authenticatedRoot, catalogName); - TaskExecutor taskExecutor = Mockito.mock(); - BasePolarisCatalog catalog = - new BasePolarisCatalog( - entityManager, - metaStoreManager, - callContext, - passthroughView, - authenticatedRoot, - taskExecutor, - new DefaultFileIOFactory()); - catalog.initialize( - catalogName, - ImmutableMap.of( - CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO")); - - Namespace namespace = Namespace.of("parent", "child1"); - TableIdentifier table = TableIdentifier.of(namespace, "table"); - - InMemoryFileIO fileIO = (InMemoryFileIO) catalog.getIo(); - - // The location of the metadata JSON file specified in the create will be forbidden. - final String metadataLocation = "http://maliciousdomain.com/metadata.json"; - NotificationRequest request = new NotificationRequest(); - request.setNotificationType(NotificationType.UPDATE); - TableUpdateNotification update = new TableUpdateNotification(); - update.setMetadataLocation(metadataLocation); - update.setTableName(table.name()); - update.setTableUuid(UUID.randomUUID().toString()); - update.setTimestamp(230950845L); - request.setPayload(update); - - fileIO.addFile( - metadataLocation, - TableMetadataParser.toJson(createSampleTableMetadata(metadataLocation)).getBytes(UTF_8)); - - PolarisCallContext polarisCallContext = callContext.getPolarisCallContext(); - if (!polarisCallContext - .getConfigurationStore() - .getConfiguration(polarisCallContext, PolarisConfiguration.SUPPORTED_CATALOG_STORAGE_TYPES) - .contains("FILE")) { - Assertions.assertThatThrownBy(() -> catalog.sendNotification(table, request)) - .isInstanceOf(ForbiddenException.class) - .hasMessageContaining("Invalid location"); - } - - // It also fails if we try to use https - final String httpsMetadataLocation = "https://maliciousdomain.com/metadata.json"; - NotificationRequest newRequest = new NotificationRequest(); - newRequest.setNotificationType(NotificationType.UPDATE); - newRequest.setPayload( - new TableUpdateNotification( - table.name(), 230950845L, UUID.randomUUID().toString(), httpsMetadataLocation, null)); - - fileIO.addFile( - httpsMetadataLocation, - TableMetadataParser.toJson(createSampleTableMetadata(metadataLocation)).getBytes(UTF_8)); - - if (!polarisCallContext - .getConfigurationStore() - .getConfiguration(polarisCallContext, PolarisConfiguration.SUPPORTED_CATALOG_STORAGE_TYPES) - .contains("FILE")) { - Assertions.assertThatThrownBy(() -> catalog.sendNotification(table, newRequest)) - .isInstanceOf(ForbiddenException.class) - .hasMessageContaining("Invalid location"); - } - } - - @Test - public void testUpdateNotificationWhenNamespacesExist() { - Assumptions.assumeTrue( - requiresNamespaceCreate(), - "Only applicable if namespaces must be created before adding children"); - Assumptions.assumeTrue( - supportsNestedNamespaces(), "Only applicable if nested namespaces are supported"); - Assumptions.assumeTrue( - supportsNotifications(), "Only applicable if notifications are supported"); - - final String tableLocation = "s3://externally-owned-bucket/table/"; - final String tableMetadataLocation = tableLocation + "metadata/v1.metadata.json"; - BasePolarisCatalog catalog = catalog(); - - Namespace namespace = Namespace.of("parent", "child1"); - - createNonExistingNamespaces(namespace); - - TableIdentifier table = TableIdentifier.of(namespace, "table"); - - NotificationRequest request = new NotificationRequest(); - request.setNotificationType(NotificationType.UPDATE); - TableUpdateNotification update = new TableUpdateNotification(); - update.setMetadataLocation(tableMetadataLocation); - update.setTableName(table.name()); - update.setTableUuid(UUID.randomUUID().toString()); - update.setTimestamp(230950845L); - request.setPayload(update); - - InMemoryFileIO fileIO = (InMemoryFileIO) catalog.getIo(); - - fileIO.addFile( - tableMetadataLocation, - TableMetadataParser.toJson(createSampleTableMetadata(tableLocation)).getBytes(UTF_8)); - - Assertions.assertThat(catalog.sendNotification(table, request)) - .as("Notification should be sent successfully") - .isTrue(); - Assertions.assertThat(catalog.namespaceExists(namespace)) - .as("Intermediate namespaces should be created") - .isTrue(); - Assertions.assertThat(catalog.tableExists(table)) - .as("Table should be created on receiving notification") - .isTrue(); - } - - @Test - public void testUpdateNotificationWhenTableExists() { - Assumptions.assumeTrue( - requiresNamespaceCreate(), - "Only applicable if namespaces must be created before adding children"); - Assumptions.assumeTrue( - supportsNestedNamespaces(), "Only applicable if nested namespaces are supported"); - Assumptions.assumeTrue( - supportsNotifications(), "Only applicable if notifications are supported"); - - final String tableLocation = "s3://externally-owned-bucket/table/"; - final String tableMetadataLocation = tableLocation + "metadata/v1.metadata.json"; - BasePolarisCatalog catalog = catalog(); - - Namespace namespace = Namespace.of("parent", "child1"); - - createNonExistingNamespaces(namespace); - - TableIdentifier table = TableIdentifier.of(namespace, "table"); - - catalog.createTable( - table, - new Schema( - Types.NestedField.required(1, "intType", Types.IntegerType.get()), - Types.NestedField.required(2, "stringType", Types.StringType.get()))); - - NotificationRequest request = new NotificationRequest(); - request.setNotificationType(NotificationType.UPDATE); - TableUpdateNotification update = new TableUpdateNotification(); - update.setMetadataLocation(tableMetadataLocation); - update.setTableName(table.name()); - update.setTableUuid(UUID.randomUUID().toString()); - update.setTimestamp(230950845L); - request.setPayload(update); - - InMemoryFileIO fileIO = (InMemoryFileIO) catalog.getIo(); - - fileIO.addFile( - tableMetadataLocation, - TableMetadataParser.toJson(createSampleTableMetadata(tableLocation)).getBytes(UTF_8)); - - Assertions.assertThat(catalog.sendNotification(table, request)) - .as("Notification should be sent successfully") - .isTrue(); - Assertions.assertThat(catalog.namespaceExists(namespace)) - .as("Intermediate namespaces should be created") - .isTrue(); - Assertions.assertThat(catalog.tableExists(table)) - .as("Table should be created on receiving notification") - .isTrue(); - } - - @Test - public void testUpdateNotificationWhenTableExistsInDisallowedLocation() { - Assumptions.assumeTrue( - requiresNamespaceCreate(), - "Only applicable if namespaces must be created before adding children"); - Assumptions.assumeTrue( - supportsNestedNamespaces(), "Only applicable if nested namespaces are supported"); - Assumptions.assumeTrue( - supportsNotifications(), "Only applicable if notifications are supported"); - - // The location of the metadata JSON file specified in the update will be forbidden. - final String tableLocation = "s3://forbidden-table-location/table/"; - final String tableMetadataLocation = tableLocation + "metadata/v1.metadata.json"; - BasePolarisCatalog catalog = catalog(); - - Namespace namespace = Namespace.of("parent", "child1"); - - createNonExistingNamespaces(namespace); - - TableIdentifier table = TableIdentifier.of(namespace, "table"); - - catalog.createTable( - table, - new Schema( - Types.NestedField.required(1, "intType", Types.IntegerType.get()), - Types.NestedField.required(2, "stringType", Types.StringType.get()))); - - NotificationRequest request = new NotificationRequest(); - request.setNotificationType(NotificationType.UPDATE); - TableUpdateNotification update = new TableUpdateNotification(); - update.setMetadataLocation(tableMetadataLocation); - update.setTableName(table.name()); - update.setTableUuid(UUID.randomUUID().toString()); - update.setTimestamp(230950845L); - request.setPayload(update); - - InMemoryFileIO fileIO = (InMemoryFileIO) catalog.getIo(); - - fileIO.addFile( - tableMetadataLocation, - TableMetadataParser.toJson(createSampleTableMetadata(tableLocation)).getBytes(UTF_8)); - - Assertions.assertThatThrownBy(() -> catalog.sendNotification(table, request)) - .isInstanceOf(ForbiddenException.class) - .hasMessageContaining("Invalid location"); - } - - @Test - public void testUpdateNotificationRejectOutOfOrderTimestamp() { - Assumptions.assumeTrue( - requiresNamespaceCreate(), - "Only applicable if namespaces must be created before adding children"); - Assumptions.assumeTrue( - supportsNestedNamespaces(), "Only applicable if nested namespaces are supported"); - Assumptions.assumeTrue( - supportsNotifications(), "Only applicable if notifications are supported"); - - final String tableLocation = "s3://externally-owned-bucket/table/"; - final String tableMetadataLocation = tableLocation + "metadata/v1.metadata.json"; - BasePolarisCatalog catalog = catalog(); - - Namespace namespace = Namespace.of("parent", "child1"); - TableIdentifier table = TableIdentifier.of(namespace, "table"); - - long timestamp = 230950845L; - NotificationRequest request = new NotificationRequest(); - request.setNotificationType(NotificationType.CREATE); - TableUpdateNotification update = new TableUpdateNotification(); - update.setMetadataLocation(tableMetadataLocation); - update.setTableName(table.name()); - update.setTableUuid(UUID.randomUUID().toString()); - update.setTimestamp(timestamp); - request.setPayload(update); - - InMemoryFileIO fileIO = (InMemoryFileIO) catalog.getIo(); - - fileIO.addFile( - tableMetadataLocation, - TableMetadataParser.toJson(createSampleTableMetadata(tableLocation)).getBytes(UTF_8)); - - catalog.sendNotification(table, request); - - // Send a notification with a timestamp same as that of the previous notification, should fail - NotificationRequest request2 = new NotificationRequest(); - request2.setNotificationType(NotificationType.UPDATE); - TableUpdateNotification update2 = new TableUpdateNotification(); - update2.setMetadataLocation(tableLocation + "metadata/v2.metadata.json"); - update2.setTableName(table.name()); - update2.setTableUuid(UUID.randomUUID().toString()); - update2.setTimestamp(timestamp); - request2.setPayload(update2); - - Assertions.assertThatThrownBy(() -> catalog.sendNotification(table, request2)) - .isInstanceOf(AlreadyExistsException.class) - .hasMessageContaining( - "A notification with a newer timestamp has been processed for table parent.child1.table"); - - // Verify that DROP notification won't be rejected due to timestamp - NotificationRequest request3 = new NotificationRequest(); - request3.setNotificationType(NotificationType.DROP); - TableUpdateNotification update3 = new TableUpdateNotification(); - update3.setMetadataLocation(tableLocation + "metadata/v2.metadata.json"); - update3.setTableName(table.name()); - update3.setTableUuid(UUID.randomUUID().toString()); - update3.setTimestamp(timestamp); - request3.setPayload(update3); - - Assertions.assertThat(catalog.sendNotification(table, request3)) - .as("Drop notification should not fail despite timestamp being outdated") - .isTrue(); - } - - @Test - public void testUpdateNotificationWhenTableExistsFileSpecifiesDisallowedLocation() { - Assumptions.assumeTrue( - requiresNamespaceCreate(), - "Only applicable if namespaces must be created before adding children"); - Assumptions.assumeTrue( - supportsNestedNamespaces(), "Only applicable if nested namespaces are supported"); - Assumptions.assumeTrue( - supportsNotifications(), "Only applicable if notifications are supported"); - - final String tableLocation = "s3://externally-owned-bucket/table/"; - final String tableMetadataLocation = tableLocation + "metadata/v1.metadata.json"; - BasePolarisCatalog catalog = catalog(); - - Namespace namespace = Namespace.of("parent", "child1"); - - createNonExistingNamespaces(namespace); - - TableIdentifier table = TableIdentifier.of(namespace, "table"); - - catalog.createTable( - table, - new Schema( - Types.NestedField.required(1, "intType", Types.IntegerType.get()), - Types.NestedField.required(2, "stringType", Types.StringType.get()))); - - NotificationRequest request = new NotificationRequest(); - request.setNotificationType(NotificationType.UPDATE); - TableUpdateNotification update = new TableUpdateNotification(); - update.setMetadataLocation(tableMetadataLocation); - update.setTableName(table.name()); - update.setTableUuid(UUID.randomUUID().toString()); - update.setTimestamp(230950845L); - request.setPayload(update); - - InMemoryFileIO fileIO = (InMemoryFileIO) catalog.getIo(); - - // Though the metadata JSON file itself is in an allowed location, make it internally specify - // a forbidden table location. - TableMetadata forbiddenMetadata = - createSampleTableMetadata("s3://forbidden-table-location/table/"); - fileIO.addFile( - tableMetadataLocation, TableMetadataParser.toJson(forbiddenMetadata).getBytes(UTF_8)); - - Assertions.assertThatThrownBy(() -> catalog.sendNotification(table, request)) - .isInstanceOf(ForbiddenException.class) - .hasMessageContaining("Invalid location"); - } - - @Test - public void testDropNotificationWhenTableAndNamespacesDontExist() { - Assumptions.assumeTrue( - requiresNamespaceCreate(), - "Only applicable if namespaces must be created before adding children"); - Assumptions.assumeTrue( - supportsNestedNamespaces(), "Only applicable if nested namespaces are supported"); - Assumptions.assumeTrue( - supportsNotifications(), "Only applicable if notifications are supported"); - - final String tableLocation = "s3://externally-owned-bucket/table/"; - final String tableMetadataLocation = tableLocation + "metadata/v1.metadata.json"; - BasePolarisCatalog catalog = catalog(); - - Namespace namespace = Namespace.of("parent", "child1"); - TableIdentifier table = TableIdentifier.of(namespace, "table"); - - NotificationRequest request = new NotificationRequest(); - request.setNotificationType(NotificationType.DROP); - TableUpdateNotification update = new TableUpdateNotification(); - update.setMetadataLocation(tableMetadataLocation); - update.setTableName(table.name()); - update.setTableUuid(UUID.randomUUID().toString()); - update.setTimestamp(230950845L); - request.setPayload(update); - - Assertions.assertThat(catalog.sendNotification(table, request)) - .as("Notification should fail since the target table doesn't exist") - .isFalse(); - Assertions.assertThat(catalog.namespaceExists(namespace)) - .as("Intermediate namespaces should not be created") - .isFalse(); - Assertions.assertThat(catalog.tableExists(table)).as("Table should not exist").isFalse(); - } - - @Test - public void testDropNotificationWhenNamespacesExist() { - Assumptions.assumeTrue( - requiresNamespaceCreate(), - "Only applicable if namespaces must be created before adding children"); - Assumptions.assumeTrue( - supportsNestedNamespaces(), "Only applicable if nested namespaces are supported"); - Assumptions.assumeTrue( - supportsNotifications(), "Only applicable if notifications are supported"); - - final String tableLocation = "s3://externally-owned-bucket/table/"; - final String tableMetadataLocation = tableLocation + "metadata/v1.metadata.json"; - BasePolarisCatalog catalog = catalog(); - - Namespace namespace = Namespace.of("parent", "child1"); - - createNonExistingNamespaces(namespace); - - TableIdentifier table = TableIdentifier.of(namespace, "table"); - - NotificationRequest request = new NotificationRequest(); - request.setNotificationType(NotificationType.DROP); - TableUpdateNotification update = new TableUpdateNotification(); - update.setMetadataLocation(tableMetadataLocation); - update.setTableName(table.name()); - update.setTableUuid(UUID.randomUUID().toString()); - update.setTimestamp(230950845L); - request.setPayload(update); - - InMemoryFileIO fileIO = (InMemoryFileIO) catalog.getIo(); - - fileIO.addFile( - tableMetadataLocation, - TableMetadataParser.toJson(createSampleTableMetadata(tableLocation)).getBytes(UTF_8)); - - Assertions.assertThat(catalog.sendNotification(table, request)) - .as("Notification should fail since table doesn't exist") - .isFalse(); - Assertions.assertThat(catalog.namespaceExists(namespace)) - .as("Intermediate namespaces should exist") - .isTrue(); - Assertions.assertThat(catalog.tableExists(table)) - .as("Table should not be created on receiving notification") - .isFalse(); - } - - @Test - public void testDropNotificationWhenTableExists() { - Assumptions.assumeTrue( - requiresNamespaceCreate(), - "Only applicable if namespaces must be created before adding children"); - Assumptions.assumeTrue( - supportsNestedNamespaces(), "Only applicable if nested namespaces are supported"); - Assumptions.assumeTrue( - supportsNotifications(), "Only applicable if notifications are supported"); - - final String tableLocation = "s3://externally-owned-bucket/table/"; - final String tableMetadataLocation = tableLocation + "metadata/v1.metadata.json"; - BasePolarisCatalog catalog = catalog(); - - Namespace namespace = Namespace.of("parent", "child1"); - - createNonExistingNamespaces(namespace); - - TableIdentifier table = TableIdentifier.of(namespace, "table"); - - catalog.createTable( - table, - new Schema( - Types.NestedField.required(1, "intType", Types.IntegerType.get()), - Types.NestedField.required(2, "stringType", Types.StringType.get()))); - - NotificationRequest request = new NotificationRequest(); - request.setNotificationType(NotificationType.DROP); - TableUpdateNotification update = new TableUpdateNotification(); - update.setMetadataLocation(tableMetadataLocation); - update.setTableName(table.name()); - update.setTableUuid(UUID.randomUUID().toString()); - update.setTimestamp(230950845L); - request.setPayload(update); - - InMemoryFileIO fileIO = (InMemoryFileIO) catalog.getIo(); - - fileIO.addFile( - tableMetadataLocation, - TableMetadataParser.toJson(createSampleTableMetadata(tableLocation)).getBytes(UTF_8)); - - Assertions.assertThat(catalog.sendNotification(table, request)) - .as("Notification should be sent successfully") - .isTrue(); - Assertions.assertThat(catalog.namespaceExists(namespace)) - .as("Intermediate namespaces should already exist") - .isTrue(); - Assertions.assertThat(catalog.tableExists(table)) - .as("Table should be dropped on receiving notification") - .isFalse(); - } - - @Test - @Override - public void testDropTableWithPurge() { - if (this.requiresNamespaceCreate()) { - ((SupportsNamespaces) catalog).createNamespace(NS); - } - - Assertions.assertThatPredicate(catalog::tableExists) - .as("Table should not exist before create") - .rejects(TABLE); - - Table table = catalog.buildTable(TABLE, SCHEMA).create(); - Assertions.assertThatPredicate(catalog::tableExists) - .as("Table should exist after create") - .accepts(TABLE); - Assertions.assertThat(table).isInstanceOf(BaseTable.class); - TableMetadata tableMetadata = ((BaseTable) table).operations().current(); - - boolean dropped = catalog.dropTable(TABLE, true); - Assertions.assertThat(dropped) - .as("Should drop a table that does exist", new Object[0]) - .isTrue(); - Assertions.assertThatPredicate(catalog::tableExists) - .as("Table should not exist after drop") - .rejects(TABLE); - List tasks = - metaStoreManager.loadTasks(polarisContext, "testExecutor", 1).getEntities(); - Assertions.assertThat(tasks).hasSize(1); - TaskEntity taskEntity = TaskEntity.of(tasks.get(0)); - EnumMap credentials = - metaStoreManager - .getSubscopedCredsForEntity( - polarisContext, - 0, - taskEntity.getId(), - true, - Set.of(tableMetadata.location()), - Set.of(tableMetadata.location())) - .getCredentials(); - Assertions.assertThat(credentials) - .isNotNull() - .isNotEmpty() - .containsEntry(PolarisCredentialProperty.AWS_KEY_ID, TEST_ACCESS_KEY) - .containsEntry(PolarisCredentialProperty.AWS_SECRET_KEY, SECRET_ACCESS_KEY) - .containsEntry(PolarisCredentialProperty.AWS_TOKEN, SESSION_TOKEN); - FileIO fileIO = - new TaskFileIOSupplier(createMockMetaStoreManagerFactory(), new DefaultFileIOFactory()) - .apply(taskEntity); - Assertions.assertThat(fileIO).isNotNull().isInstanceOf(InMemoryFileIO.class); - } - - @Test - public void testDropTableWithPurgeDisabled() { - // Create a catalog with purge disabled: - String noPurgeCatalogName = CATALOG_NAME + "_no_purge"; - String storageLocation = "s3://testDropTableWithPurgeDisabled/data"; - AwsStorageConfigInfo noPurgeStorageConfigModel = - AwsStorageConfigInfo.builder() - .setRoleArn("arn:aws:iam::012345678901:role/jdoe") - .setExternalId("externalId") - .setUserArn("aws::a:user:arn") - .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) - .build(); - adminService.createCatalog( - new CatalogEntity.Builder() - .setName(noPurgeCatalogName) - .setDefaultBaseLocation(storageLocation) - .setReplaceNewLocationPrefixWithCatalogDefault("file:") - .addProperty(PolarisConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") - .addProperty( - PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") - .addProperty(PolarisConfiguration.DROP_WITH_PURGE_ENABLED.catalogConfig(), "false") - .setStorageConfigurationInfo(noPurgeStorageConfigModel, storageLocation) - .build()); - RealmContext realmContext = () -> "realm"; - CallContext callContext = CallContext.of(realmContext, polarisContext); - PolarisPassthroughResolutionView passthroughView = - new PolarisPassthroughResolutionView( - callContext, entityManager, authenticatedRoot, noPurgeCatalogName); - BasePolarisCatalog noPurgeCatalog = - new BasePolarisCatalog( - entityManager, - metaStoreManager, - callContext, - passthroughView, - authenticatedRoot, - Mockito.mock(), - new DefaultFileIOFactory()); - noPurgeCatalog.initialize( - noPurgeCatalogName, - ImmutableMap.of( - CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO")); - - if (this.requiresNamespaceCreate()) { - ((SupportsNamespaces) noPurgeCatalog).createNamespace(NS); - } - - Assertions.assertThatPredicate(noPurgeCatalog::tableExists) - .as("Table should not exist before create") - .rejects(TABLE); - - Table table = noPurgeCatalog.buildTable(TABLE, SCHEMA).create(); - Assertions.assertThatPredicate(noPurgeCatalog::tableExists) - .as("Table should exist after create") - .accepts(TABLE); - Assertions.assertThat(table).isInstanceOf(BaseTable.class); - - // Attempt to drop the table: - Assertions.assertThatThrownBy(() -> noPurgeCatalog.dropTable(TABLE, true)) - .isInstanceOf(ForbiddenException.class) - .hasMessageContaining(PolarisConfiguration.DROP_WITH_PURGE_ENABLED.key); - } - - private TableMetadata createSampleTableMetadata(String tableLocation) { - Schema schema = - new Schema( - Types.NestedField.required(1, "intType", Types.IntegerType.get()), - Types.NestedField.required(2, "stringType", Types.StringType.get())); - PartitionSpec partitionSpec = - PartitionSpec.builderFor(schema).identity("intType").withSpecId(1000).build(); - - return TableMetadata.newTableMetadata(schema, partitionSpec, tableLocation, ImmutableMap.of()); - } - - private void createNonExistingNamespaces(Namespace namespace) { - // Pre-create namespaces if they don't exist - for (int i = 1; i <= namespace.length(); i++) { - Namespace nsLevel = Namespace.of(Arrays.copyOf(namespace.levels(), i)); - if (!catalog.namespaceExists(nsLevel)) { - catalog.createNamespace(nsLevel); - } - } - } - - @Test - public void testRetriableException() { - RuntimeException s3Exception = new RuntimeException("Access Denied"); - RuntimeException azureBlobStorageException = - new RuntimeException( - "This request is not authorized to perform this operation using this permission"); - RuntimeException gcsException = new RuntimeException("Forbidden"); - RuntimeException otherException = new RuntimeException(new IOException("Connection reset")); - Assertions.assertThat(BasePolarisCatalog.SHOULD_RETRY_REFRESH_PREDICATE.test(s3Exception)) - .isFalse(); - Assertions.assertThat( - BasePolarisCatalog.SHOULD_RETRY_REFRESH_PREDICATE.test(azureBlobStorageException)) - .isFalse(); - Assertions.assertThat(BasePolarisCatalog.SHOULD_RETRY_REFRESH_PREDICATE.test(gcsException)) - .isFalse(); - Assertions.assertThat(BasePolarisCatalog.SHOULD_RETRY_REFRESH_PREDICATE.test(otherException)) - .isTrue(); - } - - @Test - public void testFileIOWrapper() { - RealmContext realmContext = () -> "realm"; - CallContext callContext = CallContext.of(realmContext, polarisContext); - CallContext.setCurrentContext(callContext); - PolarisPassthroughResolutionView passthroughView = - new PolarisPassthroughResolutionView( - callContext, entityManager, authenticatedRoot, CATALOG_NAME); - - MeasuredFileIOFactory measured = new MeasuredFileIOFactory(); - BasePolarisCatalog catalog = - new BasePolarisCatalog( - entityManager, - metaStoreManager, - callContext, - passthroughView, - authenticatedRoot, - Mockito.mock(), - measured); - catalog.initialize( - CATALOG_NAME, - ImmutableMap.of( - CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO")); - - Assertions.assertThat(measured.getNumOutputFiles() + measured.getInputBytes()) - .as("Nothing was created yet") - .isEqualTo(0); - - catalog.createNamespace(NS); - Table table = catalog.buildTable(TABLE, SCHEMA).create(); - - // Asserting greaterThan 0 is sufficient for validating that the wrapper works without making - // assumptions about the - // specific implementations of table operations. - Assertions.assertThat(measured.getNumOutputFiles()).as("A table was created").isGreaterThan(0); - - table.updateProperties().set("foo", "bar").commit(); - Assertions.assertThat(measured.getInputBytes()) - .as("A table was read and written") - .isGreaterThan(0); - - Assertions.assertThat(catalog.dropTable(TABLE)).as("Table deletion should succeed").isTrue(); - TableCleanupTaskHandler handler = - new TableCleanupTaskHandler( - Mockito.mock(), - createMockMetaStoreManagerFactory(), - (task) -> measured.loadFileIO("org.apache.iceberg.inmemory.InMemoryFileIO", Map.of())); - handler.handleTask( - TaskEntity.of( - metaStoreManager - .loadTasks(polarisContext, "testExecutor", 1) - .getEntities() - .getFirst())); - Assertions.assertThat(measured.getNumDeletedFiles()).as("A table was deleted").isGreaterThan(0); - } -} diff --git a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogViewTest.java b/polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogViewTest.java deleted file mode 100644 index f032aff1f..000000000 --- a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogViewTest.java +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.catalog; - -import com.google.common.collect.ImmutableMap; -import io.quarkus.test.junit.QuarkusMock; -import io.quarkus.test.junit.QuarkusTest; -import jakarta.inject.Inject; -import java.io.IOException; -import java.lang.reflect.Field; -import java.lang.reflect.Method; -import java.nio.file.Path; -import java.time.Clock; -import java.util.List; -import java.util.Set; -import org.apache.iceberg.CatalogProperties; -import org.apache.iceberg.catalog.Catalog; -import org.apache.iceberg.view.ViewCatalogTests; -import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.PolarisConfiguration; -import org.apache.polaris.core.PolarisConfigurationStore; -import org.apache.polaris.core.PolarisDiagnostics; -import org.apache.polaris.core.admin.model.FileStorageConfigInfo; -import org.apache.polaris.core.admin.model.StorageConfigInfo; -import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; -import org.apache.polaris.core.auth.PolarisAuthorizerImpl; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.core.entity.CatalogEntity; -import org.apache.polaris.core.entity.PolarisEntity; -import org.apache.polaris.core.entity.PolarisEntitySubType; -import org.apache.polaris.core.entity.PolarisEntityType; -import org.apache.polaris.core.entity.PrincipalEntity; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.core.persistence.PolarisEntityManager; -import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.storage.cache.StorageCredentialCache; -import org.apache.polaris.service.admin.PolarisAdminService; -import org.apache.polaris.service.catalog.io.DefaultFileIOFactory; -import org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.TestInfo; -import org.junit.jupiter.api.io.TempDir; -import org.mockito.Mockito; - -@QuarkusTest -public class BasePolarisCatalogViewTest extends ViewCatalogTests { - public static final String CATALOG_NAME = "polaris-catalog"; - - @Inject MetaStoreManagerFactory managerFactory; - @Inject PolarisConfigurationStore configurationStore; - @Inject PolarisDiagnostics diagServices; - - private BasePolarisCatalog catalog; - - private String realmName; - private PolarisMetaStoreManager metaStoreManager; - private PolarisCallContext polarisContext; - - @BeforeAll - public static void setUpMocks() { - PolarisStorageIntegrationProviderImpl mock = - Mockito.mock(PolarisStorageIntegrationProviderImpl.class); - QuarkusMock.installMockForType(mock, PolarisStorageIntegrationProviderImpl.class); - } - - @BeforeEach - public void setUpTempDir(@TempDir Path tempDir) throws Exception { - // see https://github.com/quarkusio/quarkus/issues/13261 - Field field = ViewCatalogTests.class.getDeclaredField("tempDir"); - field.setAccessible(true); - field.set(this, tempDir); - } - - @BeforeEach - public void before(TestInfo testInfo) { - realmName = - "realm_%s_%s" - .formatted( - testInfo.getTestMethod().map(Method::getName).orElse("test"), System.nanoTime()); - RealmContext realmContext = () -> realmName; - - metaStoreManager = managerFactory.getOrCreateMetaStoreManager(realmContext); - polarisContext = - new PolarisCallContext( - managerFactory.getOrCreateSessionSupplier(realmContext).get(), - diagServices, - configurationStore, - Clock.systemDefaultZone()); - - PolarisEntityManager entityManager = - new PolarisEntityManager(metaStoreManager, new StorageCredentialCache()); - - CallContext callContext = CallContext.of(realmContext, polarisContext); - CallContext.setCurrentContext(callContext); - - PrincipalEntity rootEntity = - new PrincipalEntity( - PolarisEntity.of( - metaStoreManager - .readEntityByName( - polarisContext, - null, - PolarisEntityType.PRINCIPAL, - PolarisEntitySubType.NULL_SUBTYPE, - "root") - .getEntity())); - AuthenticatedPolarisPrincipal authenticatedRoot = - new AuthenticatedPolarisPrincipal(rootEntity, Set.of()); - - PolarisAdminService adminService = - new PolarisAdminService( - callContext, - entityManager, - metaStoreManager, - authenticatedRoot, - new PolarisAuthorizerImpl(new PolarisConfigurationStore() {})); - adminService.createCatalog( - new CatalogEntity.Builder() - .setName(CATALOG_NAME) - .addProperty(PolarisConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") - .addProperty( - PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") - .setDefaultBaseLocation("file://tmp") - .setStorageConfigurationInfo( - new FileStorageConfigInfo( - StorageConfigInfo.StorageTypeEnum.FILE, List.of("file://", "/", "*")), - "file://tmp") - .build()); - - PolarisPassthroughResolutionView passthroughView = - new PolarisPassthroughResolutionView( - callContext, entityManager, authenticatedRoot, CATALOG_NAME); - this.catalog = - new BasePolarisCatalog( - entityManager, - metaStoreManager, - callContext, - passthroughView, - authenticatedRoot, - Mockito.mock(), - new DefaultFileIOFactory()); - this.catalog.initialize( - CATALOG_NAME, - ImmutableMap.of( - CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO")); - } - - @AfterEach - public void after() throws IOException { - catalog().close(); - metaStoreManager.purge(polarisContext); - } - - @Override - protected BasePolarisCatalog catalog() { - return catalog; - } - - @Override - protected Catalog tableCatalog() { - return catalog; - } - - @Override - protected boolean requiresNamespaceCreate() { - return true; - } -} diff --git a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapperAuthzTest.java b/polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapperAuthzTest.java deleted file mode 100644 index e1a608217..000000000 --- a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapperAuthzTest.java +++ /dev/null @@ -1,1829 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.catalog; - -import com.google.common.collect.ImmutableMap; -import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.QuarkusTestProfile; -import io.quarkus.test.junit.TestProfile; -import java.time.Instant; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import org.apache.hadoop.conf.Configuration; -import org.apache.iceberg.CatalogProperties; -import org.apache.iceberg.CatalogUtil; -import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.SortOrder; -import org.apache.iceberg.TableMetadata; -import org.apache.iceberg.TableMetadataParser; -import org.apache.iceberg.catalog.Catalog; -import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.exceptions.ForbiddenException; -import org.apache.iceberg.io.FileIO; -import org.apache.iceberg.rest.requests.CommitTransactionRequest; -import org.apache.iceberg.rest.requests.CreateNamespaceRequest; -import org.apache.iceberg.rest.requests.CreateTableRequest; -import org.apache.iceberg.rest.requests.CreateViewRequest; -import org.apache.iceberg.rest.requests.ImmutableCreateViewRequest; -import org.apache.iceberg.rest.requests.RegisterTableRequest; -import org.apache.iceberg.rest.requests.RenameTableRequest; -import org.apache.iceberg.rest.requests.UpdateNamespacePropertiesRequest; -import org.apache.iceberg.rest.requests.UpdateTableRequest; -import org.apache.iceberg.view.ImmutableSQLViewRepresentation; -import org.apache.iceberg.view.ImmutableViewVersion; -import org.apache.polaris.core.admin.model.FileStorageConfigInfo; -import org.apache.polaris.core.admin.model.PrincipalWithCredentialsCredentials; -import org.apache.polaris.core.admin.model.StorageConfigInfo; -import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.core.entity.CatalogEntity; -import org.apache.polaris.core.entity.CatalogRoleEntity; -import org.apache.polaris.core.entity.PolarisPrivilege; -import org.apache.polaris.core.entity.PrincipalEntity; -import org.apache.polaris.core.persistence.PolarisEntityManager; -import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest; -import org.apache.polaris.service.admin.PolarisAuthzTestBase; -import org.apache.polaris.service.catalog.io.DefaultFileIOFactory; -import org.apache.polaris.service.config.RealmEntityManagerFactory; -import org.apache.polaris.service.context.CallContextCatalogFactory; -import org.apache.polaris.service.context.PolarisCallContextCatalogFactory; -import org.apache.polaris.service.types.NotificationRequest; -import org.apache.polaris.service.types.NotificationType; -import org.apache.polaris.service.types.TableUpdateNotification; -import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -@QuarkusTest -@TestProfile(PolarisCatalogHandlerWrapperAuthzTest.Profile.class) -public class PolarisCatalogHandlerWrapperAuthzTest extends PolarisAuthzTestBase { - private PolarisCatalogHandlerWrapper newWrapper() { - return newWrapper(Set.of()); - } - - private PolarisCatalogHandlerWrapper newWrapper(Set activatedPrincipalRoles) { - return newWrapper(activatedPrincipalRoles, CATALOG_NAME, callContextCatalogFactory); - } - - private PolarisCatalogHandlerWrapper newWrapper( - Set activatedPrincipalRoles, String catalogName, CallContextCatalogFactory factory) { - final AuthenticatedPolarisPrincipal authenticatedPrincipal = - new AuthenticatedPolarisPrincipal(principalEntity, activatedPrincipalRoles); - return new PolarisCatalogHandlerWrapper( - callContext, - entityManager, - metaStoreManager, - authenticatedPrincipal, - factory, - catalogName, - polarisAuthorizer); - } - - /** - * Tests each "sufficient" privilege individually using CATALOG_ROLE1 by granting at the - * CATALOG_NAME level, revoking after each test, and also ensuring that the request fails after - * revocation. - * - * @param sufficientPrivileges List of privileges that should be sufficient each in isolation for - * {@code action} to succeed. - * @param action The operation being tested; could also be multiple operations that should all - * succeed with the sufficient privilege - * @param cleanupAction If non-null, additional action to run to "undo" a previous success action - * in case the action has side effects. Called before revoking the sufficient privilege; - * either the cleanup privileges must be latent, or the cleanup action could be run with - * PRINCIPAL_ROLE2 while runnint {@code action} with PRINCIPAL_ROLE1. - */ - private void doTestSufficientPrivileges( - List sufficientPrivileges, Runnable action, Runnable cleanupAction) { - doTestSufficientPrivilegeSets( - sufficientPrivileges.stream().map(priv -> Set.of(priv)).toList(), - action, - cleanupAction, - PRINCIPAL_NAME); - } - - /** - * @param sufficientPrivileges each set of concurrent privileges expected to be sufficient - * together. - * @param action - * @param cleanupAction - * @param principalName - */ - private void doTestSufficientPrivilegeSets( - List> sufficientPrivileges, - Runnable action, - Runnable cleanupAction, - String principalName) { - doTestSufficientPrivilegeSets( - sufficientPrivileges, action, cleanupAction, principalName, CATALOG_NAME); - } - - /** - * @param sufficientPrivileges each set of concurrent privileges expected to be sufficient - * together. - * @param action - * @param cleanupAction - * @param principalName - * @param catalogName - */ - private void doTestSufficientPrivilegeSets( - List> sufficientPrivileges, - Runnable action, - Runnable cleanupAction, - String principalName, - String catalogName) { - doTestSufficientPrivilegeSets( - sufficientPrivileges, - action, - cleanupAction, - principalName, - (privilege) -> - adminService.grantPrivilegeOnCatalogToRole(catalogName, CATALOG_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnCatalogFromRole(catalogName, CATALOG_ROLE1, privilege)); - } - - private void doTestInsufficientPrivileges( - List insufficientPrivileges, Runnable action) { - doTestInsufficientPrivileges(insufficientPrivileges, PRINCIPAL_NAME, action); - } - - /** - * Tests each "insufficient" privilege individually using CATALOG_ROLE1 by granting at the - * CATALOG_NAME level, ensuring the action fails, then revoking after each test case. - */ - private void doTestInsufficientPrivileges( - List insufficientPrivileges, String principalName, Runnable action) { - doTestInsufficientPrivileges( - insufficientPrivileges, - principalName, - action, - (privilege) -> - adminService.grantPrivilegeOnCatalogToRole(CATALOG_NAME, CATALOG_ROLE1, privilege), - (privilege) -> - adminService.revokePrivilegeOnCatalogFromRole(CATALOG_NAME, CATALOG_ROLE1, privilege)); - } - - @Test - public void testListNamespacesAllSufficientPrivileges() { - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_LIST, - PolarisPrivilege.NAMESPACE_READ_PROPERTIES, - PolarisPrivilege.NAMESPACE_WRITE_PROPERTIES, - PolarisPrivilege.NAMESPACE_CREATE, - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT), - () -> newWrapper().listNamespaces(Namespace.of()), - null /* cleanupAction */); - } - - @Test - public void testListNamespacesInsufficientPermissions() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.NAMESPACE_DROP), - () -> newWrapper().listNamespaces(Namespace.of())); - } - - @Test - public void testInsufficientPermissionsPriorToSecretRotation() { - String principalName = "all_the_powers"; - PolarisMetaStoreManager.CreatePrincipalResult newPrincipal = - metaStoreManager.createPrincipal( - callContext.getPolarisCallContext(), - new PrincipalEntity.Builder() - .setName(principalName) - .setCreateTimestamp(Instant.now().toEpochMilli()) - .setCredentialRotationRequiredState() - .build()); - adminService.assignPrincipalRole(principalName, PRINCIPAL_ROLE1); - adminService.assignPrincipalRole(principalName, PRINCIPAL_ROLE2); - - final AuthenticatedPolarisPrincipal authenticatedPrincipal = - new AuthenticatedPolarisPrincipal( - PrincipalEntity.of(newPrincipal.getPrincipal()), Set.of()); - PolarisCatalogHandlerWrapper wrapper = - new PolarisCatalogHandlerWrapper( - callContext, - entityManager, - metaStoreManager, - authenticatedPrincipal, - callContextCatalogFactory, - CATALOG_NAME, - polarisAuthorizer); - - // a variety of actions are all disallowed because the principal's credentials must be rotated - doTestInsufficientPrivileges( - List.of(PolarisPrivilege.values()), - principalName, - () -> wrapper.listNamespaces(Namespace.of())); - Namespace ns3 = Namespace.of("ns3"); - doTestInsufficientPrivileges( - List.of(PolarisPrivilege.values()), - principalName, - () -> wrapper.createNamespace(CreateNamespaceRequest.builder().withNamespace(ns3).build())); - doTestInsufficientPrivileges( - List.of(PolarisPrivilege.values()), principalName, () -> wrapper.listTables(NS1)); - PrincipalWithCredentialsCredentials credentials = - new PrincipalWithCredentialsCredentials( - newPrincipal.getPrincipalSecrets().getPrincipalClientId(), - newPrincipal.getPrincipalSecrets().getMainSecret()); - PrincipalEntity refreshPrincipal = - rotateAndRefreshPrincipal( - metaStoreManager, principalName, credentials, callContext.getPolarisCallContext()); - final AuthenticatedPolarisPrincipal authenticatedPrincipal1 = - new AuthenticatedPolarisPrincipal(PrincipalEntity.of(refreshPrincipal), Set.of()); - PolarisCatalogHandlerWrapper refreshedWrapper = - new PolarisCatalogHandlerWrapper( - callContext, - entityManager, - metaStoreManager, - authenticatedPrincipal1, - callContextCatalogFactory, - CATALOG_NAME, - polarisAuthorizer); - - doTestSufficientPrivilegeSets( - List.of(Set.of(PolarisPrivilege.NAMESPACE_LIST)), - () -> refreshedWrapper.listNamespaces(Namespace.of()), - null, - principalName); - doTestSufficientPrivilegeSets( - List.of(Set.of(PolarisPrivilege.NAMESPACE_CREATE)), - () -> - refreshedWrapper.createNamespace( - CreateNamespaceRequest.builder().withNamespace(ns3).build()), - null, - principalName); - doTestSufficientPrivilegeSets( - List.of(Set.of(PolarisPrivilege.TABLE_LIST)), - () -> refreshedWrapper.listTables(ns3), - null, - principalName); - } - - @Test - public void testListNamespacesCatalogLevelWithPrincipalRoleActivation() { - // Grant catalog-level privilege to CATALOG_ROLE1 - Assertions.assertThat( - adminService.grantPrivilegeOnCatalogToRole( - CATALOG_NAME, CATALOG_ROLE1, PolarisPrivilege.NAMESPACE_LIST)) - .isTrue(); - Assertions.assertThat(newWrapper().listNamespaces(Namespace.of()).namespaces()) - .containsAll(List.of(NS1, NS2)); - - // Just activating PRINCIPAL_ROLE1 should also work. - Assertions.assertThat( - newWrapper(Set.of(PRINCIPAL_ROLE1)).listNamespaces(Namespace.of()).namespaces()) - .containsAll(List.of(NS1, NS2)); - - // If we only activate PRINCIPAL_ROLE2 it won't have the privilege. - Assertions.assertThatThrownBy( - () -> newWrapper(Set.of(PRINCIPAL_ROLE2)).listNamespaces(Namespace.of())) - .isInstanceOf(ForbiddenException.class) - .hasMessageContaining("is not authorized"); - - // If we revoke, then it should fail again even with all principal roles activated. - Assertions.assertThat( - adminService.revokePrivilegeOnCatalogFromRole( - CATALOG_NAME, CATALOG_ROLE1, PolarisPrivilege.NAMESPACE_LIST)) - .isTrue(); - Assertions.assertThatThrownBy(() -> newWrapper().listNamespaces(Namespace.of())) - .isInstanceOf(ForbiddenException.class); - } - - @Test - public void testListNamespacesChildOnly() { - // Grant only NS1-level privilege to CATALOG_ROLE1 - Assertions.assertThat( - adminService.grantPrivilegeOnNamespaceToRole( - CATALOG_NAME, CATALOG_ROLE1, NS1, PolarisPrivilege.NAMESPACE_LIST)) - .isTrue(); - - // Listing directly on NS1 succeeds - Assertions.assertThat(newWrapper().listNamespaces(NS1).namespaces()) - .containsAll(List.of(NS1A, NS1B)); - - // Root listing fails - Assertions.assertThatThrownBy(() -> newWrapper().listNamespaces(Namespace.of())) - .isInstanceOf(ForbiddenException.class); - - // NS2 listing fails - Assertions.assertThatThrownBy(() -> newWrapper().listNamespaces(Namespace.of())) - .isInstanceOf(ForbiddenException.class); - - // Listing on a child of NS1 succeeds - Assertions.assertThat(newWrapper().listNamespaces(NS1A).namespaces()) - .containsAll(List.of(NS1AA)); - } - - @Test - public void testCreateNamespaceAllSufficientPrivileges() { - Assertions.assertThat( - adminService.grantPrivilegeOnCatalogToRole( - CATALOG_NAME, CATALOG_ROLE2, PolarisPrivilege.NAMESPACE_DROP)) - .isTrue(); - - // Use PRINCIPAL_ROLE1 for privilege-testing, PRINCIPAL_ROLE2 for cleanup. - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_CREATE, - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT), - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE1)) - .createNamespace( - CreateNamespaceRequest.builder().withNamespace(Namespace.of("newns")).build()); - newWrapper(Set.of(PRINCIPAL_ROLE1)) - .createNamespace( - CreateNamespaceRequest.builder() - .withNamespace(Namespace.of("ns1", "ns1a", "newns")) - .build()); - }, - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE2)).dropNamespace(Namespace.of("newns")); - newWrapper(Set.of(PRINCIPAL_ROLE2)).dropNamespace(Namespace.of("ns1", "ns1a", "newns")); - }); - } - - @Test - public void testCreateNamespacesInsufficientPermissions() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.NAMESPACE_DROP, - PolarisPrivilege.NAMESPACE_READ_PROPERTIES, - PolarisPrivilege.NAMESPACE_WRITE_PROPERTIES, - PolarisPrivilege.NAMESPACE_LIST), - () -> - newWrapper() - .createNamespace( - CreateNamespaceRequest.builder().withNamespace(Namespace.of("newns")).build())); - } - - @Test - public void testLoadNamespaceMetadataSufficientPrivileges() { - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_READ_PROPERTIES, - PolarisPrivilege.NAMESPACE_WRITE_PROPERTIES, - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT), - () -> newWrapper().loadNamespaceMetadata(NS1A), - null /* cleanupAction */); - } - - @Test - public void testLoadNamespaceMetadataInsufficientPermissions() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.NAMESPACE_CREATE, - PolarisPrivilege.NAMESPACE_LIST, - PolarisPrivilege.NAMESPACE_DROP), - () -> newWrapper().loadNamespaceMetadata(NS1A)); - } - - @Test - public void testNamespaceExistsAllSufficientPrivileges() { - // TODO: If we change the behavior of existence-check to return 404 on unauthorized, - // the overall test structure will need to change (other tests catching ForbiddenException - // need to still have catalog-level "REFERENCE" equivalent privileges, and the exists() - // tests need to expect 404 instead). - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_LIST, - PolarisPrivilege.NAMESPACE_READ_PROPERTIES, - PolarisPrivilege.NAMESPACE_WRITE_PROPERTIES, - PolarisPrivilege.NAMESPACE_CREATE, - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT), - () -> newWrapper().namespaceExists(NS1A), - null /* cleanupAction */); - } - - @Test - public void testNamespaceExistsInsufficientPermissions() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.NAMESPACE_DROP), - () -> newWrapper().namespaceExists(NS1A)); - } - - @Test - public void testDropNamespaceSufficientPrivileges() { - Assertions.assertThat( - adminService.grantPrivilegeOnCatalogToRole( - CATALOG_NAME, CATALOG_ROLE2, PolarisPrivilege.NAMESPACE_CREATE)) - .isTrue(); - - // Use PRINCIPAL_ROLE1 for privilege-testing, PRINCIPAL_ROLE2 for cleanup. - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_DROP, - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT), - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE1)).dropNamespace(NS1AA); - }, - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE2)) - .createNamespace(CreateNamespaceRequest.builder().withNamespace(NS1AA).build()); - }); - } - - @Test - public void testDropNamespaceInsufficientPermissions() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.NAMESPACE_CREATE, - PolarisPrivilege.NAMESPACE_LIST, - PolarisPrivilege.NAMESPACE_READ_PROPERTIES, - PolarisPrivilege.NAMESPACE_WRITE_PROPERTIES), - () -> newWrapper().dropNamespace(NS1AA)); - } - - @Test - public void testUpdateNamespacePropertiesAllSufficientPrivileges() { - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_WRITE_PROPERTIES, - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT), - () -> { - newWrapper() - .updateNamespaceProperties( - NS1A, UpdateNamespacePropertiesRequest.builder().update("foo", "bar").build()); - newWrapper() - .updateNamespaceProperties( - NS1A, UpdateNamespacePropertiesRequest.builder().remove("foo").build()); - }, - null /* cleanupAction */); - } - - @Test - public void testUpdateNamespacePropertiesInsufficientPermissions() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.NAMESPACE_LIST, - PolarisPrivilege.NAMESPACE_READ_PROPERTIES, - PolarisPrivilege.NAMESPACE_CREATE, - PolarisPrivilege.NAMESPACE_DROP), - () -> - newWrapper() - .updateNamespaceProperties( - NS1A, UpdateNamespacePropertiesRequest.builder().update("foo", "bar").build())); - } - - @Test - public void testListTablesAllSufficientPrivileges() { - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.TABLE_LIST, - PolarisPrivilege.TABLE_READ_PROPERTIES, - PolarisPrivilege.TABLE_WRITE_PROPERTIES, - PolarisPrivilege.TABLE_READ_DATA, - PolarisPrivilege.TABLE_WRITE_DATA, - PolarisPrivilege.TABLE_CREATE, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT), - () -> newWrapper().listTables(NS1A), - null /* cleanupAction */); - } - - @Test - public void testListTablesInsufficientPermissions() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.TABLE_DROP), - () -> newWrapper().listTables(NS1A)); - } - - @Test - public void testCreateTableDirectAllSufficientPrivileges() { - Assertions.assertThat( - adminService.grantPrivilegeOnCatalogToRole( - CATALOG_NAME, CATALOG_ROLE2, PolarisPrivilege.TABLE_DROP)) - .isTrue(); - Assertions.assertThat( - adminService.grantPrivilegeOnCatalogToRole( - CATALOG_NAME, CATALOG_ROLE2, PolarisPrivilege.TABLE_WRITE_DATA)) - .isTrue(); - - final TableIdentifier newtable = TableIdentifier.of(NS2, "newtable"); - final CreateTableRequest createRequest = - CreateTableRequest.builder().withName("newtable").withSchema(SCHEMA).build(); - - // Use PRINCIPAL_ROLE1 for privilege-testing, PRINCIPAL_ROLE2 for cleanup. - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.TABLE_CREATE, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT), - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE1)).createTableDirect(NS2, createRequest); - }, - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE2)).dropTableWithPurge(newtable); - }); - } - - @Test - public void testCreateTableDirectInsufficientPermissions() { - final CreateTableRequest createRequest = - CreateTableRequest.builder().withName("newtable").withSchema(SCHEMA).build(); - - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.TABLE_DROP, - PolarisPrivilege.TABLE_READ_PROPERTIES, - PolarisPrivilege.TABLE_WRITE_PROPERTIES, - PolarisPrivilege.TABLE_READ_DATA, - PolarisPrivilege.TABLE_WRITE_DATA, - PolarisPrivilege.TABLE_LIST), - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE1)).createTableDirect(NS2, createRequest); - }); - } - - @Test - public void testCreateTableDirectWithWriteDelegationAllSufficientPrivileges() { - Assertions.assertThat( - adminService.grantPrivilegeOnCatalogToRole( - CATALOG_NAME, CATALOG_ROLE2, PolarisPrivilege.TABLE_DROP)) - .isTrue(); - Assertions.assertThat( - adminService.grantPrivilegeOnCatalogToRole( - CATALOG_NAME, CATALOG_ROLE2, PolarisPrivilege.TABLE_WRITE_DATA)) - .isTrue(); - - final TableIdentifier newtable = TableIdentifier.of(NS2, "newtable"); - final CreateTableRequest createDirectWithWriteDelegationRequest = - CreateTableRequest.builder().withName("newtable").withSchema(SCHEMA).stageCreate().build(); - - doTestSufficientPrivilegeSets( - List.of( - Set.of(PolarisPrivilege.TABLE_CREATE, PolarisPrivilege.TABLE_WRITE_DATA), - Set.of(PolarisPrivilege.CATALOG_MANAGE_CONTENT)), - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE1)) - .createTableDirectWithWriteDelegation(NS2, createDirectWithWriteDelegationRequest); - }, - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE2)).dropTableWithPurge(newtable); - }, - PRINCIPAL_NAME); - } - - @Test - public void testCreateTableDirectWithWriteDelegationInsufficientPermissions() { - final CreateTableRequest createDirectWithWriteDelegationRequest = - CreateTableRequest.builder() - .withName("directtable") - .withSchema(SCHEMA) - .stageCreate() - .build(); - - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.TABLE_DROP, - PolarisPrivilege.TABLE_CREATE, // TABLE_CREATE itself is insufficient for delegation - PolarisPrivilege.TABLE_READ_PROPERTIES, - PolarisPrivilege.TABLE_WRITE_PROPERTIES, - PolarisPrivilege.TABLE_READ_DATA, - PolarisPrivilege.TABLE_WRITE_DATA, - PolarisPrivilege.TABLE_LIST), - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE1)) - .createTableDirectWithWriteDelegation(NS2, createDirectWithWriteDelegationRequest); - }); - } - - @Test - public void testCreateTableStagedAllSufficientPrivileges() { - Assertions.assertThat( - adminService.grantPrivilegeOnCatalogToRole( - CATALOG_NAME, CATALOG_ROLE2, PolarisPrivilege.TABLE_DROP)) - .isTrue(); - - final CreateTableRequest createStagedRequest = - CreateTableRequest.builder() - .withName("stagetable") - .withSchema(SCHEMA) - .stageCreate() - .build(); - - // Use PRINCIPAL_ROLE1 for privilege-testing, PRINCIPAL_ROLE2 for cleanup. - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.TABLE_CREATE, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT), - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE1)).createTableStaged(NS2, createStagedRequest); - }, - // createTableStaged doesn't actually commit any metadata - null); - } - - @Test - public void testCreateTableStagedInsufficientPermissions() { - final CreateTableRequest createStagedRequest = - CreateTableRequest.builder() - .withName("stagetable") - .withSchema(SCHEMA) - .stageCreate() - .build(); - - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.TABLE_DROP, - PolarisPrivilege.TABLE_READ_PROPERTIES, - PolarisPrivilege.TABLE_WRITE_PROPERTIES, - PolarisPrivilege.TABLE_READ_DATA, - PolarisPrivilege.TABLE_WRITE_DATA, - PolarisPrivilege.TABLE_LIST), - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE1)).createTableStaged(NS2, createStagedRequest); - }); - } - - @Test - public void testCreateTableStagedWithWriteDelegationAllSufficientPrivileges() { - Assertions.assertThat( - adminService.grantPrivilegeOnCatalogToRole( - CATALOG_NAME, CATALOG_ROLE2, PolarisPrivilege.TABLE_DROP)) - .isTrue(); - - final CreateTableRequest createStagedWithWriteDelegationRequest = - CreateTableRequest.builder() - .withName("stagetable") - .withSchema(SCHEMA) - .stageCreate() - .build(); - - doTestSufficientPrivilegeSets( - List.of( - Set.of(PolarisPrivilege.TABLE_CREATE, PolarisPrivilege.TABLE_WRITE_DATA), - Set.of(PolarisPrivilege.CATALOG_MANAGE_CONTENT)), - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE1)) - .createTableStagedWithWriteDelegation(NS2, createStagedWithWriteDelegationRequest); - }, - // createTableStagedWithWriteDelegation doesn't actually commit any metadata - null, - PRINCIPAL_NAME); - } - - @Test - public void testCreateTableStagedWithWriteDelegationInsufficientPermissions() { - final CreateTableRequest createStagedWithWriteDelegationRequest = - CreateTableRequest.builder() - .withName("stagetable") - .withSchema(SCHEMA) - .stageCreate() - .build(); - - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.TABLE_DROP, - PolarisPrivilege.TABLE_CREATE, // TABLE_CREATE itself is insufficient for delegation - PolarisPrivilege.TABLE_READ_PROPERTIES, - PolarisPrivilege.TABLE_WRITE_PROPERTIES, - PolarisPrivilege.TABLE_READ_DATA, - PolarisPrivilege.TABLE_WRITE_DATA, - PolarisPrivilege.TABLE_LIST), - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE1)) - .createTableStagedWithWriteDelegation(NS2, createStagedWithWriteDelegationRequest); - }); - } - - @Test - public void testRegisterTableAllSufficientPrivileges() { - Assertions.assertThat( - adminService.grantPrivilegeOnCatalogToRole( - CATALOG_NAME, CATALOG_ROLE2, PolarisPrivilege.TABLE_DROP)) - .isTrue(); - Assertions.assertThat( - adminService.grantPrivilegeOnCatalogToRole( - CATALOG_NAME, CATALOG_ROLE2, PolarisPrivilege.TABLE_READ_PROPERTIES)) - .isTrue(); - - // To get a handy metadata file we can use one from another table. - // to avoid overlapping directories, drop the original table and recreate it via registerTable - final String metadataLocation = newWrapper().loadTable(TABLE_NS1_1, "all").metadataLocation(); - newWrapper(Set.of(PRINCIPAL_ROLE2)).dropTableWithoutPurge(TABLE_NS1_1); - - final RegisterTableRequest registerRequest = - new RegisterTableRequest() { - @Override - public String name() { - return TABLE_NS1_1.name(); - } - - @Override - public String metadataLocation() { - return metadataLocation; - } - }; - - // Use PRINCIPAL_ROLE1 for privilege-testing, PRINCIPAL_ROLE2 for cleanup. - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.TABLE_CREATE, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT), - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE1)).registerTable(NS1, registerRequest); - }, - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE2)).dropTableWithoutPurge(TABLE_NS1_1); - }); - } - - @Test - public void testRegisterTableInsufficientPermissions() { - Assertions.assertThat( - adminService.grantPrivilegeOnCatalogToRole( - CATALOG_NAME, CATALOG_ROLE2, PolarisPrivilege.TABLE_READ_PROPERTIES)) - .isTrue(); - - // To get a handy metadata file we can use one from another table. - final String metadataLocation = newWrapper().loadTable(TABLE_NS1_1, "all").metadataLocation(); - - final RegisterTableRequest registerRequest = - new RegisterTableRequest() { - @Override - public String name() { - return "newtable"; - } - - @Override - public String metadataLocation() { - return metadataLocation; - } - }; - - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.TABLE_DROP, - PolarisPrivilege.TABLE_READ_PROPERTIES, - PolarisPrivilege.TABLE_WRITE_PROPERTIES, - PolarisPrivilege.TABLE_READ_DATA, - PolarisPrivilege.TABLE_WRITE_DATA, - PolarisPrivilege.TABLE_LIST), - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE1)).registerTable(NS2, registerRequest); - }); - } - - @Test - public void testLoadTableSufficientPrivileges() { - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.TABLE_READ_PROPERTIES, - PolarisPrivilege.TABLE_WRITE_PROPERTIES, - PolarisPrivilege.TABLE_READ_DATA, - PolarisPrivilege.TABLE_WRITE_DATA, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT), - () -> newWrapper().loadTable(TABLE_NS1A_2, "all"), - null /* cleanupAction */); - } - - @Test - public void testLoadTableInsufficientPermissions() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.TABLE_CREATE, - PolarisPrivilege.TABLE_LIST, - PolarisPrivilege.TABLE_DROP), - () -> newWrapper().loadTable(TABLE_NS1A_2, "all")); - } - - @Test - public void testLoadTableWithReadAccessDelegationSufficientPrivileges() { - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.TABLE_READ_DATA, - PolarisPrivilege.TABLE_WRITE_DATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT), - () -> newWrapper().loadTableWithAccessDelegation(TABLE_NS1A_2, "all"), - null /* cleanupAction */); - } - - @Test - public void testLoadTableWithReadAccessDelegationInsufficientPermissions() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.TABLE_READ_PROPERTIES, - PolarisPrivilege.TABLE_WRITE_PROPERTIES, - PolarisPrivilege.TABLE_CREATE, - PolarisPrivilege.TABLE_LIST, - PolarisPrivilege.TABLE_DROP), - () -> newWrapper().loadTableWithAccessDelegation(TABLE_NS1A_2, "all")); - } - - @Test - public void testLoadTableWithWriteAccessDelegationSufficientPrivileges() { - doTestSufficientPrivileges( - List.of( - // TODO: Once we give different creds for read/write privilege, move this - // TABLE_READ_DATA into a special-case test; with only TABLE_READ_DATA we'd expet - // to receive a read-only credential. - PolarisPrivilege.TABLE_READ_DATA, - PolarisPrivilege.TABLE_WRITE_DATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT), - () -> newWrapper().loadTableWithAccessDelegation(TABLE_NS1A_2, "all"), - null /* cleanupAction */); - } - - @Test - public void testLoadTableWithWriteAccessDelegationInsufficientPermissions() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.TABLE_READ_PROPERTIES, - PolarisPrivilege.TABLE_WRITE_PROPERTIES, - PolarisPrivilege.TABLE_CREATE, - PolarisPrivilege.TABLE_LIST, - PolarisPrivilege.TABLE_DROP), - () -> newWrapper().loadTableWithAccessDelegation(TABLE_NS1A_2, "all")); - } - - @Test - public void testUpdateTableSufficientPrivileges() { - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.TABLE_WRITE_PROPERTIES, - PolarisPrivilege.TABLE_WRITE_DATA, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT), - () -> newWrapper().updateTable(TABLE_NS1A_2, new UpdateTableRequest()), - null /* cleanupAction */); - } - - @Test - public void testUpdateTableInsufficientPermissions() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.TABLE_READ_PROPERTIES, - PolarisPrivilege.TABLE_READ_DATA, - PolarisPrivilege.TABLE_CREATE, - PolarisPrivilege.TABLE_LIST, - PolarisPrivilege.TABLE_DROP), - () -> newWrapper().updateTable(TABLE_NS1A_2, new UpdateTableRequest())); - } - - @Test - public void testUpdateTableForStagedCreateSufficientPrivileges() { - // Note: This is kind of cheating by only leaning on the PolarisCatalogHandlerWrapper level - // of differentiation between updateForStageCreate vs regular update so that we don't need - // to actually set up the staged create but still test the privileges. If the underlying - // behavior diverges, we need to change this test to actually start with a stageCreate. - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.TABLE_CREATE, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT), - () -> newWrapper().updateTableForStagedCreate(TABLE_NS1A_2, new UpdateTableRequest()), - null /* cleanupAction */); - } - - @Test - public void testUpdateTableForStagedCreateInsufficientPermissions() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.TABLE_DROP, - PolarisPrivilege.TABLE_READ_PROPERTIES, - PolarisPrivilege.TABLE_WRITE_PROPERTIES, - PolarisPrivilege.TABLE_READ_DATA, - PolarisPrivilege.TABLE_WRITE_DATA, - PolarisPrivilege.TABLE_LIST), - () -> newWrapper().updateTableForStagedCreate(TABLE_NS1A_2, new UpdateTableRequest())); - } - - @Test - public void testDropTableWithoutPurgeAllSufficientPrivileges() { - Assertions.assertThat( - adminService.grantPrivilegeOnCatalogToRole( - CATALOG_NAME, CATALOG_ROLE2, PolarisPrivilege.TABLE_CREATE)) - .isTrue(); - - final CreateTableRequest createRequest = - CreateTableRequest.builder().withName(TABLE_NS1_1.name()).withSchema(SCHEMA).build(); - - // Use PRINCIPAL_ROLE1 for privilege-testing, PRINCIPAL_ROLE2 for cleanup. - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.TABLE_DROP, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT), - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE1)).dropTableWithoutPurge(TABLE_NS1_1); - }, - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE2)) - .createTableDirect(TABLE_NS1_1.namespace(), createRequest); - }); - } - - @Test - public void testDropTableWithoutPurgeInsufficientPermissions() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.TABLE_CREATE, - PolarisPrivilege.TABLE_READ_PROPERTIES, - PolarisPrivilege.TABLE_WRITE_PROPERTIES, - PolarisPrivilege.TABLE_READ_DATA, - PolarisPrivilege.TABLE_WRITE_DATA, - PolarisPrivilege.TABLE_LIST), - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE1)).dropTableWithoutPurge(TABLE_NS1_1); - }); - } - - @Test - public void testDropTableWithPurgeAllSufficientPrivileges() { - Assertions.assertThat( - adminService.grantPrivilegeOnCatalogToRole( - CATALOG_NAME, CATALOG_ROLE2, PolarisPrivilege.TABLE_CREATE)) - .isTrue(); - - final CreateTableRequest createRequest = - CreateTableRequest.builder().withName(TABLE_NS1_1.name()).withSchema(SCHEMA).build(); - - // Use PRINCIPAL_ROLE1 for privilege-testing, PRINCIPAL_ROLE2 for cleanup. - doTestSufficientPrivilegeSets( - List.of( - Set.of(PolarisPrivilege.TABLE_WRITE_DATA, PolarisPrivilege.TABLE_FULL_METADATA), - Set.of(PolarisPrivilege.TABLE_WRITE_DATA, PolarisPrivilege.TABLE_DROP), - Set.of(PolarisPrivilege.CATALOG_MANAGE_CONTENT)), - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE1)).dropTableWithPurge(TABLE_NS1_1); - }, - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE2)) - .createTableDirect(TABLE_NS1_1.namespace(), createRequest); - }, - PRINCIPAL_NAME); - } - - @Test - public void testDropTableWithPurgeInsufficientPermissions() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.TABLE_DROP, // TABLE_DROP itself is insufficient for purge - PolarisPrivilege.TABLE_CREATE, - PolarisPrivilege.TABLE_READ_PROPERTIES, - PolarisPrivilege.TABLE_WRITE_PROPERTIES, - PolarisPrivilege.TABLE_READ_DATA, - PolarisPrivilege.TABLE_WRITE_DATA, - PolarisPrivilege.TABLE_LIST), - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE1)).dropTableWithPurge(TABLE_NS1_1); - }); - } - - @Test - public void testTableExistsSufficientPrivileges() { - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.TABLE_LIST, - PolarisPrivilege.TABLE_READ_PROPERTIES, - PolarisPrivilege.TABLE_WRITE_PROPERTIES, - PolarisPrivilege.TABLE_READ_DATA, - PolarisPrivilege.TABLE_WRITE_DATA, - PolarisPrivilege.TABLE_CREATE, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT), - () -> newWrapper().tableExists(TABLE_NS1A_2), - null /* cleanupAction */); - } - - @Test - public void testTableExistsInsufficientPermissions() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.TABLE_DROP), - () -> newWrapper().tableExists(TABLE_NS1A_2)); - } - - @Test - public void testRenameTableAllSufficientPrivileges() { - final TableIdentifier srcTable = TABLE_NS1_1; - final TableIdentifier dstTable = TableIdentifier.of(NS1AA, "newtable"); - final RenameTableRequest rename1 = - RenameTableRequest.builder().withSource(srcTable).withDestination(dstTable).build(); - final RenameTableRequest rename2 = - RenameTableRequest.builder().withSource(dstTable).withDestination(srcTable).build(); - - doTestSufficientPrivilegeSets( - List.of( - Set.of(PolarisPrivilege.TABLE_FULL_METADATA), - Set.of(PolarisPrivilege.TABLE_CREATE, PolarisPrivilege.TABLE_DROP), - Set.of(PolarisPrivilege.CATALOG_MANAGE_CONTENT)), - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE1)).renameTable(rename1); - }, - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE1)).renameTable(rename2); - }, - PRINCIPAL_NAME); - } - - @Test - public void testRenameTableInsufficientPermissions() { - final TableIdentifier srcTable = TABLE_NS1_1; - final TableIdentifier dstTable = TableIdentifier.of(NS1AA, "newtable"); - final RenameTableRequest rename1 = - RenameTableRequest.builder().withSource(srcTable).withDestination(dstTable).build(); - - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.TABLE_DROP, - PolarisPrivilege.TABLE_CREATE, - PolarisPrivilege.TABLE_READ_PROPERTIES, - PolarisPrivilege.TABLE_WRITE_PROPERTIES, - PolarisPrivilege.TABLE_READ_DATA, - PolarisPrivilege.TABLE_WRITE_DATA, - PolarisPrivilege.TABLE_LIST), - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE1)).renameTable(rename1); - }); - } - - @Test - public void testRenameTablePrivilegesOnWrongSourceOrDestination() { - final TableIdentifier srcTable = TABLE_NS2_1; - final TableIdentifier dstTable = TableIdentifier.of(NS1AA, "newtable"); - final RenameTableRequest rename1 = - RenameTableRequest.builder().withSource(srcTable).withDestination(dstTable).build(); - final RenameTableRequest rename2 = - RenameTableRequest.builder().withSource(dstTable).withDestination(srcTable).build(); - - // Minimum privileges should succeed -- drop on src, create on dst parent. - Assertions.assertThat( - adminService.grantPrivilegeOnTableToRole( - CATALOG_NAME, CATALOG_ROLE1, srcTable, PolarisPrivilege.TABLE_DROP)) - .isTrue(); - Assertions.assertThat( - adminService.grantPrivilegeOnNamespaceToRole( - CATALOG_NAME, CATALOG_ROLE1, dstTable.namespace(), PolarisPrivilege.TABLE_CREATE)) - .isTrue(); - - // Initial rename should succeed - newWrapper().renameTable(rename1); - - // Inverse operation should fail - Assertions.assertThatThrownBy(() -> newWrapper().renameTable(rename2)) - .isInstanceOf(ForbiddenException.class); - - // Now grant TABLE_DROP on dst - Assertions.assertThat( - adminService.grantPrivilegeOnTableToRole( - CATALOG_NAME, CATALOG_ROLE1, dstTable, PolarisPrivilege.TABLE_DROP)) - .isTrue(); - - // Still not enough without TABLE_CREATE at source - Assertions.assertThatThrownBy(() -> newWrapper().renameTable(rename2)) - .isInstanceOf(ForbiddenException.class); - - // Even grant CATALOG_MANAGE_CONTENT under all of NS1 - Assertions.assertThat( - adminService.grantPrivilegeOnNamespaceToRole( - CATALOG_NAME, CATALOG_ROLE1, NS1, PolarisPrivilege.CATALOG_MANAGE_CONTENT)) - .isTrue(); - - // Still not enough to rename back to src since src was NS2. - Assertions.assertThatThrownBy(() -> newWrapper().renameTable(rename2)) - .isInstanceOf(ForbiddenException.class); - - // Finally, grant TABLE_CREATE on NS2 and it should succeed to rename back to src. - Assertions.assertThat( - adminService.grantPrivilegeOnNamespaceToRole( - CATALOG_NAME, CATALOG_ROLE1, NS2, PolarisPrivilege.TABLE_CREATE)) - .isTrue(); - newWrapper().renameTable(rename2); - } - - @Test - public void testCommitTransactionSufficientPrivileges() { - CommitTransactionRequest req = - new CommitTransactionRequest( - List.of( - UpdateTableRequest.create(TABLE_NS1_1, List.of(), List.of()), - UpdateTableRequest.create(TABLE_NS1A_1, List.of(), List.of()), - UpdateTableRequest.create(TABLE_NS1B_1, List.of(), List.of()), - UpdateTableRequest.create(TABLE_NS2_1, List.of(), List.of()))); - - doTestSufficientPrivilegeSets( - List.of( - Set.of(PolarisPrivilege.CATALOG_MANAGE_CONTENT), - Set.of(PolarisPrivilege.TABLE_FULL_METADATA), - Set.of(PolarisPrivilege.TABLE_CREATE, PolarisPrivilege.TABLE_WRITE_DATA), - Set.of(PolarisPrivilege.TABLE_CREATE, PolarisPrivilege.TABLE_WRITE_PROPERTIES)), - () -> newWrapper().commitTransaction(req), - null, - PRINCIPAL_NAME /* cleanupAction */); - } - - @Test - public void testCommitTransactionInsufficientPermissions() { - CommitTransactionRequest req = - new CommitTransactionRequest( - List.of( - UpdateTableRequest.create(TABLE_NS1_1, List.of(), List.of()), - UpdateTableRequest.create(TABLE_NS1A_1, List.of(), List.of()), - UpdateTableRequest.create(TABLE_NS1B_1, List.of(), List.of()), - UpdateTableRequest.create(TABLE_NS2_1, List.of(), List.of()))); - - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.TABLE_READ_PROPERTIES, - PolarisPrivilege.TABLE_WRITE_PROPERTIES, - PolarisPrivilege.TABLE_READ_DATA, - PolarisPrivilege.TABLE_WRITE_DATA, - PolarisPrivilege.TABLE_CREATE, - PolarisPrivilege.TABLE_LIST, - PolarisPrivilege.TABLE_DROP), - () -> newWrapper().commitTransaction(req)); - } - - @Test - public void testCommitTransactionMixedPermissions() { - CommitTransactionRequest req = - new CommitTransactionRequest( - List.of( - UpdateTableRequest.create(TABLE_NS1_1, List.of(), List.of()), - UpdateTableRequest.create(TABLE_NS1A_1, List.of(), List.of()), - UpdateTableRequest.create(TABLE_NS1B_1, List.of(), List.of()), - UpdateTableRequest.create(TABLE_NS2_1, List.of(), List.of()))); - - // Grant TABLE_CREATE for all of NS1 - Assertions.assertThat( - adminService.grantPrivilegeOnNamespaceToRole( - CATALOG_NAME, CATALOG_ROLE1, NS1, PolarisPrivilege.TABLE_CREATE)) - .isTrue(); - Assertions.assertThatThrownBy(() -> newWrapper().commitTransaction(req)) - .isInstanceOf(ForbiddenException.class); - - // Grant TABLE_FULL_METADATA directly on TABLE_NS1_1 - Assertions.assertThat( - adminService.grantPrivilegeOnTableToRole( - CATALOG_NAME, CATALOG_ROLE1, TABLE_NS1_1, PolarisPrivilege.TABLE_FULL_METADATA)) - .isTrue(); - Assertions.assertThatThrownBy(() -> newWrapper().commitTransaction(req)) - .isInstanceOf(ForbiddenException.class); - - // Grant TABLE_WRITE_PROPERTIES on NS1A namespace - Assertions.assertThat( - adminService.grantPrivilegeOnNamespaceToRole( - CATALOG_NAME, CATALOG_ROLE1, NS1A, PolarisPrivilege.TABLE_WRITE_PROPERTIES)) - .isTrue(); - Assertions.assertThatThrownBy(() -> newWrapper().commitTransaction(req)) - .isInstanceOf(ForbiddenException.class); - - // Grant TABLE_WRITE_DATA directly on TABLE_NS1B_1 - Assertions.assertThat( - adminService.grantPrivilegeOnTableToRole( - CATALOG_NAME, CATALOG_ROLE1, TABLE_NS1B_1, PolarisPrivilege.TABLE_WRITE_DATA)) - .isTrue(); - Assertions.assertThatThrownBy(() -> newWrapper().commitTransaction(req)) - .isInstanceOf(ForbiddenException.class); - - // Grant TABLE_WRITE_PROPERTIES directly on TABLE_NS2_1 - Assertions.assertThat( - adminService.grantPrivilegeOnTableToRole( - CATALOG_NAME, CATALOG_ROLE1, TABLE_NS2_1, PolarisPrivilege.TABLE_WRITE_PROPERTIES)) - .isTrue(); - Assertions.assertThatThrownBy(() -> newWrapper().commitTransaction(req)) - .isInstanceOf(ForbiddenException.class); - - // Also grant TABLE_CREATE directly on TABLE_NS2_1 - // TODO: If we end up having fine-grained differentiation between updateForStagedCreate - // and update, then this one should only be TABLE_CREATE on the *parent* of this last table - // and the table shouldn't exist. - Assertions.assertThat( - adminService.grantPrivilegeOnTableToRole( - CATALOG_NAME, CATALOG_ROLE1, TABLE_NS2_1, PolarisPrivilege.TABLE_CREATE)) - .isTrue(); - newWrapper().commitTransaction(req); - } - - @Test - public void testListViewsAllSufficientPrivileges() { - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.VIEW_LIST, - PolarisPrivilege.VIEW_READ_PROPERTIES, - PolarisPrivilege.VIEW_WRITE_PROPERTIES, - PolarisPrivilege.VIEW_CREATE, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT), - () -> newWrapper().listViews(NS1A), - null /* cleanupAction */); - } - - @Test - public void testListViewsInsufficientPermissions() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.VIEW_DROP), - () -> newWrapper().listViews(NS1A)); - } - - @Test - public void testCreateViewAllSufficientPrivileges() { - Assertions.assertThat( - adminService.grantPrivilegeOnCatalogToRole( - CATALOG_NAME, CATALOG_ROLE2, PolarisPrivilege.VIEW_DROP)) - .isTrue(); - - final TableIdentifier newview = TableIdentifier.of(NS2, "newview"); - final CreateViewRequest createRequest = - ImmutableCreateViewRequest.builder() - .name("newview") - .schema(SCHEMA) - .viewVersion( - ImmutableViewVersion.builder() - .versionId(1) - .timestampMillis(System.currentTimeMillis()) - .schemaId(1) - .defaultNamespace(NS1) - .addRepresentations( - ImmutableSQLViewRepresentation.builder() - .sql(VIEW_QUERY) - .dialect("spark") - .build()) - .build()) - .build(); - - // Use PRINCIPAL_ROLE1 for privilege-testing, PRINCIPAL_ROLE2 for cleanup. - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.VIEW_CREATE, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT), - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE1)).createView(NS2, createRequest); - }, - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE2)).dropView(newview); - }); - } - - @Test - public void testCreateViewInsufficientPermissions() { - final CreateViewRequest createRequest = - ImmutableCreateViewRequest.builder() - .name("newview") - .schema(SCHEMA) - .viewVersion( - ImmutableViewVersion.builder() - .versionId(1) - .timestampMillis(System.currentTimeMillis()) - .schemaId(1) - .defaultNamespace(NS1) - .addRepresentations( - ImmutableSQLViewRepresentation.builder() - .sql(VIEW_QUERY) - .dialect("spark") - .build()) - .build()) - .build(); - - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.VIEW_DROP, - PolarisPrivilege.VIEW_READ_PROPERTIES, - PolarisPrivilege.VIEW_WRITE_PROPERTIES, - PolarisPrivilege.VIEW_LIST), - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE1)).createView(NS2, createRequest); - }); - } - - @Test - public void testLoadViewSufficientPrivileges() { - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.VIEW_READ_PROPERTIES, - PolarisPrivilege.VIEW_WRITE_PROPERTIES, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT), - () -> newWrapper().loadView(VIEW_NS1A_2), - null /* cleanupAction */); - } - - @Test - public void testLoadViewInsufficientPermissions() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.VIEW_CREATE, - PolarisPrivilege.VIEW_LIST, - PolarisPrivilege.VIEW_DROP), - () -> newWrapper().loadView(VIEW_NS1A_2)); - } - - @Test - public void testUpdateViewSufficientPrivileges() { - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.VIEW_WRITE_PROPERTIES, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT), - () -> newWrapper().replaceView(VIEW_NS1A_2, new UpdateTableRequest()), - null /* cleanupAction */); - } - - @Test - public void testUpdateViewInsufficientPermissions() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.VIEW_READ_PROPERTIES, - PolarisPrivilege.VIEW_CREATE, - PolarisPrivilege.VIEW_LIST, - PolarisPrivilege.VIEW_DROP), - () -> newWrapper().replaceView(VIEW_NS1A_2, new UpdateTableRequest())); - } - - @Test - public void testDropViewAllSufficientPrivileges() { - Assertions.assertThat( - adminService.grantPrivilegeOnCatalogToRole( - CATALOG_NAME, CATALOG_ROLE2, PolarisPrivilege.VIEW_CREATE)) - .isTrue(); - - final CreateViewRequest createRequest = - ImmutableCreateViewRequest.builder() - .name(VIEW_NS1_1.name()) - .schema(SCHEMA) - .viewVersion( - ImmutableViewVersion.builder() - .versionId(1) - .timestampMillis(System.currentTimeMillis()) - .schemaId(1) - .defaultNamespace(NS1) - .addRepresentations( - ImmutableSQLViewRepresentation.builder() - .sql(VIEW_QUERY) - .dialect("spark") - .build()) - .build()) - .build(); - - // Use PRINCIPAL_ROLE1 for privilege-testing, PRINCIPAL_ROLE2 for cleanup. - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.VIEW_DROP, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT), - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE1)).dropView(VIEW_NS1_1); - }, - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE2)).createView(VIEW_NS1_1.namespace(), createRequest); - }); - } - - @Test - public void testDropViewInsufficientPermissions() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.VIEW_CREATE, - PolarisPrivilege.VIEW_READ_PROPERTIES, - PolarisPrivilege.VIEW_WRITE_PROPERTIES, - PolarisPrivilege.VIEW_LIST), - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE1)).dropView(VIEW_NS1_1); - }); - } - - @Test - public void testViewExistsSufficientPrivileges() { - doTestSufficientPrivileges( - List.of( - PolarisPrivilege.VIEW_LIST, - PolarisPrivilege.VIEW_READ_PROPERTIES, - PolarisPrivilege.VIEW_WRITE_PROPERTIES, - PolarisPrivilege.VIEW_CREATE, - PolarisPrivilege.VIEW_FULL_METADATA, - PolarisPrivilege.CATALOG_MANAGE_CONTENT), - () -> newWrapper().viewExists(VIEW_NS1A_2), - null /* cleanupAction */); - } - - @Test - public void testViewExistsInsufficientPermissions() { - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.VIEW_DROP), - () -> newWrapper().viewExists(VIEW_NS1A_2)); - } - - @Test - public void testRenameViewAllSufficientPrivileges() { - final TableIdentifier srcView = VIEW_NS1_1; - final TableIdentifier dstView = TableIdentifier.of(NS1AA, "newview"); - final RenameTableRequest rename1 = - RenameTableRequest.builder().withSource(srcView).withDestination(dstView).build(); - final RenameTableRequest rename2 = - RenameTableRequest.builder().withSource(dstView).withDestination(srcView).build(); - - doTestSufficientPrivilegeSets( - List.of( - Set.of(PolarisPrivilege.VIEW_FULL_METADATA), - Set.of(PolarisPrivilege.CATALOG_MANAGE_CONTENT), - Set.of(PolarisPrivilege.VIEW_DROP, PolarisPrivilege.VIEW_CREATE)), - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE1)).renameView(rename1); - }, - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE1)).renameView(rename2); - }, - PRINCIPAL_NAME); - } - - @Test - public void testRenameViewInsufficientPermissions() { - final TableIdentifier srcView = VIEW_NS1_1; - final TableIdentifier dstView = TableIdentifier.of(NS1AA, "newview"); - final RenameTableRequest rename1 = - RenameTableRequest.builder().withSource(srcView).withDestination(dstView).build(); - - doTestInsufficientPrivileges( - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.VIEW_DROP, - PolarisPrivilege.VIEW_CREATE, - PolarisPrivilege.VIEW_READ_PROPERTIES, - PolarisPrivilege.VIEW_WRITE_PROPERTIES, - PolarisPrivilege.VIEW_LIST), - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE1)).renameView(rename1); - }); - } - - @Test - public void testRenameViewPrivilegesOnWrongSourceOrDestination() { - final TableIdentifier srcView = VIEW_NS2_1; - final TableIdentifier dstView = TableIdentifier.of(NS1AA, "newview"); - final RenameTableRequest rename1 = - RenameTableRequest.builder().withSource(srcView).withDestination(dstView).build(); - final RenameTableRequest rename2 = - RenameTableRequest.builder().withSource(dstView).withDestination(srcView).build(); - - // Minimum privileges should succeed -- drop on src, create on dst parent. - Assertions.assertThat( - adminService.grantPrivilegeOnViewToRole( - CATALOG_NAME, CATALOG_ROLE1, srcView, PolarisPrivilege.VIEW_DROP)) - .isTrue(); - Assertions.assertThat( - adminService.grantPrivilegeOnNamespaceToRole( - CATALOG_NAME, CATALOG_ROLE1, dstView.namespace(), PolarisPrivilege.VIEW_CREATE)) - .isTrue(); - - // Initial rename should succeed - newWrapper().renameView(rename1); - - // Inverse operation should fail - Assertions.assertThatThrownBy(() -> newWrapper().renameView(rename2)) - .isInstanceOf(ForbiddenException.class); - - // Now grant VIEW_DROP on dst - Assertions.assertThat( - adminService.grantPrivilegeOnViewToRole( - CATALOG_NAME, CATALOG_ROLE1, dstView, PolarisPrivilege.VIEW_DROP)) - .isTrue(); - - // Still not enough without VIEW_CREATE at source - Assertions.assertThatThrownBy(() -> newWrapper().renameView(rename2)) - .isInstanceOf(ForbiddenException.class); - - // Even grant CATALOG_MANAGE_CONTENT under all of NS1 - Assertions.assertThat( - adminService.grantPrivilegeOnNamespaceToRole( - CATALOG_NAME, CATALOG_ROLE1, NS1, PolarisPrivilege.CATALOG_MANAGE_CONTENT)) - .isTrue(); - - // Still not enough to rename back to src since src was NS2. - Assertions.assertThatThrownBy(() -> newWrapper().renameView(rename2)) - .isInstanceOf(ForbiddenException.class); - - // Finally, grant VIEW_CREATE on NS2 and it should succeed to rename back to src. - Assertions.assertThat( - adminService.grantPrivilegeOnNamespaceToRole( - CATALOG_NAME, CATALOG_ROLE1, NS2, PolarisPrivilege.VIEW_CREATE)) - .isTrue(); - newWrapper().renameView(rename2); - } - - @Test - public void testSendNotificationSufficientPrivileges() { - String externalCatalog = "externalCatalog"; - String storageLocation = - "file:///tmp/send_notification_sufficient_privileges_" + System.currentTimeMillis(); - - FileStorageConfigInfo storageConfigModel = - FileStorageConfigInfo.builder() - .setStorageType(StorageConfigInfo.StorageTypeEnum.FILE) - .build(); - adminService.createCatalog( - new CatalogEntity.Builder() - .setName(externalCatalog) - .setDefaultBaseLocation(storageLocation) - .setStorageConfigurationInfo(storageConfigModel, storageLocation) - .setCatalogType("EXTERNAL") - .build()); - adminService.createCatalogRole( - externalCatalog, new CatalogRoleEntity.Builder().setName(CATALOG_ROLE1).build()); - adminService.createCatalogRole( - externalCatalog, new CatalogRoleEntity.Builder().setName(CATALOG_ROLE2).build()); - - adminService.assignPrincipalRole(PRINCIPAL_NAME, PRINCIPAL_ROLE1); - adminService.assignCatalogRoleToPrincipalRole(PRINCIPAL_ROLE1, externalCatalog, CATALOG_ROLE1); - adminService.assignCatalogRoleToPrincipalRole(PRINCIPAL_ROLE2, externalCatalog, CATALOG_ROLE2); - Assertions.assertThat( - adminService.grantPrivilegeOnCatalogToRole( - externalCatalog, CATALOG_ROLE2, PolarisPrivilege.TABLE_DROP)) - .isTrue(); - Assertions.assertThat( - adminService.grantPrivilegeOnCatalogToRole( - externalCatalog, CATALOG_ROLE2, PolarisPrivilege.NAMESPACE_DROP)) - .isTrue(); - - Namespace namespace = Namespace.of("extns1", "extns2"); - TableIdentifier table = TableIdentifier.of(namespace, "tbl1"); - - String tableUuid = UUID.randomUUID().toString(); - - NotificationRequest createRequest = new NotificationRequest(); - createRequest.setNotificationType(NotificationType.CREATE); - TableUpdateNotification createPayload = new TableUpdateNotification(); - createPayload.setMetadataLocation( - String.format("%s/bucket/table/metadata/v1.metadata.json", storageLocation)); - createPayload.setTableName(table.name()); - createPayload.setTableUuid(tableUuid); - createPayload.setTimestamp(230950845L); - createRequest.setPayload(createPayload); - - NotificationRequest updateRequest = new NotificationRequest(); - updateRequest.setNotificationType(NotificationType.UPDATE); - TableUpdateNotification updatePayload = new TableUpdateNotification(); - updatePayload.setMetadataLocation( - String.format("%s/bucket/table/metadata/v2.metadata.json", storageLocation)); - updatePayload.setTableName(table.name()); - updatePayload.setTableUuid(tableUuid); - updatePayload.setTimestamp(330950845L); - updateRequest.setPayload(updatePayload); - - NotificationRequest dropRequest = new NotificationRequest(); - dropRequest.setNotificationType(NotificationType.DROP); - TableUpdateNotification dropPayload = new TableUpdateNotification(); - dropPayload.setTableName(table.name()); - dropPayload.setTableUuid(tableUuid); - dropPayload.setTimestamp(430950845L); - dropRequest.setPayload(dropPayload); - - NotificationRequest validateRequest = new NotificationRequest(); - validateRequest.setNotificationType(NotificationType.VALIDATE); - TableUpdateNotification validatePayload = new TableUpdateNotification(); - validatePayload.setMetadataLocation( - String.format("%s/bucket/table/metadata/v1.metadata.json", storageLocation)); - validatePayload.setTableName(table.name()); - validatePayload.setTableUuid(tableUuid); - validatePayload.setTimestamp(530950845L); - validateRequest.setPayload(validatePayload); - - PolarisCallContextCatalogFactory factory = - new PolarisCallContextCatalogFactory( - new RealmEntityManagerFactory(managerFactory) { - @Override - public PolarisEntityManager getOrCreateEntityManager(RealmContext realmContext) { - return entityManager; - } - }, - managerFactory, - Mockito.mock(), - new DefaultFileIOFactory()) { - @Override - public Catalog createCallContextCatalog( - CallContext context, - AuthenticatedPolarisPrincipal authenticatedPolarisPrincipal, - PolarisResolutionManifest resolvedManifest) { - Catalog catalog = - super.createCallContextCatalog( - context, authenticatedPolarisPrincipal, resolvedManifest); - String fileIoImpl = "org.apache.iceberg.inmemory.InMemoryFileIO"; - catalog.initialize( - externalCatalog, ImmutableMap.of(CatalogProperties.FILE_IO_IMPL, fileIoImpl)); - - FileIO fileIO = CatalogUtil.loadFileIO(fileIoImpl, Map.of(), new Configuration()); - TableMetadata tableMetadata = - TableMetadata.buildFromEmpty() - .addSchema(SCHEMA, SCHEMA.highestFieldId()) - .setLocation( - String.format("%s/bucket/table/metadata/v1.metadata.json", storageLocation)) - .addPartitionSpec(PartitionSpec.unpartitioned()) - .addSortOrder(SortOrder.unsorted()) - .assignUUID() - .build(); - TableMetadataParser.overwrite( - tableMetadata, fileIO.newOutputFile(createPayload.getMetadataLocation())); - TableMetadataParser.overwrite( - tableMetadata, fileIO.newOutputFile(updatePayload.getMetadataLocation())); - return catalog; - } - }; - - List> sufficientPrivilegeSets = - List.of( - Set.of(PolarisPrivilege.CATALOG_MANAGE_CONTENT), - Set.of(PolarisPrivilege.TABLE_FULL_METADATA, PolarisPrivilege.NAMESPACE_FULL_METADATA), - Set.of( - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.NAMESPACE_CREATE, - PolarisPrivilege.NAMESPACE_DROP), - Set.of( - PolarisPrivilege.TABLE_CREATE, - PolarisPrivilege.TABLE_DROP, - PolarisPrivilege.TABLE_WRITE_PROPERTIES, - PolarisPrivilege.NAMESPACE_FULL_METADATA), - Set.of( - PolarisPrivilege.TABLE_CREATE, - PolarisPrivilege.TABLE_DROP, - PolarisPrivilege.TABLE_WRITE_PROPERTIES, - PolarisPrivilege.NAMESPACE_CREATE, - PolarisPrivilege.NAMESPACE_DROP)); - doTestSufficientPrivilegeSets( - sufficientPrivilegeSets, - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE1), externalCatalog, factory) - .sendNotification(table, createRequest); - newWrapper(Set.of(PRINCIPAL_ROLE1), externalCatalog, factory) - .sendNotification(table, updateRequest); - newWrapper(Set.of(PRINCIPAL_ROLE1), externalCatalog, factory) - .sendNotification(table, dropRequest); - newWrapper(Set.of(PRINCIPAL_ROLE1), externalCatalog, factory) - .sendNotification(table, validateRequest); - }, - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE2), externalCatalog, factory) - .dropNamespace(Namespace.of("extns1", "extns2")); - newWrapper(Set.of(PRINCIPAL_ROLE2), externalCatalog, factory) - .dropNamespace(Namespace.of("extns1")); - }, - PRINCIPAL_NAME, - externalCatalog); - - // Also test VALIDATE in isolation - doTestSufficientPrivilegeSets( - sufficientPrivilegeSets, - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE1), externalCatalog, factory) - .sendNotification(table, validateRequest); - }, - null /* cleanupAction */, - PRINCIPAL_NAME, - externalCatalog); - } - - @Test - public void testSendNotificationInsufficientPermissions() { - Namespace namespace = Namespace.of("ns1", "ns2"); - TableIdentifier table = TableIdentifier.of(namespace, "tbl1"); - - NotificationRequest request = new NotificationRequest(); - TableUpdateNotification update = new TableUpdateNotification(); - update.setMetadataLocation("file:///tmp/bucket/table/metadata/v1.metadata.json"); - update.setTableName(table.name()); - update.setTableUuid(UUID.randomUUID().toString()); - update.setTimestamp(230950845L); - request.setPayload(update); - - List insufficientPrivileges = - List.of( - PolarisPrivilege.NAMESPACE_FULL_METADATA, - PolarisPrivilege.TABLE_FULL_METADATA, - PolarisPrivilege.VIEW_FULL_METADATA); - - // Independently test insufficient privileges in isolation. - request.setNotificationType(NotificationType.CREATE); - doTestInsufficientPrivileges( - insufficientPrivileges, - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE1)).sendNotification(table, request); - }); - - request.setNotificationType(NotificationType.UPDATE); - doTestInsufficientPrivileges( - insufficientPrivileges, - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE1)).sendNotification(table, request); - }); - - request.setNotificationType(NotificationType.DROP); - doTestInsufficientPrivileges( - insufficientPrivileges, - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE1)).sendNotification(table, request); - }); - - request.setNotificationType(NotificationType.VALIDATE); - doTestInsufficientPrivileges( - insufficientPrivileges, - () -> { - newWrapper(Set.of(PRINCIPAL_ROLE1)).sendNotification(table, request); - }); - } - - public static class Profile implements QuarkusTestProfile { - - @Override - public Map getConfigOverrides() { - return Map.of( - "polaris.config.feature-configurations.ALLOW_EXTERNAL_METADATA_FILE_LOCATION", "true"); - } - } -} diff --git a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/PolarisPassthroughResolutionView.java b/polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/PolarisPassthroughResolutionView.java deleted file mode 100644 index 313e0265f..000000000 --- a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/PolarisPassthroughResolutionView.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.catalog; - -import java.util.Arrays; -import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; -import org.apache.polaris.core.catalog.PolarisCatalogHelpers; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.entity.PolarisEntitySubType; -import org.apache.polaris.core.entity.PolarisEntityType; -import org.apache.polaris.core.persistence.PolarisEntityManager; -import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; -import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest; -import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifestCatalogView; -import org.apache.polaris.core.persistence.resolver.ResolverPath; - -/** - * For test purposes or for elevated-privilege scenarios where entity resolution is allowed to - * directly access a PolarisEntityManager/PolarisMetaStoreManager without being part of an - * authorization-gated PolarisResolutionManifest, this class delegates entity resolution directly to - * new single-use PolarisResolutionManifests for each desired resolved path without defining a fixed - * set of resolved entities that need to be checked against authorizable operations. - */ -public class PolarisPassthroughResolutionView implements PolarisResolutionManifestCatalogView { - private final PolarisEntityManager entityManager; - private final CallContext callContext; - private final AuthenticatedPolarisPrincipal authenticatedPrincipal; - private final String catalogName; - - public PolarisPassthroughResolutionView( - CallContext callContext, - PolarisEntityManager entityManager, - AuthenticatedPolarisPrincipal authenticatedPrincipal, - String catalogName) { - this.entityManager = entityManager; - this.callContext = callContext; - this.authenticatedPrincipal = authenticatedPrincipal; - this.catalogName = catalogName; - } - - @Override - public PolarisResolvedPathWrapper getResolvedReferenceCatalogEntity() { - PolarisResolutionManifest manifest = - entityManager.prepareResolutionManifest(callContext, authenticatedPrincipal, catalogName); - manifest.resolveAll(); - return manifest.getResolvedReferenceCatalogEntity(); - } - - @Override - public PolarisResolvedPathWrapper getResolvedPath(Object key) { - PolarisResolutionManifest manifest = - entityManager.prepareResolutionManifest(callContext, authenticatedPrincipal, catalogName); - - if (key instanceof Namespace namespace) { - manifest.addPath( - new ResolverPath(Arrays.asList(namespace.levels()), PolarisEntityType.NAMESPACE), - namespace); - manifest.resolveAll(); - return manifest.getResolvedPath(namespace); - } else { - throw new IllegalStateException( - String.format( - "Trying to getResolvedPath(key) for %s with class %s", key, key.getClass())); - } - } - - @Override - public PolarisResolvedPathWrapper getResolvedPath(Object key, PolarisEntitySubType subType) { - PolarisResolutionManifest manifest = - entityManager.prepareResolutionManifest(callContext, authenticatedPrincipal, catalogName); - - if (key instanceof TableIdentifier identifier) { - manifest.addPath( - new ResolverPath( - PolarisCatalogHelpers.tableIdentifierToList(identifier), - PolarisEntityType.TABLE_LIKE), - identifier); - manifest.resolveAll(); - return manifest.getResolvedPath(identifier, subType); - } else { - throw new IllegalStateException( - String.format( - "Trying to getResolvedPath(key, subType) for %s with class %s and subType %s", - key, key.getClass(), subType)); - } - } - - @Override - public PolarisResolvedPathWrapper getPassthroughResolvedPath(Object key) { - PolarisResolutionManifest manifest = - entityManager.prepareResolutionManifest(callContext, authenticatedPrincipal, catalogName); - - if (key instanceof Namespace namespace) { - manifest.addPassthroughPath( - new ResolverPath(Arrays.asList(namespace.levels()), PolarisEntityType.NAMESPACE), - namespace); - return manifest.getPassthroughResolvedPath(namespace); - } else { - throw new IllegalStateException( - String.format( - "Trying to getResolvedPath(key) for %s with class %s", key, key.getClass())); - } - } - - @Override - public PolarisResolvedPathWrapper getPassthroughResolvedPath( - Object key, PolarisEntitySubType subType) { - PolarisResolutionManifest manifest = - entityManager.prepareResolutionManifest(callContext, authenticatedPrincipal, catalogName); - - if (key instanceof TableIdentifier identifier) { - manifest.addPassthroughPath( - new ResolverPath( - PolarisCatalogHelpers.tableIdentifierToList(identifier), - PolarisEntityType.TABLE_LIKE), - identifier); - return manifest.getPassthroughResolvedPath(identifier, subType); - } else { - throw new IllegalStateException( - String.format( - "Trying to getResolvedPath(key, subType) for %s with class %s and subType %s", - key, key.getClass(), subType)); - } - } -} diff --git a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogIntegrationTest.java b/polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogIntegrationTest.java deleted file mode 100644 index aedebc25e..000000000 --- a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogIntegrationTest.java +++ /dev/null @@ -1,973 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.catalog; - -import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -import com.google.common.collect.ImmutableMap; -import io.quarkus.test.junit.QuarkusTest; -import jakarta.inject.Inject; -import jakarta.ws.rs.client.Entity; -import jakarta.ws.rs.core.Response; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.UUID; -import org.apache.iceberg.BaseTable; -import org.apache.iceberg.BaseTransaction; -import org.apache.iceberg.CatalogProperties; -import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.Schema; -import org.apache.iceberg.Table; -import org.apache.iceberg.Transaction; -import org.apache.iceberg.UpdatePartitionSpec; -import org.apache.iceberg.UpdateSchema; -import org.apache.iceberg.catalog.CatalogTests; -import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.catalog.SessionCatalog; -import org.apache.iceberg.catalog.TableCommit; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.exceptions.CommitFailedException; -import org.apache.iceberg.exceptions.ForbiddenException; -import org.apache.iceberg.expressions.Expressions; -import org.apache.iceberg.rest.HTTPClient; -import org.apache.iceberg.rest.RESTCatalog; -import org.apache.iceberg.rest.auth.OAuth2Properties; -import org.apache.iceberg.rest.responses.ErrorResponse; -import org.apache.iceberg.types.Types; -import org.apache.polaris.core.PolarisConfiguration; -import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; -import org.apache.polaris.core.admin.model.Catalog; -import org.apache.polaris.core.admin.model.CatalogGrant; -import org.apache.polaris.core.admin.model.CatalogPrivilege; -import org.apache.polaris.core.admin.model.CatalogRole; -import org.apache.polaris.core.admin.model.FileStorageConfigInfo; -import org.apache.polaris.core.admin.model.GrantResource; -import org.apache.polaris.core.admin.model.GrantResources; -import org.apache.polaris.core.admin.model.NamespaceGrant; -import org.apache.polaris.core.admin.model.NamespacePrivilege; -import org.apache.polaris.core.admin.model.PolarisCatalog; -import org.apache.polaris.core.admin.model.StorageConfigInfo; -import org.apache.polaris.core.admin.model.TableGrant; -import org.apache.polaris.core.admin.model.TablePrivilege; -import org.apache.polaris.core.admin.model.UpdateCatalogRequest; -import org.apache.polaris.core.admin.model.ViewGrant; -import org.apache.polaris.core.admin.model.ViewPrivilege; -import org.apache.polaris.core.entity.CatalogEntity; -import org.apache.polaris.core.entity.PolarisEntityConstants; -import org.apache.polaris.service.auth.BasePolarisAuthenticator; -import org.apache.polaris.service.test.PolarisIntegrationTestHelper; -import org.apache.polaris.service.types.NotificationRequest; -import org.apache.polaris.service.types.NotificationType; -import org.apache.polaris.service.types.TableUpdateNotification; -import org.assertj.core.api.Assertions; -import org.assertj.core.api.InstanceOfAssertFactories; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInfo; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.TestInstance.Lifecycle; - -/** - * Import the full core Iceberg catalog tests by hitting the REST service via the RESTCatalog - * client. - */ -@QuarkusTest -@TestInstance(Lifecycle.PER_CLASS) -public class PolarisRestCatalogIntegrationTest extends CatalogTests { - private static final String TEST_ROLE_ARN = - Optional.ofNullable(System.getenv("INTEGRATION_TEST_ROLE_ARN")) - .orElse("arn:aws:iam::123456789012:role/my-role"); - private static final String S3_BUCKET_BASE = - Optional.ofNullable(System.getenv("INTEGRATION_TEST_S3_PATH")) - .orElse("file:///tmp/buckets/my-bucket"); - - protected static final String VIEW_QUERY = "select * from ns1.layer1_table"; - - private RESTCatalog restCatalog; - private String currentCatalogName; - - private final String catalogBaseLocation = - S3_BUCKET_BASE + "/" + System.getenv("USER") + "/path/to/data"; - - @Inject PolarisIntegrationTestHelper testHelper; - - @BeforeAll - public void setUp(TestInfo testInfo) { - testHelper.setUp(testInfo); - } - - @AfterAll - public void tearDown() { - testHelper.tearDown(); - } - - @BeforeEach - void before(TestInfo testInfo) { - testInfo - .getTestMethod() - .ifPresent( - method -> { - currentCatalogName = method.getName() + UUID.randomUUID(); - AwsStorageConfigInfo awsConfigModel = - AwsStorageConfigInfo.builder() - .setRoleArn(TEST_ROLE_ARN) - .setExternalId("externalId") - .setUserArn("a:user:arn") - .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) - .setAllowedLocations(List.of("s3://my-old-bucket/path/to/data")) - .build(); - org.apache.polaris.core.admin.model.CatalogProperties.Builder catalogPropsBuilder = - org.apache.polaris.core.admin.model.CatalogProperties.builder(catalogBaseLocation) - .addProperty( - PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), - "true") - .addProperty( - PolarisConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), - "true"); - if (!S3_BUCKET_BASE.startsWith("file:/")) { - catalogPropsBuilder.addProperty( - CatalogEntity.REPLACE_NEW_LOCATION_PREFIX_WITH_CATALOG_DEFAULT_KEY, "file:"); - } - Catalog catalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName(currentCatalogName) - .setProperties(catalogPropsBuilder.build()) - .setStorageConfigInfo( - S3_BUCKET_BASE.startsWith("file:/") - ? new FileStorageConfigInfo( - StorageConfigInfo.StorageTypeEnum.FILE, List.of("file://")) - : awsConfigModel) - .build(); - try (Response response = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs", - testHelper.localPort)) - .request("application/json") - .header("Authorization", "Bearer " + testHelper.adminToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .post(Entity.json(catalog))) { - assertThat(response) - .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - - // Create a new CatalogRole that has CATALOG_MANAGE_CONTENT and CATALOG_MANAGE_ACCESS - CatalogRole newRole = new CatalogRole("custom-admin"); - try (Response response = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles", - testHelper.localPort, currentCatalogName)) - .request("application/json") - .header("Authorization", "Bearer " + testHelper.adminToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .post(Entity.json(newRole))) { - assertThat(response) - .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - CatalogGrant grantResource = - new CatalogGrant( - CatalogPrivilege.CATALOG_MANAGE_CONTENT, GrantResource.TypeEnum.CATALOG); - try (Response response = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/custom-admin/grants", - testHelper.localPort, currentCatalogName)) - .request("application/json") - .header("Authorization", "Bearer " + testHelper.adminToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .put(Entity.json(grantResource))) { - assertThat(response) - .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - CatalogGrant grantAccessResource = - new CatalogGrant( - CatalogPrivilege.CATALOG_MANAGE_ACCESS, GrantResource.TypeEnum.CATALOG); - try (Response response = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/custom-admin/grants", - testHelper.localPort, currentCatalogName)) - .request("application/json") - .header("Authorization", "Bearer " + testHelper.adminToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .put(Entity.json(grantAccessResource))) { - assertThat(response) - .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - - // Assign this new CatalogRole to the service_admin PrincipalRole - try (Response response = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/custom-admin", - testHelper.localPort, currentCatalogName)) - .request("application/json") - .header("Authorization", "Bearer " + testHelper.adminToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .get()) { - assertThat(response) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus); - CatalogRole catalogRole = response.readEntity(CatalogRole.class); - try (Response assignResponse = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/management/v1/principal-roles/catalog-admin/catalog-roles/%s", - testHelper.localPort, currentCatalogName)) - .request("application/json") - .header("Authorization", "Bearer " + testHelper.adminToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .put(Entity.json(catalogRole))) { - assertThat(assignResponse) - .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - } - - SessionCatalog.SessionContext context = SessionCatalog.SessionContext.createEmpty(); - this.restCatalog = - new RESTCatalog( - context, - (config) -> - HTTPClient.builder(config) - .uri(config.get(CatalogProperties.URI)) - .build()); - this.restCatalog.initialize( - "polaris", - ImmutableMap.of( - CatalogProperties.URI, - "http://localhost:" + testHelper.localPort + "/api/catalog", - OAuth2Properties.CREDENTIAL, - testHelper.snowmanCredentials.clientId() - + ":" - + testHelper.snowmanCredentials.clientSecret(), - OAuth2Properties.SCOPE, - BasePolarisAuthenticator.PRINCIPAL_ROLE_ALL, - CatalogProperties.FILE_IO_IMPL, - "org.apache.iceberg.inmemory.InMemoryFileIO", - "warehouse", - currentCatalogName, - "header." + REALM_PROPERTY_KEY, - testHelper.realm)); - }); - } - - @Override - protected RESTCatalog catalog() { - return restCatalog; - } - - @Override - protected boolean requiresNamespaceCreate() { - return true; - } - - @Override - protected boolean supportsNestedNamespaces() { - return true; - } - - @Override - protected boolean supportsServerSideRetry() { - return true; - } - - @Override - protected boolean overridesRequestedLocation() { - return true; - } - - private void createCatalogRole(String catalogRoleName) { - CatalogRole catalogRole = new CatalogRole(catalogRoleName); - try (Response response = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles", - testHelper.localPort, currentCatalogName)) - .request("application/json") - .header("Authorization", "Bearer " + testHelper.userToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .post(Entity.json(catalogRole))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - } - - private void addGrant(String catalogRoleName, GrantResource grant) { - try (Response response = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/%s/grants", - testHelper.localPort, currentCatalogName, catalogRoleName)) - .request("application/json") - .header("Authorization", "Bearer " + testHelper.userToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .put(Entity.json(grant))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - } - - @Test - public void testListGrantsOnCatalogObjectsToCatalogRoles() { - restCatalog.createNamespace(Namespace.of("ns1")); - restCatalog.createNamespace(Namespace.of("ns1", "ns1a")); - restCatalog.createNamespace(Namespace.of("ns2")); - - restCatalog.buildTable(TableIdentifier.of(Namespace.of("ns1"), "tbl1"), SCHEMA).create(); - restCatalog - .buildTable(TableIdentifier.of(Namespace.of("ns1", "ns1a"), "tbl1"), SCHEMA) - .create(); - restCatalog.buildTable(TableIdentifier.of(Namespace.of("ns2"), "tbl2"), SCHEMA).create(); - - restCatalog - .buildView(TableIdentifier.of(Namespace.of("ns1"), "view1")) - .withSchema(SCHEMA) - .withDefaultNamespace(Namespace.of("ns1")) - .withQuery("spark", VIEW_QUERY) - .create(); - restCatalog - .buildView(TableIdentifier.of(Namespace.of("ns1", "ns1a"), "view1")) - .withSchema(SCHEMA) - .withDefaultNamespace(Namespace.of("ns1")) - .withQuery("spark", VIEW_QUERY) - .create(); - restCatalog - .buildView(TableIdentifier.of(Namespace.of("ns2"), "view2")) - .withSchema(SCHEMA) - .withDefaultNamespace(Namespace.of("ns1")) - .withQuery("spark", VIEW_QUERY) - .create(); - - CatalogGrant catalogGrant1 = - new CatalogGrant(CatalogPrivilege.CATALOG_MANAGE_CONTENT, GrantResource.TypeEnum.CATALOG); - - CatalogGrant catalogGrant2 = - new CatalogGrant(CatalogPrivilege.NAMESPACE_FULL_METADATA, GrantResource.TypeEnum.CATALOG); - - CatalogGrant catalogGrant3 = - new CatalogGrant(CatalogPrivilege.VIEW_FULL_METADATA, GrantResource.TypeEnum.CATALOG); - - NamespaceGrant namespaceGrant1 = - new NamespaceGrant( - List.of("ns1"), - NamespacePrivilege.NAMESPACE_FULL_METADATA, - GrantResource.TypeEnum.NAMESPACE); - - NamespaceGrant namespaceGrant2 = - new NamespaceGrant( - List.of("ns1", "ns1a"), - NamespacePrivilege.TABLE_CREATE, - GrantResource.TypeEnum.NAMESPACE); - - NamespaceGrant namespaceGrant3 = - new NamespaceGrant( - List.of("ns2"), - NamespacePrivilege.VIEW_READ_PROPERTIES, - GrantResource.TypeEnum.NAMESPACE); - - TableGrant tableGrant1 = - new TableGrant( - List.of("ns1"), - "tbl1", - TablePrivilege.TABLE_FULL_METADATA, - GrantResource.TypeEnum.TABLE); - - TableGrant tableGrant2 = - new TableGrant( - List.of("ns1", "ns1a"), - "tbl1", - TablePrivilege.TABLE_READ_DATA, - GrantResource.TypeEnum.TABLE); - - TableGrant tableGrant3 = - new TableGrant( - List.of("ns2"), "tbl2", TablePrivilege.TABLE_WRITE_DATA, GrantResource.TypeEnum.TABLE); - - ViewGrant viewGrant1 = - new ViewGrant( - List.of("ns1"), "view1", ViewPrivilege.VIEW_FULL_METADATA, GrantResource.TypeEnum.VIEW); - - ViewGrant viewGrant2 = - new ViewGrant( - List.of("ns1", "ns1a"), - "view1", - ViewPrivilege.VIEW_READ_PROPERTIES, - GrantResource.TypeEnum.VIEW); - - ViewGrant viewGrant3 = - new ViewGrant( - List.of("ns2"), - "view2", - ViewPrivilege.VIEW_WRITE_PROPERTIES, - GrantResource.TypeEnum.VIEW); - - createCatalogRole("catalogrole1"); - createCatalogRole("catalogrole2"); - - List role1Grants = - List.of( - catalogGrant1, - catalogGrant2, - namespaceGrant1, - namespaceGrant2, - tableGrant1, - tableGrant2, - viewGrant1, - viewGrant2); - role1Grants.forEach(grant -> addGrant("catalogrole1", grant)); - List role2Grants = - List.of( - catalogGrant1, - catalogGrant3, - namespaceGrant1, - namespaceGrant3, - tableGrant1, - tableGrant3, - viewGrant1, - viewGrant3); - role2Grants.forEach(grant -> addGrant("catalogrole2", grant)); - - // List grants for catalogrole1 - try (Response response = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/%s/grants", - testHelper.localPort, currentCatalogName, "catalogrole1")) - .request("application/json") - .header("Authorization", "Bearer " + testHelper.userToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .get()) { - assertThat(response) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus) - .extracting(r -> r.readEntity(GrantResources.class)) - .extracting(GrantResources::getGrants) - .asInstanceOf(InstanceOfAssertFactories.list(GrantResource.class)) - .containsExactlyInAnyOrder(role1Grants.toArray(new GrantResource[0])); - } - - // List grants for catalogrole2 - try (Response response = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/%s/grants", - testHelper.localPort, currentCatalogName, "catalogrole2")) - .request("application/json") - .header("Authorization", "Bearer " + testHelper.userToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .get()) { - assertThat(response) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus) - .extracting(r -> r.readEntity(GrantResources.class)) - .extracting(GrantResources::getGrants) - .asInstanceOf(InstanceOfAssertFactories.list(GrantResource.class)) - .containsExactlyInAnyOrder(role2Grants.toArray(new GrantResource[0])); - } - } - - @Test - public void testListGrantsAfterRename() { - restCatalog.createNamespace(Namespace.of("ns1")); - restCatalog.createNamespace(Namespace.of("ns1", "ns1a")); - restCatalog.createNamespace(Namespace.of("ns2")); - - restCatalog - .buildTable(TableIdentifier.of(Namespace.of("ns1", "ns1a"), "tbl1"), SCHEMA) - .create(); - - TableGrant tableGrant1 = - new TableGrant( - List.of("ns1", "ns1a"), - "tbl1", - TablePrivilege.TABLE_FULL_METADATA, - GrantResource.TypeEnum.TABLE); - - createCatalogRole("catalogrole1"); - addGrant("catalogrole1", tableGrant1); - - // Grants will follow the table through the rename - restCatalog.renameTable( - TableIdentifier.of(Namespace.of("ns1", "ns1a"), "tbl1"), - TableIdentifier.of(Namespace.of("ns2"), "newtable")); - - TableGrant expectedGrant = - new TableGrant( - List.of("ns2"), - "newtable", - TablePrivilege.TABLE_FULL_METADATA, - GrantResource.TypeEnum.TABLE); - - try (Response response = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/%s/grants", - testHelper.localPort, currentCatalogName, "catalogrole1")) - .request("application/json") - .header("Authorization", "Bearer " + testHelper.userToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .get()) { - assertThat(response) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus) - .extracting(r -> r.readEntity(GrantResources.class)) - .extracting(GrantResources::getGrants) - .asInstanceOf(InstanceOfAssertFactories.list(GrantResource.class)) - .containsExactly(expectedGrant); - } - } - - @Test - public void testCreateTableWithOverriddenBaseLocation() { - try (Response response = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - testHelper.localPort, currentCatalogName)) - .request("application/json") - .header("Authorization", "Bearer " + testHelper.adminToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .get()) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - Catalog catalog = response.readEntity(Catalog.class); - Map catalogProps = new HashMap<>(catalog.getProperties().toMap()); - catalogProps.put( - PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "false"); - try (Response updateResponse = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - testHelper.localPort, catalog.getName())) - .request("application/json") - .header("Authorization", "Bearer " + testHelper.adminToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .put( - Entity.json( - new UpdateCatalogRequest( - catalog.getEntityVersion(), - catalogProps, - catalog.getStorageConfigInfo())))) { - assertThat(updateResponse).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - } - } - - restCatalog.createNamespace(Namespace.of("ns1")); - restCatalog.createNamespace( - Namespace.of("ns1", "ns1a"), - ImmutableMap.of( - PolarisEntityConstants.ENTITY_BASE_LOCATION, - catalogBaseLocation + "/ns1/ns1a-override")); - - TableIdentifier tableIdentifier = TableIdentifier.of(Namespace.of("ns1", "ns1a"), "tbl1"); - restCatalog - .buildTable(tableIdentifier, SCHEMA) - .withLocation(catalogBaseLocation + "/ns1/ns1a-override/tbl1-override") - .create(); - Table table = restCatalog.loadTable(tableIdentifier); - assertThat(table) - .isNotNull() - .isInstanceOf(BaseTable.class) - .asInstanceOf(InstanceOfAssertFactories.type(BaseTable.class)) - .returns(catalogBaseLocation + "/ns1/ns1a-override/tbl1-override", BaseTable::location); - } - - @Test - public void testCreateTableWithOverriddenBaseLocationCannotOverlapSibling() { - try (Response response = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - testHelper.localPort, currentCatalogName)) - .request("application/json") - .header("Authorization", "Bearer " + testHelper.adminToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .get()) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - Catalog catalog = response.readEntity(Catalog.class); - Map catalogProps = new HashMap<>(catalog.getProperties().toMap()); - catalogProps.put( - PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "false"); - try (Response updateResponse = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - testHelper.localPort, catalog.getName())) - .request("application/json") - .header("Authorization", "Bearer " + testHelper.adminToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .put( - Entity.json( - new UpdateCatalogRequest( - catalog.getEntityVersion(), - catalogProps, - catalog.getStorageConfigInfo())))) { - assertThat(updateResponse).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - } - } - - restCatalog.createNamespace(Namespace.of("ns1")); - restCatalog.createNamespace( - Namespace.of("ns1", "ns1a"), - ImmutableMap.of( - PolarisEntityConstants.ENTITY_BASE_LOCATION, - catalogBaseLocation + "/ns1/ns1a-override")); - - TableIdentifier tableIdentifier = TableIdentifier.of(Namespace.of("ns1", "ns1a"), "tbl1"); - restCatalog - .buildTable(tableIdentifier, SCHEMA) - .withLocation(catalogBaseLocation + "/ns1/ns1a-override/tbl1-override") - .create(); - Table table = restCatalog.loadTable(tableIdentifier); - assertThat(table) - .isNotNull() - .isInstanceOf(BaseTable.class) - .asInstanceOf(InstanceOfAssertFactories.type(BaseTable.class)) - .returns(catalogBaseLocation + "/ns1/ns1a-override/tbl1-override", BaseTable::location); - - Assertions.assertThatThrownBy( - () -> - restCatalog - .buildTable(TableIdentifier.of(Namespace.of("ns1", "ns1a"), "tbl2"), SCHEMA) - .withLocation(catalogBaseLocation + "/ns1/ns1a-override/tbl1-override") - .create()) - .isInstanceOf(ForbiddenException.class) - .hasMessageContaining("because it conflicts with existing table or namespace"); - } - - @Test - public void testCreateTableWithOverriddenBaseLocationMustResideInNsDirectory() { - try (Response response = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - testHelper.localPort, currentCatalogName)) - .request("application/json") - .header("Authorization", "Bearer " + testHelper.adminToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .get()) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - Catalog catalog = response.readEntity(Catalog.class); - Map catalogProps = new HashMap<>(catalog.getProperties().toMap()); - catalogProps.put( - PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "false"); - try (Response updateResponse = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - testHelper.localPort, catalog.getName())) - .request("application/json") - .header("Authorization", "Bearer " + testHelper.adminToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .put( - Entity.json( - new UpdateCatalogRequest( - catalog.getEntityVersion(), - catalogProps, - catalog.getStorageConfigInfo())))) { - assertThat(updateResponse).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - } - } - - restCatalog.createNamespace(Namespace.of("ns1")); - restCatalog.createNamespace( - Namespace.of("ns1", "ns1a"), - ImmutableMap.of( - PolarisEntityConstants.ENTITY_BASE_LOCATION, - catalogBaseLocation + "/ns1/ns1a-override")); - - TableIdentifier tableIdentifier = TableIdentifier.of(Namespace.of("ns1", "ns1a"), "tbl1"); - assertThatThrownBy( - () -> - restCatalog - .buildTable(tableIdentifier, SCHEMA) - .withLocation(catalogBaseLocation + "/ns1/ns1a/tbl1-override") - .create()) - .isInstanceOf(ForbiddenException.class); - } - - @Test - public void testSendNotificationInternalCatalog() { - NotificationRequest notification = new NotificationRequest(); - notification.setNotificationType(NotificationType.CREATE); - notification.setPayload( - new TableUpdateNotification( - "tbl1", - System.currentTimeMillis(), - UUID.randomUUID().toString(), - "s3://my-bucket/path/to/metadata.json", - null)); - restCatalog.createNamespace(Namespace.of("ns1")); - String notificationUrl = - String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/ns1/tables/tbl1/notifications", - testHelper.localPort, currentCatalogName); - try (Response response = - testHelper - .client - .target(notificationUrl) - .request("application/json") - .header("Authorization", "Bearer " + testHelper.userToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .post(Entity.json(notification))) { - assertThat(response) - .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus) - .extracting(r -> r.readEntity(ErrorResponse.class)) - .returns("Cannot update internal catalog via notifications", ErrorResponse::message); - } - - // NotificationType.VALIDATE should also surface the same error. - notification.setNotificationType(NotificationType.VALIDATE); - try (Response response = - testHelper - .client - .target(notificationUrl) - .request("application/json") - .header("Authorization", "Bearer " + testHelper.userToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .post(Entity.json(notification))) { - assertThat(response) - .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus) - .extracting(r -> r.readEntity(ErrorResponse.class)) - .returns("Cannot update internal catalog via notifications", ErrorResponse::message); - } - } - - // Test copied from iceberg/core/src/test/java/org/apache/iceberg/rest/TestRESTCatalog.java - // TODO: If TestRESTCatalog can be refactored to be more usable as a shared base test class, - // just inherit these test cases from that instead of copying them here. - @Test - public void diffAgainstSingleTable() { - Namespace namespace = Namespace.of("namespace"); - TableIdentifier identifier = TableIdentifier.of(namespace, "multipleDiffsAgainstSingleTable"); - - if (requiresNamespaceCreate()) { - catalog().createNamespace(namespace); - } - - Table table = catalog().buildTable(identifier, SCHEMA).create(); - Transaction transaction = table.newTransaction(); - - UpdateSchema updateSchema = - transaction.updateSchema().addColumn("new_col", Types.LongType.get()); - Schema expectedSchema = updateSchema.apply(); - updateSchema.commit(); - - UpdatePartitionSpec updateSpec = - transaction.updateSpec().addField("shard", Expressions.bucket("id", 16)); - PartitionSpec expectedSpec = updateSpec.apply(); - updateSpec.commit(); - - TableCommit tableCommit = - TableCommit.create( - identifier, - ((BaseTransaction) transaction).startMetadata(), - ((BaseTransaction) transaction).currentMetadata()); - - restCatalog.commitTransaction(tableCommit); - - Table loaded = catalog().loadTable(identifier); - assertThat(loaded.schema().asStruct()).isEqualTo(expectedSchema.asStruct()); - assertThat(loaded.spec().fields()).isEqualTo(expectedSpec.fields()); - } - - // Test copied from iceberg/core/src/test/java/org/apache/iceberg/rest/TestRESTCatalog.java - // TODO: If TestRESTCatalog can be refactored to be more usable as a shared base test class, - // just inherit these test cases from that instead of copying them here. - @Test - public void multipleDiffsAgainstMultipleTables() { - Namespace namespace = Namespace.of("multiDiffNamespace"); - TableIdentifier identifier1 = TableIdentifier.of(namespace, "multiDiffTable1"); - TableIdentifier identifier2 = TableIdentifier.of(namespace, "multiDiffTable2"); - - if (requiresNamespaceCreate()) { - catalog().createNamespace(namespace); - } - - Table table1 = catalog().buildTable(identifier1, SCHEMA).create(); - Table table2 = catalog().buildTable(identifier2, SCHEMA).create(); - Transaction t1Transaction = table1.newTransaction(); - Transaction t2Transaction = table2.newTransaction(); - - UpdateSchema updateSchema = - t1Transaction.updateSchema().addColumn("new_col", Types.LongType.get()); - Schema expectedSchema = updateSchema.apply(); - updateSchema.commit(); - - UpdateSchema updateSchema2 = - t2Transaction.updateSchema().addColumn("new_col2", Types.LongType.get()); - Schema expectedSchema2 = updateSchema2.apply(); - updateSchema2.commit(); - - TableCommit tableCommit1 = - TableCommit.create( - identifier1, - ((BaseTransaction) t1Transaction).startMetadata(), - ((BaseTransaction) t1Transaction).currentMetadata()); - - TableCommit tableCommit2 = - TableCommit.create( - identifier2, - ((BaseTransaction) t2Transaction).startMetadata(), - ((BaseTransaction) t2Transaction).currentMetadata()); - - restCatalog.commitTransaction(tableCommit1, tableCommit2); - - assertThat(catalog().loadTable(identifier1).schema().asStruct()) - .isEqualTo(expectedSchema.asStruct()); - - assertThat(catalog().loadTable(identifier2).schema().asStruct()) - .isEqualTo(expectedSchema2.asStruct()); - } - - // Test copied from iceberg/core/src/test/java/org/apache/iceberg/rest/TestRESTCatalog.java - // TODO: If TestRESTCatalog can be refactored to be more usable as a shared base test class, - // just inherit these test cases from that instead of copying them here. - @Test - public void multipleDiffsAgainstMultipleTablesLastFails() { - Namespace namespace = Namespace.of("multiDiffNamespace"); - TableIdentifier identifier1 = TableIdentifier.of(namespace, "multiDiffTable1"); - TableIdentifier identifier2 = TableIdentifier.of(namespace, "multiDiffTable2"); - - if (requiresNamespaceCreate()) { - catalog().createNamespace(namespace); - } - - catalog().createTable(identifier1, SCHEMA); - catalog().createTable(identifier2, SCHEMA); - - Table table1 = catalog().loadTable(identifier1); - Table table2 = catalog().loadTable(identifier2); - Schema originalSchemaOne = table1.schema(); - - Transaction t1Transaction = catalog().loadTable(identifier1).newTransaction(); - t1Transaction.updateSchema().addColumn("new_col1", Types.LongType.get()).commit(); - - Transaction t2Transaction = catalog().loadTable(identifier2).newTransaction(); - t2Transaction.updateSchema().renameColumn("data", "new-column").commit(); - - // delete the colum that is being renamed in the above TX to cause a conflict - table2.updateSchema().deleteColumn("data").commit(); - Schema updatedSchemaTwo = table2.schema(); - - TableCommit tableCommit1 = - TableCommit.create( - identifier1, - ((BaseTransaction) t1Transaction).startMetadata(), - ((BaseTransaction) t1Transaction).currentMetadata()); - - TableCommit tableCommit2 = - TableCommit.create( - identifier2, - ((BaseTransaction) t2Transaction).startMetadata(), - ((BaseTransaction) t2Transaction).currentMetadata()); - - assertThatThrownBy(() -> restCatalog.commitTransaction(tableCommit1, tableCommit2)) - .isInstanceOf(CommitFailedException.class) - .hasMessageContaining("Requirement failed: current schema changed: expected id 0 != 1"); - - Schema schema1 = catalog().loadTable(identifier1).schema(); - assertThat(schema1.asStruct()).isEqualTo(originalSchemaOne.asStruct()); - - Schema schema2 = catalog().loadTable(identifier2).schema(); - assertThat(schema2.asStruct()).isEqualTo(updatedSchemaTwo.asStruct()); - assertThat(schema2.findField("data")).isNull(); - assertThat(schema2.findField("new-column")).isNull(); - assertThat(schema2.columns()).hasSize(1); - } - - @Test - public void testMultipleConflictingCommitsToSingleTableInTransaction() { - Namespace namespace = Namespace.of("ns1"); - TableIdentifier identifier = - TableIdentifier.of(namespace, "multipleConflictingCommitsAgainstSingleTable"); - - if (requiresNamespaceCreate()) { - catalog().createNamespace(namespace); - } - - // Start two independent transactions on the same base table. - Table table = catalog().buildTable(identifier, SCHEMA).create(); - Schema originalSchema = catalog().loadTable(identifier).schema(); - Transaction transaction1 = table.newTransaction(); - Transaction transaction2 = table.newTransaction(); - - transaction1.updateSchema().renameColumn("data", "new-column1").commit(); - transaction2.updateSchema().renameColumn("data", "new-column2").commit(); - - TableCommit tableCommit1 = - TableCommit.create( - identifier, - ((BaseTransaction) transaction1).startMetadata(), - ((BaseTransaction) transaction1).currentMetadata()); - TableCommit tableCommit2 = - TableCommit.create( - identifier, - ((BaseTransaction) transaction2).startMetadata(), - ((BaseTransaction) transaction2).currentMetadata()); - - // "Initial" commit requirements will succeed for both commits being based on the original - // table but should fail the entire transaction on the second commit. - assertThatThrownBy(() -> restCatalog.commitTransaction(tableCommit1, tableCommit2)) - .isInstanceOf(CommitFailedException.class); - - // If an implementation validates all UpdateRequirements up-front, then it might pass - // tests where the UpdateRequirement fails up-front without being atomic. Here we can - // catch such scenarios where update requirements appear to be fine up-front but will - // fail when trying to commit the second update, and verify that nothing was actually - // committed in the end. - Schema latestCommittedSchema = catalog().loadTable(identifier).schema(); - assertThat(latestCommittedSchema.asStruct()).isEqualTo(originalSchema.asStruct()); - } -} diff --git a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogViewIntegrationTest.java b/polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogViewIntegrationTest.java deleted file mode 100644 index 57adb301e..000000000 --- a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogViewIntegrationTest.java +++ /dev/null @@ -1,282 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.catalog; - -import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; -import static org.assertj.core.api.Assertions.assertThat; - -import com.google.common.collect.ImmutableMap; -import io.quarkus.test.junit.QuarkusTest; -import jakarta.inject.Inject; -import jakarta.ws.rs.client.Entity; -import jakarta.ws.rs.core.Response; -import java.lang.reflect.Field; -import java.nio.file.Path; -import java.util.List; -import java.util.Optional; -import org.apache.iceberg.CatalogProperties; -import org.apache.iceberg.catalog.SessionCatalog; -import org.apache.iceberg.rest.HTTPClient; -import org.apache.iceberg.rest.RESTCatalog; -import org.apache.iceberg.rest.auth.OAuth2Properties; -import org.apache.iceberg.view.ViewCatalogTests; -import org.apache.polaris.core.PolarisConfiguration; -import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; -import org.apache.polaris.core.admin.model.Catalog; -import org.apache.polaris.core.admin.model.CatalogGrant; -import org.apache.polaris.core.admin.model.CatalogPrivilege; -import org.apache.polaris.core.admin.model.CatalogRole; -import org.apache.polaris.core.admin.model.FileStorageConfigInfo; -import org.apache.polaris.core.admin.model.GrantResource; -import org.apache.polaris.core.admin.model.PolarisCatalog; -import org.apache.polaris.core.admin.model.StorageConfigInfo; -import org.apache.polaris.core.entity.CatalogEntity; -import org.apache.polaris.service.auth.BasePolarisAuthenticator; -import org.apache.polaris.service.test.PolarisIntegrationTestHelper; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.TestInfo; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.io.TempDir; - -/** - * Import the full core Iceberg catalog tests by hitting the REST service via the RESTCatalog - * client. - */ -@QuarkusTest -@TestInstance(TestInstance.Lifecycle.PER_CLASS) -public class PolarisRestCatalogViewIntegrationTest extends ViewCatalogTests { - public static final String TEST_ROLE_ARN = - Optional.ofNullable(System.getenv("INTEGRATION_TEST_ROLE_ARN")) - .orElse("arn:aws:iam::123456789012:role/my-role"); - public static final String S3_BUCKET_BASE = - Optional.ofNullable(System.getenv("INTEGRATION_TEST_S3_PATH")) - .orElse("file:///tmp/buckets/my-bucket"); - - private RESTCatalog restCatalog; - - @Inject PolarisIntegrationTestHelper testHelper; - - @BeforeAll - public void setUp(TestInfo testInfo) { - testHelper.setUp(testInfo); - } - - @AfterAll - public void tearDown() { - testHelper.tearDown(); - } - - @BeforeEach - public void setUpTempDir(@TempDir Path tempDir) throws Exception { - // see https://github.com/quarkusio/quarkus/issues/13261 - Field field = ViewCatalogTests.class.getDeclaredField("tempDir"); - field.setAccessible(true); - field.set(this, tempDir); - } - - @BeforeEach - void before(TestInfo testInfo) { - testInfo - .getTestMethod() - .ifPresent( - method -> { - String catalogName = method.getName(); - try (Response response = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - testHelper.localPort, catalogName)) - .request("application/json") - .header("Authorization", "Bearer " + testHelper.adminToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .get()) { - if (response.getStatus() == Response.Status.OK.getStatusCode()) { - // Already exists! Must be in a parameterized test. - // Quick hack to get a unique catalogName. - // TODO: Have a while-loop instead with consecutive incrementing suffixes. - catalogName = catalogName + System.currentTimeMillis(); - } - } - - AwsStorageConfigInfo awsConfigModel = - AwsStorageConfigInfo.builder() - .setRoleArn(TEST_ROLE_ARN) - .setExternalId("externalId") - .setUserArn("userArn") - .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) - .setAllowedLocations(List.of("s3://my-old-bucket/path/to/data")) - .build(); - org.apache.polaris.core.admin.model.CatalogProperties props = - org.apache.polaris.core.admin.model.CatalogProperties.builder( - S3_BUCKET_BASE + "/" + System.getenv("USER") + "/path/to/data") - .addProperty( - CatalogEntity.REPLACE_NEW_LOCATION_PREFIX_WITH_CATALOG_DEFAULT_KEY, - "file:") - .addProperty( - PolarisConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), - "true") - .addProperty( - PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), - "true") - .build(); - Catalog catalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName(catalogName) - .setProperties(props) - .setStorageConfigInfo( - S3_BUCKET_BASE.startsWith("file:") - ? new FileStorageConfigInfo( - StorageConfigInfo.StorageTypeEnum.FILE, List.of("file://")) - : awsConfigModel) - .build(); - try (Response response = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs", - testHelper.localPort)) - .request("application/json") - .header("Authorization", "Bearer " + testHelper.adminToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .post(Entity.json(catalog))) { - assertThat(response) - .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - CatalogRole newRole = new CatalogRole("admin"); - try (Response response = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles", - testHelper.localPort, catalogName)) - .request("application/json") - .header("Authorization", "Bearer " + testHelper.adminToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .post(Entity.json(newRole))) { - assertThat(response) - .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - CatalogGrant grantResource = - new CatalogGrant( - CatalogPrivilege.CATALOG_MANAGE_CONTENT, GrantResource.TypeEnum.CATALOG); - try (Response response = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/admin/grants", - testHelper.localPort, catalogName)) - .request("application/json") - .header("Authorization", "Bearer " + testHelper.adminToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .put(Entity.json(grantResource))) { - assertThat(response) - .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - - try (Response response = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/admin", - testHelper.localPort, catalogName)) - .request("application/json") - .header("Authorization", "Bearer " + testHelper.adminToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .get()) { - assertThat(response) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus); - CatalogRole catalogRole = response.readEntity(CatalogRole.class); - try (Response ignore = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/management/v1/principal-roles/catalog-admin/catalog-roles/%s", - testHelper.localPort, catalogName)) - .request("application/json") - .header("Authorization", "Bearer " + testHelper.adminToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .put(Entity.json(catalogRole))) { - assertThat(response) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus); - } - } - - SessionCatalog.SessionContext context = SessionCatalog.SessionContext.createEmpty(); - this.restCatalog = - new RESTCatalog( - context, - (config) -> - HTTPClient.builder(config) - .uri(config.get(CatalogProperties.URI)) - .build()); - this.restCatalog.initialize( - "polaris", - ImmutableMap.of( - CatalogProperties.URI, - "http://localhost:" + testHelper.localPort + "/api/catalog", - OAuth2Properties.CREDENTIAL, - testHelper.snowmanCredentials.clientId() - + ":" - + testHelper.snowmanCredentials.clientSecret(), - OAuth2Properties.SCOPE, - BasePolarisAuthenticator.PRINCIPAL_ROLE_ALL, - CatalogProperties.FILE_IO_IMPL, - "org.apache.iceberg.inmemory.InMemoryFileIO", - "warehouse", - catalogName, - "header." + REALM_PROPERTY_KEY, - testHelper.realm)); - }); - } - - @Override - protected RESTCatalog catalog() { - return restCatalog; - } - - @Override - protected org.apache.iceberg.catalog.Catalog tableCatalog() { - return restCatalog; - } - - @Override - protected boolean requiresNamespaceCreate() { - return true; - } - - @Override - protected boolean supportsServerSideRetry() { - return true; - } - - @Override - protected boolean overridesRequestedLocation() { - return true; - } -} diff --git a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/PolarisSparkIntegrationTest.java b/polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/PolarisSparkIntegrationTest.java deleted file mode 100644 index ffd032006..000000000 --- a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/PolarisSparkIntegrationTest.java +++ /dev/null @@ -1,389 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.catalog; - -import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -import com.adobe.testing.s3mock.testcontainers.S3MockContainer; -import io.quarkus.test.junit.QuarkusTest; -import jakarta.inject.Inject; -import jakarta.ws.rs.client.Entity; -import jakarta.ws.rs.core.Response; -import java.time.Instant; -import java.util.List; -import java.util.Map; -import org.apache.iceberg.rest.requests.ImmutableRegisterTableRequest; -import org.apache.iceberg.rest.responses.LoadTableResponse; -import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; -import org.apache.polaris.core.admin.model.Catalog; -import org.apache.polaris.core.admin.model.CatalogProperties; -import org.apache.polaris.core.admin.model.ExternalCatalog; -import org.apache.polaris.core.admin.model.PolarisCatalog; -import org.apache.polaris.core.admin.model.StorageConfigInfo; -import org.apache.polaris.service.test.PolarisIntegrationTestHelper; -import org.apache.polaris.service.types.NotificationRequest; -import org.apache.polaris.service.types.NotificationType; -import org.apache.polaris.service.types.TableUpdateNotification; -import org.apache.spark.sql.Dataset; -import org.apache.spark.sql.Row; -import org.apache.spark.sql.SparkSession; -import org.intellij.lang.annotations.Language; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInfo; -import org.junit.jupiter.api.TestInstance; -import org.slf4j.LoggerFactory; - -@QuarkusTest -@TestInstance(TestInstance.Lifecycle.PER_CLASS) -public class PolarisSparkIntegrationTest { - - public static final String CATALOG_NAME = "mycatalog"; - public static final String EXTERNAL_CATALOG_NAME = "external_catalog"; - - private final S3MockContainer s3Container = - new S3MockContainer("3.11.0").withInitialBuckets("my-bucket,my-old-bucket"); - - private SparkSession spark; - - @Inject PolarisIntegrationTestHelper testHelper; - - @BeforeAll - public void setUp(TestInfo testInfo) { - s3Container.start(); - testHelper.setUp(testInfo); - } - - @AfterAll - public void tearDown() { - testHelper.tearDown(); - } - - @AfterAll - public void cleanup() { - s3Container.stop(); - } - - @BeforeEach - public void before() { - AwsStorageConfigInfo awsConfigModel = - AwsStorageConfigInfo.builder() - .setRoleArn("arn:aws:iam::123456789012:role/my-role") - .setExternalId("externalId") - .setUserArn("userArn") - .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) - .setAllowedLocations(List.of("s3://my-old-bucket/path/to/data")) - .build(); - CatalogProperties props = new CatalogProperties("s3://my-bucket/path/to/data"); - props.putAll( - Map.of( - "table-default.s3.endpoint", - s3Container.getHttpEndpoint(), - "table-default.s3.path-style-access", - "true", - "table-default.s3.access-key-id", - "foo", - "table-default.s3.secret-access-key", - "bar", - "s3.endpoint", - s3Container.getHttpEndpoint(), - "s3.path-style-access", - "true", - "s3.access-key-id", - "foo", - "s3.secret-access-key", - "bar")); - Catalog catalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName(CATALOG_NAME) - .setProperties(props) - .setStorageConfigInfo(awsConfigModel) - .build(); - - try (Response response = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs", testHelper.localPort)) - .request("application/json") - .header("Authorization", "BEARER " + testHelper.adminToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .post(Entity.json(catalog))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - - CatalogProperties externalProps = new CatalogProperties("s3://my-bucket/path/to/data"); - externalProps.putAll( - Map.of( - "table-default.s3.endpoint", - s3Container.getHttpEndpoint(), - "table-default.s3.path-style-access", - "true", - "table-default.s3.access-key-id", - "foo", - "table-default.s3.secret-access-key", - "bar", - "s3.endpoint", - s3Container.getHttpEndpoint(), - "s3.path-style-access", - "true", - "s3.access-key-id", - "foo", - "s3.secret-access-key", - "bar")); - Catalog externalCatalog = - ExternalCatalog.builder() - .setType(Catalog.TypeEnum.EXTERNAL) - .setName(EXTERNAL_CATALOG_NAME) - .setProperties(externalProps) - .setStorageConfigInfo(awsConfigModel) - .setRemoteUrl("http://dummy_url") - .build(); - try (Response response = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs", testHelper.localPort)) - .request("application/json") - .header("Authorization", "BEARER " + testHelper.adminToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .post(Entity.json(externalCatalog))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - SparkSession.Builder sessionBuilder = - SparkSession.builder() - .master("local[1]") - .config("spark.hadoop.fs.s3.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem") - .config( - "spark.hadoop.fs.s3.aws.credentials.provider", - "org.apache.hadoop.fs.s3.TemporaryAWSCredentialsProvider") - .config("spark.hadoop.fs.s3.access.key", "foo") - .config("spark.hadoop.fs.s3.secret.key", "bar") - .config( - "spark.sql.extensions", - "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions") - .config("spark.ui.showConsoleProgress", false) - .config("spark.ui.enabled", "false"); - spark = - withCatalog(withCatalog(sessionBuilder, CATALOG_NAME), EXTERNAL_CATALOG_NAME).getOrCreate(); - - onSpark("USE " + CATALOG_NAME); - } - - private SparkSession.Builder withCatalog(SparkSession.Builder builder, String catalogName) { - return builder - .config( - String.format("spark.sql.catalog.%s", catalogName), - "org.apache.iceberg.spark.SparkCatalog") - .config(String.format("spark.sql.catalog.%s.type", catalogName), "rest") - .config( - String.format("spark.sql.catalog.%s.uri", catalogName), - "http://localhost:" + testHelper.localPort + "/api/catalog") - .config(String.format("spark.sql.catalog.%s.warehouse", catalogName), catalogName) - .config(String.format("spark.sql.catalog.%s.scope", catalogName), "PRINCIPAL_ROLE:ALL") - .config(String.format("spark.sql.catalog.%s.header.realm", catalogName), testHelper.realm) - .config(String.format("spark.sql.catalog.%s.token", catalogName), testHelper.adminToken) - .config(String.format("spark.sql.catalog.%s.s3.access-key-id", catalogName), "fakekey") - .config( - String.format("spark.sql.catalog.%s.s3.secret-access-key", catalogName), "fakesecret") - .config(String.format("spark.sql.catalog.%s.s3.region", catalogName), "us-west-2"); - } - - @AfterEach - public void after() { - cleanupCatalog(CATALOG_NAME); - cleanupCatalog(EXTERNAL_CATALOG_NAME); - try { - SparkSession.clearDefaultSession(); - SparkSession.clearActiveSession(); - spark.close(); - } catch (Exception e) { - LoggerFactory.getLogger(getClass()).error("Unable to close spark session", e); - } - } - - private void cleanupCatalog(String catalogName) { - onSpark("USE " + catalogName); - List namespaces = onSpark("SHOW NAMESPACES").collectAsList(); - for (Row namespace : namespaces) { - List tables = onSpark("SHOW TABLES IN " + namespace.getString(0)).collectAsList(); - for (Row table : tables) { - onSpark("DROP TABLE " + namespace.getString(0) + "." + table.getString(1)); - } - List views = onSpark("SHOW VIEWS IN " + namespace.getString(0)).collectAsList(); - for (Row view : views) { - onSpark("DROP VIEW " + namespace.getString(0) + "." + view.getString(1)); - } - onSpark("DROP NAMESPACE " + namespace.getString(0)); - } - try (Response response = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/" + catalogName, - testHelper.localPort)) - .request("application/json") - .header("Authorization", "BEARER " + testHelper.adminToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .delete()) { - assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } - } - - @Test - public void testCreateTable() { - long namespaceCount = onSpark("SHOW NAMESPACES").count(); - assertThat(namespaceCount).isEqualTo(0L); - - onSpark("CREATE NAMESPACE ns1"); - onSpark("USE ns1"); - onSpark("CREATE TABLE tb1 (col1 integer, col2 string)"); - onSpark("INSERT INTO tb1 VALUES (1, 'a'), (2, 'b'), (3, 'c')"); - long recordCount = onSpark("SELECT * FROM tb1").count(); - assertThat(recordCount).isEqualTo(3); - } - - @Test - public void testCreateAndUpdateExternalTable() { - long namespaceCount = onSpark("SHOW NAMESPACES").count(); - assertThat(namespaceCount).isEqualTo(0L); - - onSpark("CREATE NAMESPACE ns1"); - onSpark("USE ns1"); - onSpark("CREATE TABLE tb1 (col1 integer, col2 string)"); - onSpark("INSERT INTO tb1 VALUES (1, 'a'), (2, 'b'), (3, 'c')"); - long recordCount = onSpark("SELECT * FROM tb1").count(); - assertThat(recordCount).isEqualTo(3); - - onSpark("USE " + EXTERNAL_CATALOG_NAME); - List existingNamespaces = onSpark("SHOW NAMESPACES").collectAsList(); - assertThat(existingNamespaces).isEmpty(); - - onSpark("CREATE NAMESPACE externalns1"); - onSpark("USE externalns1"); - List existingTables = onSpark("SHOW TABLES").collectAsList(); - assertThat(existingTables).isEmpty(); - - LoadTableResponse tableResponse = loadTable(CATALOG_NAME, "ns1", "tb1"); - try (Response registerResponse = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/catalog/v1/" - + EXTERNAL_CATALOG_NAME - + "/namespaces/externalns1/register", - testHelper.localPort)) - .request("application/json") - .header("Authorization", "BEARER " + testHelper.adminToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .post( - Entity.json( - ImmutableRegisterTableRequest.builder() - .name("mytb1") - .metadataLocation(tableResponse.metadataLocation()) - .build()))) { - assertThat(registerResponse).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - } - - long tableCount = onSpark("SHOW TABLES").count(); - assertThat(tableCount).isEqualTo(1); - List tables = onSpark("SHOW TABLES").collectAsList(); - assertThat(tables).hasSize(1).extracting(row -> row.getString(1)).containsExactly("mytb1"); - long rowCount = onSpark("SELECT * FROM mytb1").count(); - assertThat(rowCount).isEqualTo(3); - assertThatThrownBy(() -> onSpark("INSERT INTO mytb1 VALUES (20, 'new_text')")) - .isInstanceOf(Exception.class); - - onSpark("INSERT INTO " + CATALOG_NAME + ".ns1.tb1 VALUES (20, 'new_text')"); - tableResponse = loadTable(CATALOG_NAME, "ns1", "tb1"); - TableUpdateNotification updateNotification = - new TableUpdateNotification( - "mytb1", - Instant.now().toEpochMilli(), - tableResponse.tableMetadata().uuid(), - tableResponse.metadataLocation(), - tableResponse.tableMetadata()); - NotificationRequest notificationRequest = new NotificationRequest(); - notificationRequest.setPayload(updateNotification); - notificationRequest.setNotificationType(NotificationType.UPDATE); - try (Response notifyResponse = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/externalns1/tables/mytb1/notifications", - testHelper.localPort, EXTERNAL_CATALOG_NAME)) - .request("application/json") - .header("Authorization", "BEARER " + testHelper.adminToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .post(Entity.json(notificationRequest))) { - assertThat(notifyResponse) - .returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } - // refresh the table so it queries for the latest metadata.json - onSpark("REFRESH TABLE mytb1"); - rowCount = onSpark("SELECT * FROM mytb1").count(); - assertThat(rowCount).isEqualTo(4); - } - - @Test - public void testCreateView() { - long namespaceCount = onSpark("SHOW NAMESPACES").count(); - assertThat(namespaceCount).isEqualTo(0L); - - onSpark("CREATE NAMESPACE ns1"); - onSpark("USE ns1"); - onSpark("CREATE TABLE tb1 (col1 integer, col2 string)"); - onSpark("INSERT INTO tb1 VALUES (1, 'a'), (2, 'b'), (3, 'c')"); - onSpark("CREATE VIEW view1 AS SELECT * FROM tb1"); - long recordCount = onSpark("SELECT * FROM view1").count(); - assertThat(recordCount).isEqualTo(3); - } - - private LoadTableResponse loadTable(String catalog, String namespace, String table) { - try (Response response = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/tables/%s", - testHelper.localPort, catalog, namespace, table)) - .request("application/json") - .header("Authorization", "BEARER " + testHelper.adminToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .get()) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - return response.readEntity(LoadTableResponse.class); - } - } - - private Dataset onSpark(@Language("SQL") String sql) { - return spark.sql(sql); - } -} diff --git a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/io/MeasuredFileIOFactory.java b/polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/io/MeasuredFileIOFactory.java deleted file mode 100644 index 0def6dc88..000000000 --- a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/catalog/io/MeasuredFileIOFactory.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.catalog.io; - -import com.fasterxml.jackson.annotation.JsonTypeName; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import org.apache.hadoop.conf.Configuration; -import org.apache.iceberg.CatalogUtil; -import org.apache.iceberg.DataFile; -import org.apache.iceberg.DeleteFile; -import org.apache.iceberg.ManifestFile; -import org.apache.iceberg.io.FileIO; -import org.apache.iceberg.io.InputFile; -import org.apache.iceberg.io.OutputFile; - -/** A FileIOFactory that measures the number of bytes read, files written, and files deleted. */ -@JsonTypeName("measured") -public class MeasuredFileIOFactory implements FileIOFactory { - private final List ios; - - public MeasuredFileIOFactory() { - ios = new ArrayList<>(); - } - - @Override - public FileIO loadFileIO(String ioImpl, Map properties) { - MeasuredFileIO wrapped = - new MeasuredFileIO(CatalogUtil.loadFileIO(ioImpl, properties, new Configuration())); - ios.add(wrapped); - return wrapped; - } - - public long getInputBytes() { - long sum = 0; - for (MeasuredFileIO io : ios) { - sum += io.inputBytes; - } - return sum; - } - - public long getNumOutputFiles() { - long sum = 0; - for (MeasuredFileIO io : ios) { - sum += io.numOutputFiles; - } - return sum; - } - - public long getNumDeletedFiles() { - long sum = 0; - for (MeasuredFileIO io : ios) { - sum += io.numDeletedFiles; - } - return sum; - } - - public static class MeasuredFileIO implements FileIO { - private final FileIO io; - private long inputBytes; - private int numOutputFiles; - private int numDeletedFiles; - - public MeasuredFileIO(FileIO io) { - this.io = io; - } - - private InputFile measureInputFile(InputFile inner) { - inputBytes += inner.getLength(); - return inner; - } - - @Override - public InputFile newInputFile(String path) { - return measureInputFile(io.newInputFile(path)); - } - - @Override - public InputFile newInputFile(String path, long length) { - return measureInputFile(io.newInputFile(path, length)); - } - - @Override - public InputFile newInputFile(DataFile file) { - return measureInputFile(io.newInputFile(file)); - } - - @Override - public InputFile newInputFile(DeleteFile file) { - return measureInputFile(io.newInputFile(file)); - } - - @Override - public InputFile newInputFile(ManifestFile manifest) { - return measureInputFile(io.newInputFile(manifest)); - } - - @Override - public OutputFile newOutputFile(String path) { - numOutputFiles++; - return io.newOutputFile(path); - } - - @Override - public void deleteFile(String path) { - numDeletedFiles++; - io.deleteFile(path); - } - - @Override - public void deleteFile(InputFile file) { - numDeletedFiles++; - io.deleteFile(file); - } - - @Override - public void deleteFile(OutputFile file) { - numDeletedFiles++; - io.deleteFile(file); - } - - @Override - public Map properties() { - return io.properties(); - } - - @Override - public void initialize(Map properties) { - io.initialize(properties); - } - - @Override - public void close() { - io.close(); - } - } -} diff --git a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/entity/CatalogEntityTest.java b/polaris-service-quarkus/src/test/java/org/apache/polaris/service/entity/CatalogEntityTest.java deleted file mode 100644 index 167a87687..000000000 --- a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/entity/CatalogEntityTest.java +++ /dev/null @@ -1,224 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.entity; - -import java.util.List; -import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; -import org.apache.polaris.core.admin.model.AzureStorageConfigInfo; -import org.apache.polaris.core.admin.model.Catalog; -import org.apache.polaris.core.admin.model.CatalogProperties; -import org.apache.polaris.core.admin.model.GcpStorageConfigInfo; -import org.apache.polaris.core.admin.model.PolarisCatalog; -import org.apache.polaris.core.admin.model.StorageConfigInfo; -import org.apache.polaris.core.entity.CatalogEntity; -import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; - -public class CatalogEntityTest { - - @Test - public void testInvalidAllowedLocationPrefix() { - String storageLocation = "unsupportPrefix://mybucket/path"; - AwsStorageConfigInfo awsStorageConfigModel = - AwsStorageConfigInfo.builder() - .setRoleArn("arn:aws:iam::012345678901:role/jdoe") - .setExternalId("externalId") - .setUserArn("aws::a:user:arn") - .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) - .setAllowedLocations(List.of(storageLocation, "s3://externally-owned-bucket")) - .build(); - CatalogProperties prop = new CatalogProperties(storageLocation); - Catalog awsCatalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName("name") - .setProperties(prop) - .setStorageConfigInfo(awsStorageConfigModel) - .build(); - Assertions.assertThatThrownBy(() -> CatalogEntity.fromCatalog(awsCatalog)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining( - "Location prefix not allowed: 'unsupportPrefix://mybucket/path', expected prefixes"); - - // Invalid azure prefix - AzureStorageConfigInfo azureStorageConfigModel = - AzureStorageConfigInfo.builder() - .setAllowedLocations( - List.of(storageLocation, "abfs://container@storageaccount.blob.windows.net/path")) - .setStorageType(StorageConfigInfo.StorageTypeEnum.AZURE) - .setTenantId("tenantId") - .build(); - Catalog azureCatalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName("name") - .setProperties( - new CatalogProperties("abfs://container@storageaccount.blob.windows.net/path")) - .setStorageConfigInfo(azureStorageConfigModel) - .build(); - Assertions.assertThatThrownBy(() -> CatalogEntity.fromCatalog(azureCatalog)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Invalid azure location uri unsupportPrefix://mybucket/path"); - - // invalid gcp prefix - GcpStorageConfigInfo gcpStorageConfigModel = - GcpStorageConfigInfo.builder() - .setStorageType(StorageConfigInfo.StorageTypeEnum.GCS) - .setAllowedLocations(List.of(storageLocation, "gs://externally-owned-bucket")) - .build(); - Catalog gcpCatalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName("name") - .setProperties(new CatalogProperties("gs://externally-owned-bucket")) - .setStorageConfigInfo(gcpStorageConfigModel) - .build(); - Assertions.assertThatThrownBy(() -> CatalogEntity.fromCatalog(gcpCatalog)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining( - "Location prefix not allowed: 'unsupportPrefix://mybucket/path', expected prefixes"); - } - - @Test - public void testExceedMaxAllowedLocations() { - String storageLocation = "s3://mybucket/path/"; - AwsStorageConfigInfo awsStorageConfigModel = - AwsStorageConfigInfo.builder() - .setRoleArn("arn:aws:iam::012345678901:role/jdoe") - .setExternalId("externalId") - .setUserArn("aws::a:user:arn") - .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) - .setAllowedLocations( - List.of( - storageLocation + "1/", - storageLocation + "2/", - storageLocation + "3/", - storageLocation + "4/", - storageLocation + "5/", - storageLocation + "6/")) - .build(); - CatalogProperties prop = new CatalogProperties(storageLocation); - Catalog awsCatalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName("name") - .setProperties(prop) - .setStorageConfigInfo(awsStorageConfigModel) - .build(); - Assertions.assertThatThrownBy(() -> CatalogEntity.fromCatalog(awsCatalog)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Number of allowed locations exceeds 5"); - } - - @Test - public void testValidAllowedLocationPrefix() { - String basedLocation = "s3://externally-owned-bucket"; - AwsStorageConfigInfo awsStorageConfigModel = - AwsStorageConfigInfo.builder() - .setRoleArn("arn:aws:iam::012345678901:role/jdoe") - .setExternalId("externalId") - .setUserArn("aws::a:user:arn") - .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) - .setAllowedLocations(List.of(basedLocation)) - .build(); - - CatalogProperties prop = new CatalogProperties(basedLocation); - Catalog awsCatalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName("name") - .setProperties(prop) - .setStorageConfigInfo(awsStorageConfigModel) - .build(); - Assertions.assertThatNoException().isThrownBy(() -> CatalogEntity.fromCatalog(awsCatalog)); - - basedLocation = "abfs://container@storageaccount.blob.windows.net/path"; - prop.put(CatalogEntity.DEFAULT_BASE_LOCATION_KEY, basedLocation); - AzureStorageConfigInfo azureStorageConfigModel = - AzureStorageConfigInfo.builder() - .setAllowedLocations(List.of(basedLocation)) - .setStorageType(StorageConfigInfo.StorageTypeEnum.AZURE) - .setTenantId("tenantId") - .build(); - Catalog azureCatalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName("name") - .setProperties(new CatalogProperties(basedLocation)) - .setStorageConfigInfo(azureStorageConfigModel) - .build(); - Assertions.assertThatNoException().isThrownBy(() -> CatalogEntity.fromCatalog(azureCatalog)); - - basedLocation = "gs://externally-owned-bucket"; - prop.put(CatalogEntity.DEFAULT_BASE_LOCATION_KEY, basedLocation); - GcpStorageConfigInfo gcpStorageConfigModel = - GcpStorageConfigInfo.builder() - .setStorageType(StorageConfigInfo.StorageTypeEnum.GCS) - .setAllowedLocations(List.of(basedLocation)) - .build(); - Catalog gcpCatalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName("name") - .setProperties(new CatalogProperties(basedLocation)) - .setStorageConfigInfo(gcpStorageConfigModel) - .build(); - Assertions.assertThatNoException().isThrownBy(() -> CatalogEntity.fromCatalog(gcpCatalog)); - } - - @ParameterizedTest - @ValueSource(strings = {"", "arn:aws:iam::0123456:role/jdoe", "aws-cn", "aws-us-gov"}) - public void testInvalidArn(String roleArn) { - String basedLocation = "s3://externally-owned-bucket"; - AwsStorageConfigInfo awsStorageConfigModel = - AwsStorageConfigInfo.builder() - .setRoleArn(roleArn) - .setExternalId("externalId") - .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) - .setAllowedLocations(List.of(basedLocation)) - .build(); - - CatalogProperties prop = new CatalogProperties(basedLocation); - Catalog awsCatalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName("name") - .setProperties(prop) - .setStorageConfigInfo(awsStorageConfigModel) - .build(); - String expectedMessage = ""; - switch (roleArn) { - case "": - expectedMessage = "ARN cannot be null or empty"; - break; - case "aws-cn": - case "aws-us-gov": - expectedMessage = "AWS China or Gov Cloud are temporarily not supported"; - break; - default: - expectedMessage = "Invalid role ARN format"; - } - ; - Assertions.assertThatThrownBy(() -> CatalogEntity.fromCatalog(awsCatalog)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage(expectedMessage); - } -} diff --git a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/ratelimiter/MockRealmTokenBucketRateLimiter.java b/polaris-service-quarkus/src/test/java/org/apache/polaris/service/ratelimiter/MockRealmTokenBucketRateLimiter.java deleted file mode 100644 index 6e5c5da51..000000000 --- a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/ratelimiter/MockRealmTokenBucketRateLimiter.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.ratelimiter; - -import java.time.Duration; -import java.time.Instant; -import java.time.ZoneOffset; -import org.threeten.extra.MutableClock; - -/** RealmTokenBucketRateLimiter with a mock clock */ -public class MockRealmTokenBucketRateLimiter extends RealmTokenBucketRateLimiter { - public static MutableClock CLOCK = MutableClock.of(Instant.now(), ZoneOffset.UTC); - - public MockRealmTokenBucketRateLimiter(long requestsPerSecond, Duration window) { - super(requestsPerSecond, window, CLOCK); - } -} diff --git a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/ratelimiter/RateLimitResultAsserter.java b/polaris-service-quarkus/src/test/java/org/apache/polaris/service/ratelimiter/RateLimitResultAsserter.java deleted file mode 100644 index a133257b7..000000000 --- a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/ratelimiter/RateLimitResultAsserter.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.ratelimiter; - -import org.junit.jupiter.api.Assertions; - -/** Utility class for testing rate limiters. Lets you easily assert the result of tryAcquire(). */ -public class RateLimitResultAsserter { - private final RateLimiter rateLimiter; - - public RateLimitResultAsserter(RateLimiter rateLimiter) { - this.rateLimiter = rateLimiter; - } - - public void canAcquire(int times) { - for (int i = 0; i < times; i++) { - Assertions.assertTrue(rateLimiter.tryAcquire()); - } - } - - public void cantAcquire() { - for (int i = 0; i < 5; i++) { - Assertions.assertFalse(rateLimiter.tryAcquire()); - } - } -} diff --git a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/ratelimiter/RateLimiterFilterIntegrationTest.java b/polaris-service-quarkus/src/test/java/org/apache/polaris/service/ratelimiter/RateLimiterFilterIntegrationTest.java deleted file mode 100644 index c183acfc2..000000000 --- a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/ratelimiter/RateLimiterFilterIntegrationTest.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.ratelimiter; - -import io.quarkus.test.junit.QuarkusMock; -import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.QuarkusTestProfile; -import io.quarkus.test.junit.TestProfile; -import jakarta.inject.Inject; -import jakarta.ws.rs.core.Response; -import java.time.Duration; -import java.util.Map; -import java.util.function.Consumer; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.service.test.PolarisIntegrationTestHelper; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInfo; -import org.junit.jupiter.api.TestInstance; -import org.threeten.extra.MutableClock; - -/** Main integration tests for rate limiting */ -@QuarkusTest -@TestInstance(TestInstance.Lifecycle.PER_CLASS) -@TestProfile(RateLimiterFilterIntegrationTest.Profile.class) -public class RateLimiterFilterIntegrationTest { - - private static final long REQUESTS_PER_SECOND = 5; - private static final Duration WINDOW = Duration.ofSeconds(10); - - @Inject PolarisIntegrationTestHelper testHelper; - - @BeforeAll - public void setUp(TestInfo testInfo) { - QuarkusMock.installMockForType( - new MockRealmTokenBucketRateLimiter(REQUESTS_PER_SECOND, WINDOW), RateLimiter.class); - testHelper.setUp(testInfo); - } - - @AfterAll - public void tearDown() { - testHelper.tearDown(); - } - - @Test - public void testRateLimiter() { - Consumer requestAsserter = TestUtil.constructRequestAsserter(testHelper); - CallContext.setCurrentContext(CallContext.of(() -> "myrealm", null)); - - MutableClock clock = MockRealmTokenBucketRateLimiter.CLOCK; - clock.add(WINDOW.multipliedBy(2)); // Clear any counters from before this test - - for (int i = 0; i < REQUESTS_PER_SECOND * WINDOW.getSeconds(); i++) { - requestAsserter.accept(Response.Status.OK); - } - requestAsserter.accept(Response.Status.TOO_MANY_REQUESTS); - - clock.add(WINDOW.multipliedBy(4)); // Clear any counters from during this test - } - - public static class Profile implements QuarkusTestProfile { - - @Override - public Map getConfigOverrides() { - return Map.of( - "polaris.rate-limiter.type", "realm-token-bucket", - "polaris.rate-limiter.realm-token-bucket.requests-per-second", - String.valueOf(REQUESTS_PER_SECOND), - "polaris.rate-limiter.realm-token-bucket.window", WINDOW.toString()); - } - } -} diff --git a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/ratelimiter/RealmTokenBucketRateLimiterTest.java b/polaris-service-quarkus/src/test/java/org/apache/polaris/service/ratelimiter/RealmTokenBucketRateLimiterTest.java deleted file mode 100644 index 048f13c12..000000000 --- a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/ratelimiter/RealmTokenBucketRateLimiterTest.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.ratelimiter; - -import java.time.Duration; -import org.apache.polaris.core.context.CallContext; -import org.junit.jupiter.api.Test; -import org.threeten.extra.MutableClock; - -/** Main unit test class for TokenBucketRateLimiter */ -public class RealmTokenBucketRateLimiterTest { - @Test - void testDifferentBucketsDontTouch() { - RateLimiter rateLimiter = new MockRealmTokenBucketRateLimiter(10, Duration.ofSeconds(10)); - RateLimitResultAsserter asserter = new RateLimitResultAsserter(rateLimiter); - MutableClock clock = MockRealmTokenBucketRateLimiter.CLOCK; - - for (int i = 0; i < 202; i++) { - String realm = (i % 2 == 0) ? "realm1" : "realm2"; - CallContext.setCurrentContext(CallContext.of(() -> realm, null)); - - if (i < 200) { - asserter.canAcquire(1); - } else { - asserter.cantAcquire(); - } - } - - clock.add(Duration.ofSeconds(1)); - for (int i = 0; i < 22; i++) { - String realm = (i % 2 == 0) ? "realm1" : "realm2"; - CallContext.setCurrentContext(CallContext.of(() -> realm, null)); - - if (i < 20) { - asserter.canAcquire(1); - } else { - asserter.cantAcquire(); - } - } - } -} diff --git a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/ratelimiter/TestUtil.java b/polaris-service-quarkus/src/test/java/org/apache/polaris/service/ratelimiter/TestUtil.java deleted file mode 100644 index cb4742d64..000000000 --- a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/ratelimiter/TestUtil.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.ratelimiter; - -import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; -import static org.assertj.core.api.Assertions.assertThat; - -import jakarta.ws.rs.core.Response; -import jakarta.ws.rs.core.Response.Status; -import java.util.function.Consumer; -import org.apache.polaris.service.test.PolarisIntegrationTestHelper; - -/** Common test utils for testing rate limiting */ -public class TestUtil { - public static Consumer constructRequestAsserter(PolarisIntegrationTestHelper testHelper) { - return (Response.Status status) -> { - try (Response response = - testHelper - .client - .target( - String.format( - "http://localhost:%d/api/management/v1/principal-roles", - testHelper.localPort)) - .request("application/json") - .header("Authorization", "Bearer " + testHelper.adminToken) - .header(REALM_PROPERTY_KEY, testHelper.realm) - .get()) { - assertThat(response).returns(status.getStatusCode(), Response::getStatus); - } - }; - } -} diff --git a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/ratelimiter/TokenBucketRateLimiterTest.java b/polaris-service-quarkus/src/test/java/org/apache/polaris/service/ratelimiter/TokenBucketRateLimiterTest.java deleted file mode 100644 index 92b874bd1..000000000 --- a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/ratelimiter/TokenBucketRateLimiterTest.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.ratelimiter; - -import java.time.Clock; -import java.time.Duration; -import java.time.Instant; -import java.time.ZoneOffset; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.atomic.AtomicInteger; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.threeten.extra.MutableClock; - -/** Main unit test class for TokenBucketRateLimiter */ -public class TokenBucketRateLimiterTest { - @Test - void testBasic() { - MutableClock clock = MutableClock.of(Instant.now(), ZoneOffset.UTC); - clock.add(Duration.ofSeconds(5)); - - RateLimitResultAsserter asserter = - new RateLimitResultAsserter(new TokenBucketRateLimiter(10, 100, clock)); - - asserter.canAcquire(100); - asserter.cantAcquire(); - - clock.add(Duration.ofSeconds(1)); - asserter.canAcquire(10); - asserter.cantAcquire(); - - clock.add(Duration.ofSeconds(10)); - asserter.canAcquire(100); - asserter.cantAcquire(); - } - - /** - * Starts several threads that try to query the rate limiter at the same time, ensuring that we - * only allow "maxTokens" requests - */ - @Test - void testConcurrent() throws InterruptedException { - int maxTokens = 100; - int numTasks = 5000; // FIXME 50000 yields OOME - int tokensPerSecond = 10; // Can be anything above 0 - - TokenBucketRateLimiter rl = - new TokenBucketRateLimiter( - tokensPerSecond, maxTokens, Clock.fixed(Instant.now(), ZoneOffset.UTC)); - AtomicInteger numAcquired = new AtomicInteger(); - CountDownLatch startLatch = new CountDownLatch(numTasks); - CountDownLatch endLatch = new CountDownLatch(numTasks); - - try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { - for (int i = 0; i < numTasks; i++) { - executor.submit( - () -> { - try { - // Enforce that tasks pause until all tasks are submitted - startLatch.countDown(); - startLatch.await(); - - if (rl.tryAcquire()) { - numAcquired.incrementAndGet(); - } - - endLatch.countDown(); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - }); - } - } - - endLatch.await(); - Assertions.assertEquals(maxTokens, numAcquired.get()); - } -} diff --git a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/task/ManifestFileCleanupTaskHandlerTest.java b/polaris-service-quarkus/src/test/java/org/apache/polaris/service/task/ManifestFileCleanupTaskHandlerTest.java deleted file mode 100644 index 8f654e5c5..000000000 --- a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/task/ManifestFileCleanupTaskHandlerTest.java +++ /dev/null @@ -1,226 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.task; - -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.assertj.core.api.Assertions.assertThatPredicate; - -import io.quarkus.test.junit.QuarkusTest; -import jakarta.inject.Inject; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.Executors; -import java.util.concurrent.atomic.AtomicInteger; -import org.apache.commons.codec.binary.Base64; -import org.apache.iceberg.ManifestFile; -import org.apache.iceberg.ManifestFiles; -import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.inmemory.InMemoryFileIO; -import org.apache.iceberg.io.FileIO; -import org.apache.iceberg.io.OutputFile; -import org.apache.iceberg.io.PositionOutputStream; -import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.core.entity.AsyncTaskType; -import org.apache.polaris.core.entity.TaskEntity; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.junit.jupiter.api.Test; - -@QuarkusTest -class ManifestFileCleanupTaskHandlerTest { - @Inject MetaStoreManagerFactory metaStoreManagerFactory; - - private final RealmContext realmContext = () -> "realmName"; - - @Test - public void testCleanupFileNotExists() throws IOException { - PolarisCallContext polarisCallContext = - new PolarisCallContext( - metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), - new PolarisDefaultDiagServiceImpl()); - try (CallContext callCtx = CallContext.of(realmContext, polarisCallContext)) { - CallContext.setCurrentContext(callCtx); - FileIO fileIO = new InMemoryFileIO(); - TableIdentifier tableIdentifier = - TableIdentifier.of(Namespace.of("db1", "schema1"), "table1"); - ManifestFileCleanupTaskHandler handler = - new ManifestFileCleanupTaskHandler((task) -> fileIO, Executors.newSingleThreadExecutor()); - ManifestFile manifestFile = - TaskTestUtils.manifestFile( - fileIO, "manifest1.avro", 1L, "dataFile1.parquet", "dataFile2.parquet"); - fileIO.deleteFile(manifestFile.path()); - TaskEntity task = - new TaskEntity.Builder() - .withTaskType(AsyncTaskType.FILE_CLEANUP) - .withData( - new ManifestFileCleanupTaskHandler.ManifestCleanupTask( - tableIdentifier, - Base64.encodeBase64String(ManifestFiles.encode(manifestFile)))) - .setName(UUID.randomUUID().toString()) - .build(); - assertThatPredicate(handler::canHandleTask).accepts(task); - assertThatPredicate(handler::handleTask).accepts(task); - } - } - - @Test - public void testCleanupFileManifestExistsDataFilesDontExist() throws IOException { - PolarisCallContext polarisCallContext = - new PolarisCallContext( - metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), - new PolarisDefaultDiagServiceImpl()); - try (CallContext callCtx = CallContext.of(realmContext, polarisCallContext)) { - CallContext.setCurrentContext(callCtx); - FileIO fileIO = new InMemoryFileIO(); - TableIdentifier tableIdentifier = - TableIdentifier.of(Namespace.of("db1", "schema1"), "table1"); - ManifestFileCleanupTaskHandler handler = - new ManifestFileCleanupTaskHandler((task) -> fileIO, Executors.newSingleThreadExecutor()); - ManifestFile manifestFile = - TaskTestUtils.manifestFile( - fileIO, "manifest1.avro", 100L, "dataFile1.parquet", "dataFile2.parquet"); - TaskEntity task = - new TaskEntity.Builder() - .withTaskType(AsyncTaskType.FILE_CLEANUP) - .withData( - new ManifestFileCleanupTaskHandler.ManifestCleanupTask( - tableIdentifier, - Base64.encodeBase64String(ManifestFiles.encode(manifestFile)))) - .setName(UUID.randomUUID().toString()) - .build(); - assertThatPredicate(handler::canHandleTask).accepts(task); - assertThatPredicate(handler::handleTask).accepts(task); - } - } - - @Test - public void testCleanupFiles() throws IOException { - PolarisCallContext polarisCallContext = - new PolarisCallContext( - metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), - new PolarisDefaultDiagServiceImpl()); - try (CallContext callCtx = CallContext.of(realmContext, polarisCallContext)) { - CallContext.setCurrentContext(callCtx); - FileIO fileIO = - new InMemoryFileIO() { - @Override - public void close() { - // no-op - } - }; - TableIdentifier tableIdentifier = - TableIdentifier.of(Namespace.of("db1", "schema1"), "table1"); - ManifestFileCleanupTaskHandler handler = - new ManifestFileCleanupTaskHandler((task) -> fileIO, Executors.newSingleThreadExecutor()); - String dataFile1Path = "dataFile1.parquet"; - OutputFile dataFile1 = fileIO.newOutputFile(dataFile1Path); - PositionOutputStream out1 = dataFile1.createOrOverwrite(); - out1.write("the data".getBytes(UTF_8)); - out1.close(); - String dataFile2Path = "dataFile2.parquet"; - OutputFile dataFile2 = fileIO.newOutputFile(dataFile2Path); - PositionOutputStream out2 = dataFile2.createOrOverwrite(); - out2.write("the data".getBytes(UTF_8)); - out2.close(); - ManifestFile manifestFile = - TaskTestUtils.manifestFile(fileIO, "manifest1.avro", 100L, dataFile1Path, dataFile2Path); - TaskEntity task = - new TaskEntity.Builder() - .withTaskType(AsyncTaskType.FILE_CLEANUP) - .withData( - new ManifestFileCleanupTaskHandler.ManifestCleanupTask( - tableIdentifier, - Base64.encodeBase64String(ManifestFiles.encode(manifestFile)))) - .setName(UUID.randomUUID().toString()) - .build(); - assertThatPredicate(handler::canHandleTask).accepts(task); - assertThatPredicate(handler::handleTask).accepts(task); - assertThatPredicate((String f) -> TaskUtils.exists(f, fileIO)).rejects(dataFile1Path); - assertThatPredicate((String f) -> TaskUtils.exists(f, fileIO)).rejects(dataFile2Path); - } - } - - @Test - public void testCleanupFilesWithRetries() throws IOException { - PolarisCallContext polarisCallContext = - new PolarisCallContext( - metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), - new PolarisDefaultDiagServiceImpl()); - try (CallContext callCtx = CallContext.of(realmContext, polarisCallContext)) { - CallContext.setCurrentContext(callCtx); - Map retryCounter = new HashMap<>(); - FileIO fileIO = - new InMemoryFileIO() { - @Override - public void close() { - // no-op - } - - @Override - public void deleteFile(String location) { - int attempts = - retryCounter - .computeIfAbsent(location, k -> new AtomicInteger(0)) - .incrementAndGet(); - if (attempts < 3) { - throw new RuntimeException("I'm failing to test retries"); - } else { - // succeed on the third attempt - super.deleteFile(location); - } - } - }; - - TableIdentifier tableIdentifier = - TableIdentifier.of(Namespace.of("db1", "schema1"), "table1"); - ManifestFileCleanupTaskHandler handler = - new ManifestFileCleanupTaskHandler((task) -> fileIO, Executors.newSingleThreadExecutor()); - String dataFile1Path = "dataFile1.parquet"; - OutputFile dataFile1 = fileIO.newOutputFile(dataFile1Path); - PositionOutputStream out1 = dataFile1.createOrOverwrite(); - out1.write("the data".getBytes(UTF_8)); - out1.close(); - String dataFile2Path = "dataFile2.parquet"; - OutputFile dataFile2 = fileIO.newOutputFile(dataFile2Path); - PositionOutputStream out2 = dataFile2.createOrOverwrite(); - out2.write("the data".getBytes(UTF_8)); - out2.close(); - ManifestFile manifestFile = - TaskTestUtils.manifestFile(fileIO, "manifest1.avro", 100L, dataFile1Path, dataFile2Path); - TaskEntity task = - new TaskEntity.Builder() - .withTaskType(AsyncTaskType.FILE_CLEANUP) - .withData( - new ManifestFileCleanupTaskHandler.ManifestCleanupTask( - tableIdentifier, - Base64.encodeBase64String(ManifestFiles.encode(manifestFile)))) - .setName(UUID.randomUUID().toString()) - .build(); - assertThatPredicate(handler::canHandleTask).accepts(task); - assertThatPredicate(handler::handleTask).accepts(task); - assertThatPredicate((String f) -> TaskUtils.exists(f, fileIO)).rejects(dataFile1Path); - assertThatPredicate((String f) -> TaskUtils.exists(f, fileIO)).rejects(dataFile2Path); - } - } -} diff --git a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/task/TableCleanupTaskHandlerTest.java b/polaris-service-quarkus/src/test/java/org/apache/polaris/service/task/TableCleanupTaskHandlerTest.java deleted file mode 100644 index cb771b1bf..000000000 --- a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/task/TableCleanupTaskHandlerTest.java +++ /dev/null @@ -1,364 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.task; - -import static org.assertj.core.api.Assertions.assertThat; - -import io.quarkus.test.junit.QuarkusTest; -import jakarta.inject.Inject; -import java.io.IOException; -import java.util.List; -import org.apache.commons.codec.binary.Base64; -import org.apache.iceberg.ManifestFile; -import org.apache.iceberg.ManifestFiles; -import org.apache.iceberg.Snapshot; -import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.inmemory.InMemoryFileIO; -import org.apache.iceberg.io.FileIO; -import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.core.entity.AsyncTaskType; -import org.apache.polaris.core.entity.PolarisBaseEntity; -import org.apache.polaris.core.entity.PolarisEntityType; -import org.apache.polaris.core.entity.TableLikeEntity; -import org.apache.polaris.core.entity.TaskEntity; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; -import org.slf4j.LoggerFactory; - -@QuarkusTest -class TableCleanupTaskHandlerTest { - @Inject MetaStoreManagerFactory metaStoreManagerFactory; - - private final RealmContext realmContext = () -> "realmName"; - - @Test - public void testTableCleanup() throws IOException { - PolarisCallContext polarisCallContext = - new PolarisCallContext( - metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), - new PolarisDefaultDiagServiceImpl()); - try (CallContext callCtx = CallContext.of(realmContext, polarisCallContext)) { - CallContext.setCurrentContext(callCtx); - FileIO fileIO = new InMemoryFileIO(); - TableIdentifier tableIdentifier = - TableIdentifier.of(Namespace.of("db1", "schema1"), "table1"); - TableCleanupTaskHandler handler = - new TableCleanupTaskHandler(Mockito.mock(), metaStoreManagerFactory, (task) -> fileIO); - long snapshotId = 100L; - ManifestFile manifestFile = - TaskTestUtils.manifestFile( - fileIO, "manifest1.avro", snapshotId, "dataFile1.parquet", "dataFile2.parquet"); - TestSnapshot snapshot = - TaskTestUtils.newSnapshot(fileIO, "manifestList.avro", 1, snapshotId, 99L, manifestFile); - String metadataFile = "v1-49494949.metadata.json"; - TaskTestUtils.writeTableMetadata(fileIO, metadataFile, snapshot); - - TaskEntity task = - new TaskEntity.Builder() - .setName("cleanup_" + tableIdentifier.toString()) - .withTaskType(AsyncTaskType.ENTITY_CLEANUP_SCHEDULER) - .withData( - new TableLikeEntity.Builder(tableIdentifier, metadataFile) - .setName("table1") - .setCatalogId(1) - .setCreateTimestamp(100) - .build()) - .build(); - Assertions.assertThatPredicate(handler::canHandleTask).accepts(task); - - CallContext.setCurrentContext(CallContext.of(realmContext, polarisCallContext)); - handler.handleTask(task); - - assertThat( - metaStoreManagerFactory - .getOrCreateMetaStoreManager(realmContext) - .loadTasks(polarisCallContext, "test", 1) - .getEntities()) - .hasSize(1) - .satisfiesExactly( - taskEntity -> - assertThat(taskEntity) - .returns(PolarisEntityType.TASK.getCode(), PolarisBaseEntity::getTypeCode) - .extracting(entity -> TaskEntity.of(entity)) - .returns(AsyncTaskType.FILE_CLEANUP, TaskEntity::getTaskType) - .returns( - new ManifestFileCleanupTaskHandler.ManifestCleanupTask( - tableIdentifier, - Base64.encodeBase64String(ManifestFiles.encode(manifestFile))), - entity -> - entity.readData( - ManifestFileCleanupTaskHandler.ManifestCleanupTask.class))); - } - } - - @Test - public void testTableCleanupHandlesAlreadyDeletedMetadata() throws IOException { - PolarisCallContext polarisCallContext = - new PolarisCallContext( - metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), - new PolarisDefaultDiagServiceImpl()); - try (CallContext callCtx = CallContext.of(realmContext, polarisCallContext)) { - CallContext.setCurrentContext(callCtx); - FileIO fileIO = - new InMemoryFileIO() { - @Override - public void close() { - // no-op - } - }; - TableIdentifier tableIdentifier = - TableIdentifier.of(Namespace.of("db1", "schema1"), "table1"); - TableCleanupTaskHandler handler = - new TableCleanupTaskHandler(Mockito.mock(), metaStoreManagerFactory, (task) -> fileIO); - long snapshotId = 100L; - ManifestFile manifestFile = - TaskTestUtils.manifestFile( - fileIO, "manifest1.avro", snapshotId, "dataFile1.parquet", "dataFile2.parquet"); - TestSnapshot snapshot = - TaskTestUtils.newSnapshot(fileIO, "manifestList.avro", 1, snapshotId, 99L, manifestFile); - String metadataFile = "v1-49494949.metadata.json"; - TaskTestUtils.writeTableMetadata(fileIO, metadataFile, snapshot); - - TableLikeEntity tableLikeEntity = - new TableLikeEntity.Builder(tableIdentifier, metadataFile) - .setName("table1") - .setCatalogId(1) - .setCreateTimestamp(100) - .build(); - TaskEntity task = - new TaskEntity.Builder() - .setName("cleanup_" + tableIdentifier.toString()) - .withTaskType(AsyncTaskType.ENTITY_CLEANUP_SCHEDULER) - .withData(tableLikeEntity) - .build(); - Assertions.assertThatPredicate(handler::canHandleTask).accepts(task); - - CallContext.setCurrentContext(CallContext.of(realmContext, polarisCallContext)); - - // handle the same task twice - // the first one should successfully delete the metadata - List results = List.of(handler.handleTask(task), handler.handleTask(task)); - assertThat(results).containsExactly(true, true); - - // both tasks successfully executed, but only one should queue subtasks - assertThat( - metaStoreManagerFactory - .getOrCreateMetaStoreManager(realmContext) - .loadTasks(polarisCallContext, "test", 5) - .getEntities()) - .hasSize(1); - } - } - - @Test - public void testTableCleanupDuplicatesTasksIfFileStillExists() throws IOException { - PolarisCallContext polarisCallContext = - new PolarisCallContext( - metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), - new PolarisDefaultDiagServiceImpl()); - try (CallContext callCtx = CallContext.of(realmContext, polarisCallContext)) { - CallContext.setCurrentContext(callCtx); - FileIO fileIO = - new InMemoryFileIO() { - @Override - public void deleteFile(String location) { - LoggerFactory.getLogger(TableCleanupTaskHandler.class) - .info( - "Not deleting file at location {} to simulate concurrent tasks runs", - location); - // don't do anything - } - - @Override - public void close() { - // no-op - } - }; - TableIdentifier tableIdentifier = - TableIdentifier.of(Namespace.of("db1", "schema1"), "table1"); - TableCleanupTaskHandler handler = - new TableCleanupTaskHandler(Mockito.mock(), metaStoreManagerFactory, (task) -> fileIO); - long snapshotId = 100L; - ManifestFile manifestFile = - TaskTestUtils.manifestFile( - fileIO, "manifest1.avro", snapshotId, "dataFile1.parquet", "dataFile2.parquet"); - TestSnapshot snapshot = - TaskTestUtils.newSnapshot(fileIO, "manifestList.avro", 1, snapshotId, 99L, manifestFile); - String metadataFile = "v1-49494949.metadata.json"; - TaskTestUtils.writeTableMetadata(fileIO, metadataFile, snapshot); - - TaskEntity task = - new TaskEntity.Builder() - .setName("cleanup_" + tableIdentifier.toString()) - .withTaskType(AsyncTaskType.ENTITY_CLEANUP_SCHEDULER) - .withData( - new TableLikeEntity.Builder(tableIdentifier, metadataFile) - .setName("table1") - .setCatalogId(1) - .setCreateTimestamp(100) - .build()) - .build(); - Assertions.assertThatPredicate(handler::canHandleTask).accepts(task); - - CallContext.setCurrentContext(CallContext.of(realmContext, polarisCallContext)); - - // handle the same task twice - // the first one should successfully delete the metadata - List results = List.of(handler.handleTask(task), handler.handleTask(task)); - assertThat(results).containsExactly(true, true); - - // both tasks successfully executed, but only one should queue subtasks - assertThat( - metaStoreManagerFactory - .getOrCreateMetaStoreManager(realmContext) - .loadTasks(polarisCallContext, "test", 5) - .getEntities()) - .hasSize(2) - .satisfiesExactly( - taskEntity -> - assertThat(taskEntity) - .returns(PolarisEntityType.TASK.getCode(), PolarisBaseEntity::getTypeCode) - .extracting(entity -> TaskEntity.of(entity)) - .returns(AsyncTaskType.FILE_CLEANUP, TaskEntity::getTaskType) - .returns( - new ManifestFileCleanupTaskHandler.ManifestCleanupTask( - tableIdentifier, - Base64.encodeBase64String(ManifestFiles.encode(manifestFile))), - entity -> - entity.readData( - ManifestFileCleanupTaskHandler.ManifestCleanupTask.class)), - taskEntity -> - assertThat(taskEntity) - .returns(PolarisEntityType.TASK.getCode(), PolarisBaseEntity::getTypeCode) - .extracting(entity -> TaskEntity.of(entity)) - .returns(AsyncTaskType.FILE_CLEANUP, TaskEntity::getTaskType) - .returns( - new ManifestFileCleanupTaskHandler.ManifestCleanupTask( - tableIdentifier, - Base64.encodeBase64String(ManifestFiles.encode(manifestFile))), - entity -> - entity.readData( - ManifestFileCleanupTaskHandler.ManifestCleanupTask.class))); - } - } - - @Test - public void testTableCleanupMultipleSnapshots() throws IOException { - PolarisCallContext polarisCallContext = - new PolarisCallContext( - metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), - new PolarisDefaultDiagServiceImpl()); - try (CallContext callCtx = CallContext.of(realmContext, polarisCallContext)) { - CallContext.setCurrentContext(callCtx); - FileIO fileIO = new InMemoryFileIO(); - TableIdentifier tableIdentifier = - TableIdentifier.of(Namespace.of("db1", "schema1"), "table1"); - TableCleanupTaskHandler handler = - new TableCleanupTaskHandler(Mockito.mock(), metaStoreManagerFactory, (task) -> fileIO); - long snapshotId1 = 100L; - ManifestFile manifestFile1 = - TaskTestUtils.manifestFile( - fileIO, "manifest1.avro", snapshotId1, "dataFile1.parquet", "dataFile2.parquet"); - ManifestFile manifestFile2 = - TaskTestUtils.manifestFile( - fileIO, "manifest2.avro", snapshotId1, "dataFile3.parquet", "dataFile4.parquet"); - Snapshot snapshot = - TaskTestUtils.newSnapshot( - fileIO, "manifestList.avro", 1, snapshotId1, 99L, manifestFile1, manifestFile2); - ManifestFile manifestFile3 = - TaskTestUtils.manifestFile( - fileIO, "manifest3.avro", snapshot.snapshotId() + 1, "dataFile5.parquet"); - Snapshot snapshot2 = - TaskTestUtils.newSnapshot( - fileIO, - "manifestList2.avro", - snapshot.sequenceNumber() + 1, - snapshot.snapshotId() + 1, - snapshot.snapshotId(), - manifestFile1, - manifestFile3); // exclude manifest2 from the new snapshot - String metadataFile = "v1-295495059.metadata.json"; - TaskTestUtils.writeTableMetadata(fileIO, metadataFile, snapshot, snapshot2); - - TaskEntity task = - new TaskEntity.Builder() - .withTaskType(AsyncTaskType.ENTITY_CLEANUP_SCHEDULER) - .withData( - new TableLikeEntity.Builder(tableIdentifier, metadataFile) - .setName("table1") - .setCatalogId(1) - .setCreateTimestamp(100) - .build()) - .build(); - Assertions.assertThatPredicate(handler::canHandleTask).accepts(task); - - CallContext.setCurrentContext(CallContext.of(realmContext, polarisCallContext)); - handler.handleTask(task); - - assertThat( - metaStoreManagerFactory - .getOrCreateMetaStoreManager(realmContext) - .loadTasks(polarisCallContext, "test", 5) - .getEntities()) - // all three manifests should be present, even though one is excluded from the latest - // snapshot - .hasSize(3) - .satisfiesExactlyInAnyOrder( - taskEntity -> - assertThat(taskEntity) - .returns(PolarisEntityType.TASK.getCode(), PolarisBaseEntity::getTypeCode) - .extracting(entity -> TaskEntity.of(entity)) - .returns( - new ManifestFileCleanupTaskHandler.ManifestCleanupTask( - tableIdentifier, - Base64.encodeBase64String(ManifestFiles.encode(manifestFile1))), - entity -> - entity.readData( - ManifestFileCleanupTaskHandler.ManifestCleanupTask.class)), - taskEntity -> - assertThat(taskEntity) - .returns(PolarisEntityType.TASK.getCode(), PolarisBaseEntity::getTypeCode) - .extracting(entity -> TaskEntity.of(entity)) - .returns( - new ManifestFileCleanupTaskHandler.ManifestCleanupTask( - tableIdentifier, - Base64.encodeBase64String(ManifestFiles.encode(manifestFile2))), - entity -> - entity.readData( - ManifestFileCleanupTaskHandler.ManifestCleanupTask.class)), - taskEntity -> - assertThat(taskEntity) - .returns(PolarisEntityType.TASK.getCode(), PolarisBaseEntity::getTypeCode) - .extracting(entity -> TaskEntity.of(entity)) - .returns( - new ManifestFileCleanupTaskHandler.ManifestCleanupTask( - tableIdentifier, - Base64.encodeBase64String(ManifestFiles.encode(manifestFile3))), - entity -> - entity.readData( - ManifestFileCleanupTaskHandler.ManifestCleanupTask.class))); - } - } -} diff --git a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/task/TaskTestUtils.java b/polaris-service-quarkus/src/test/java/org/apache/polaris/service/task/TaskTestUtils.java deleted file mode 100644 index 818f87654..000000000 --- a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/task/TaskTestUtils.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.task; - -import jakarta.annotation.Nonnull; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.List; -import java.util.UUID; -import org.apache.iceberg.DataFile; -import org.apache.iceberg.DataFiles; -import org.apache.iceberg.FileFormat; -import org.apache.iceberg.ManifestFile; -import org.apache.iceberg.ManifestFiles; -import org.apache.iceberg.ManifestWriter; -import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.Schema; -import org.apache.iceberg.Snapshot; -import org.apache.iceberg.SortOrder; -import org.apache.iceberg.TableMetadata; -import org.apache.iceberg.TableMetadataParser; -import org.apache.iceberg.avro.Avro; -import org.apache.iceberg.io.FileAppender; -import org.apache.iceberg.io.FileIO; -import org.apache.iceberg.io.PositionOutputStream; -import org.apache.iceberg.types.Types; - -public class TaskTestUtils { - static ManifestFile manifestFile( - FileIO fileIO, String manifestFilePath, long snapshotId, String... dataFiles) - throws IOException { - ManifestWriter writer = - ManifestFiles.write( - 2, PartitionSpec.unpartitioned(), fileIO.newOutputFile(manifestFilePath), snapshotId); - for (String dataFile : dataFiles) { - writer.add( - new DataFiles.Builder(PartitionSpec.unpartitioned()) - .withFileSizeInBytes(100L) - .withFormat(FileFormat.PARQUET) - .withPath(dataFile) - .withRecordCount(10) - .build()); - } - writer.close(); - return writer.toManifestFile(); - } - - static void writeTableMetadata(FileIO fileIO, String metadataFile, Snapshot... snapshots) - throws IOException { - TableMetadata.Builder tmBuidler = - TableMetadata.buildFromEmpty() - .setLocation("path/to/table") - .addSchema( - new Schema( - List.of(Types.NestedField.of(1, false, "field1", Types.StringType.get()))), - 1) - .addSortOrder(SortOrder.unsorted()) - .assignUUID(UUID.randomUUID().toString()) - .addPartitionSpec(PartitionSpec.unpartitioned()); - for (Snapshot snapshot : snapshots) { - tmBuidler.addSnapshot(snapshot); - } - TableMetadata tableMetadata = tmBuidler.build(); - PositionOutputStream out = fileIO.newOutputFile(metadataFile).createOrOverwrite(); - out.write(TableMetadataParser.toJson(tableMetadata).getBytes(StandardCharsets.UTF_8)); - out.close(); - } - - static @Nonnull TestSnapshot newSnapshot( - FileIO fileIO, - String manifestListLocation, - long sequenceNumber, - long snapshotId, - long parentSnapshot, - ManifestFile... manifestFiles) - throws IOException { - FileAppender manifestListWriter = - Avro.write(fileIO.newOutputFile(manifestListLocation)) - .schema(ManifestFile.schema()) - .named("manifest_file") - .overwrite() - .build(); - manifestListWriter.addAll(Arrays.asList(manifestFiles)); - manifestListWriter.close(); - TestSnapshot snapshot = - new TestSnapshot(sequenceNumber, snapshotId, parentSnapshot, 1L, manifestListLocation); - return snapshot; - } -} diff --git a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/task/TestSnapshot.java b/polaris-service-quarkus/src/test/java/org/apache/polaris/service/task/TestSnapshot.java deleted file mode 100644 index 9ecf310c5..000000000 --- a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/task/TestSnapshot.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.task; - -import com.google.common.collect.Lists; -import java.io.IOException; -import java.util.List; -import java.util.Map; -import org.apache.iceberg.DataFile; -import org.apache.iceberg.GenericManifestFile; -import org.apache.iceberg.GenericPartitionFieldSummary; -import org.apache.iceberg.ManifestContent; -import org.apache.iceberg.ManifestFile; -import org.apache.iceberg.Snapshot; -import org.apache.iceberg.avro.Avro; -import org.apache.iceberg.exceptions.RuntimeIOException; -import org.apache.iceberg.io.CloseableIterable; -import org.apache.iceberg.io.FileIO; - -final class TestSnapshot implements Snapshot { - private final long sequenceNumber; - private final long snapshotId; - private final long parentSnapshot; - private final long timestampMillis; - private final String manifestListLocation; - - public TestSnapshot( - long sequenceNumber, - long snapshotId, - long parentSnapshot, - long timestampMillis, - String manifestListLocation) { - this.sequenceNumber = sequenceNumber; - this.snapshotId = snapshotId; - this.parentSnapshot = parentSnapshot; - this.timestampMillis = timestampMillis; - this.manifestListLocation = manifestListLocation; - } - - @Override - public long sequenceNumber() { - return sequenceNumber; - } - - @Override - public long snapshotId() { - return snapshotId; - } - - @Override - public Long parentId() { - return parentSnapshot; - } - - @Override - public long timestampMillis() { - return timestampMillis; - } - - @Override - public List allManifests(FileIO io) { - try (CloseableIterable files = - Avro.read(io.newInputFile(manifestListLocation)) - .rename("manifest_file", GenericManifestFile.class.getName()) - .rename("partitions", GenericPartitionFieldSummary.class.getName()) - .rename("r508", GenericPartitionFieldSummary.class.getName()) - .classLoader(GenericManifestFile.class.getClassLoader()) - .project(ManifestFile.schema()) - .reuseContainers(false) - .build()) { - - return Lists.newLinkedList(files); - - } catch (IOException e) { - throw new RuntimeIOException(e, "Cannot read manifest list file: %s", manifestListLocation); - } - } - - @Override - public List dataManifests(FileIO io) { - return allManifests(io).stream() - .filter(mf -> mf.content().equals(ManifestContent.DATA)) - .toList(); - } - - @Override - public List deleteManifests(FileIO io) { - return allManifests(io).stream() - .filter(mf -> mf.content().equals(ManifestContent.DELETES)) - .toList(); - } - - @Override - public String operation() { - return "op"; - } - - @Override - public Map summary() { - return Map.of(); - } - - @Override - public Iterable addedDataFiles(FileIO io) { - return null; - } - - @Override - public Iterable removedDataFiles(FileIO io) { - return null; - } - - @Override - public String manifestListLocation() { - return manifestListLocation; - } -} diff --git a/polaris-service-quarkus/README.md b/polaris-service/README-quarkus.md similarity index 91% rename from polaris-service-quarkus/README.md rename to polaris-service/README-quarkus.md index 34ef23081..bdc76ca9b 100644 --- a/polaris-service-quarkus/README.md +++ b/polaris-service/README-quarkus.md @@ -80,11 +80,11 @@ You can find more details here: https://quarkus.io/guides/config # TODO * Modify `CallContext` and remove all usages of ThreadLocal, replace with proper context propagation. -* Complete utests/itests in `polaris-service-quarkus` -* Remove dropwizard references (in `polaris-core` and `polaris-service-quarkus`) -* Remove `@TimedApi` from `polaris-core` (`org.apache.polaris.core.resource.TimedApi`) -* Remove `polaris-service` and rename `polaris-service-quarkus` as `polaris-service` -* Create `polaris-cli` module +* Complete utests/itests in `polaris-service` +* Use `@QuarkustIntegrationTest` for integration tests (require root credential ID via env var) +* Re-introduce `TestEnvironmentExtension` +* Adapt `@TimedApi` from `polaris-core` (`org.apache.polaris.core.resource.TimedApi`) and fix tests +* Create `polaris-cli` module, add Bootstrap and Purge commands * Update documentation/README/... * Do we want to support existing json configuration file as configuration source ? diff --git a/polaris-service/build.gradle.kts b/polaris-service/build.gradle.kts index bc7a4a4f5..64e28d424 100644 --- a/polaris-service/build.gradle.kts +++ b/polaris-service/build.gradle.kts @@ -17,14 +17,12 @@ * under the License. */ -import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar import org.openapitools.generator.gradle.plugin.tasks.GenerateTask plugins { + alias(libs.plugins.quarkus) alias(libs.plugins.openapi.generator) id("polaris-server") - id("polaris-license-report") - id("polaris-shadow-jar") id("application") } @@ -35,55 +33,40 @@ dependencies { implementation("org.apache.iceberg:iceberg-api") implementation("org.apache.iceberg:iceberg-core") implementation("org.apache.iceberg:iceberg-aws") - implementation(libs.hadoop.common) { - exclude("org.slf4j", "slf4j-reload4j") - exclude("org.slf4j", "slf4j-log4j12") - exclude("ch.qos.reload4j", "reload4j") - exclude("log4j", "log4j") - exclude("org.apache.zookeeper", "zookeeper") - } - implementation(libs.hadoop.hdfs.client) - implementation(platform(libs.dropwizard.bom)) - implementation("io.dropwizard:dropwizard-core") - implementation("io.dropwizard:dropwizard-auth") - implementation("io.dropwizard:dropwizard-json-logging") + implementation(platform(libs.quarkus.bom)) + implementation("io.quarkus:quarkus-logging-json") + implementation("io.quarkus:quarkus-rest") + implementation("io.quarkus:quarkus-rest-jackson") + implementation("io.quarkus:quarkus-hibernate-validator") + implementation("io.quarkus:quarkus-smallrye-health") + implementation("io.quarkus:quarkus-micrometer") + implementation("io.quarkus:quarkus-opentelemetry") + implementation("io.quarkus:quarkus-container-image-docker") + + implementation("org.apache.commons:commons-lang3:3.17.0") + + compileOnly(libs.jakarta.enterprise.cdi.api) + compileOnly(libs.jakarta.inject.api) + compileOnly(libs.jakarta.validation.api) + compileOnly(libs.jakarta.ws.rs.api) implementation(platform(libs.jackson.bom)) - implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-xml") implementation("com.fasterxml.jackson.core:jackson-annotations") - - implementation(platform(libs.opentelemetry.bom)) - implementation("io.opentelemetry:opentelemetry-api") - implementation("io.opentelemetry:opentelemetry-sdk-trace") - implementation("io.opentelemetry:opentelemetry-exporter-logging") - implementation(libs.opentelemetry.semconv) + implementation("com.fasterxml.jackson.core:jackson-core") + implementation("com.fasterxml.jackson.core:jackson-databind") implementation(libs.caffeine) implementation(libs.guava) implementation(libs.slf4j.api) - implementation(libs.prometheus.metrics.exporter.servlet.jakarta) - implementation(platform(libs.micrometer.bom)) - implementation("io.micrometer:micrometer-core") - implementation("io.micrometer:micrometer-registry-prometheus") - - compileOnly(libs.swagger.annotations) - compileOnly(libs.jetbrains.annotations) - compileOnly(libs.spotbugs.annotations) - implementation(libs.swagger.jaxrs) - implementation(libs.javax.annotation.api) - implementation(libs.hadoop.client.api) + implementation(libs.hadoop.client.runtime) implementation(libs.auth0.jwt) - implementation(libs.logback.core) implementation(libs.bouncycastle.bcprov) - compileOnly(libs.jetbrains.annotations) - compileOnly(libs.spotbugs.annotations) - implementation(platform(libs.google.cloud.storage.bom)) implementation("com.google.cloud:google-cloud-storage") implementation(platform(libs.awssdk.bom)) @@ -93,12 +76,14 @@ dependencies { implementation(platform(libs.azuresdk.bom)) implementation("com.azure:azure-core") + implementation("io.quarkus:quarkus-micrometer-registry-prometheus") + + compileOnly(libs.swagger.annotations) + + implementation(libs.jakarta.servlet.api) + testImplementation("org.apache.iceberg:iceberg-api:${libs.versions.iceberg.get()}:tests") testImplementation("org.apache.iceberg:iceberg-core:${libs.versions.iceberg.get()}:tests") - testImplementation("io.dropwizard:dropwizard-testing") - testImplementation(platform(libs.testcontainers.bom)) - testImplementation("org.testcontainers:testcontainers") - testImplementation(libs.s3mock.testcontainers) testImplementation("org.apache.iceberg:iceberg-spark-3.5_2.12") testImplementation("org.apache.iceberg:iceberg-spark-extensions-3.5_2.12") @@ -107,6 +92,7 @@ dependencies { exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") exclude("org.apache.logging.log4j", "log4j-api") exclude("org.apache.logging.log4j", "log4j-1.2-api") + exclude("org.slf4j", "jul-to-slf4j") } testImplementation("software.amazon.awssdk:glue") @@ -114,16 +100,28 @@ dependencies { testImplementation("software.amazon.awssdk:dynamodb") testImplementation(platform(libs.junit.bom)) - testImplementation("org.junit.jupiter:junit-jupiter") + testImplementation(libs.bundles.junit.testing) testImplementation(libs.assertj.core) testImplementation(libs.mockito.core) - testRuntimeOnly("org.junit.platform:junit-platform-launcher") - testRuntimeOnly(project(":polaris-eclipselink")) -} + testImplementation(platform(libs.quarkus.bom)) + testImplementation("io.quarkus:quarkus-junit5") + testImplementation("io.quarkus:quarkus-junit5-mockito") + testImplementation("io.quarkus:quarkus-rest-client") + testImplementation("io.quarkus:quarkus-rest-client-jackson") + testImplementation("io.rest-assured:rest-assured") + + testImplementation(platform(libs.testcontainers.bom)) + testImplementation("org.testcontainers:testcontainers") + testImplementation(libs.s3mock.testcontainers) -if (project.properties.get("eclipseLink") == "true") { - dependencies { implementation(project(":polaris-eclipselink")) } + // required for PolarisSparkIntegrationTest + testImplementation(enforcedPlatform("org.scala-lang:scala-library:2.12.18")) + testImplementation(enforcedPlatform("org.scala-lang:scala-reflect:2.12.18")) + testImplementation(libs.javax.servlet.api) + testImplementation( + enforcedPlatform("org.antlr:antlr4-runtime:4.9.3") + ) // cannot be higher than 4.9.3 } openApiGenerate { @@ -218,6 +216,11 @@ sourceSets { main { java { srcDir(project.layout.buildDirectory.dir("generated/src/main/java")) } } } +tasks.withType(Test::class.java).configureEach { + systemProperty("java.util.logging.manager", "org.jboss.logmanager.LogManager") + addSparkJvmOptions() +} + tasks.named("test").configure { if (System.getenv("AWS_REGION") == null) { environment("AWS_REGION", "us-west-2") @@ -227,46 +230,31 @@ tasks.named("test").configure { maxParallelForks = 4 } -tasks.register("runApp").configure { - if (System.getenv("AWS_REGION") == null) { - environment("AWS_REGION", "us-west-2") - } - classpath = sourceSets["main"].runtimeClasspath - mainClass = "org.apache.polaris.service.PolarisApplication" - args("server", "$rootDir/polaris-server.yml") -} - -application { mainClass = "org.apache.polaris.service.PolarisApplication" } - -tasks.named("jar") { - manifest { attributes["Main-Class"] = "org.apache.polaris.service.PolarisApplication" } -} - -tasks.register("testJar") { - archiveClassifier.set("tests") - from(sourceSets.test.get().output) -} - -val shadowJar = - tasks.named("shadowJar") { - manifest { attributes["Main-Class"] = "org.apache.polaris.service.PolarisApplication" } - mergeServiceFiles() - isZip64 = true - finalizedBy("startScripts") - } - -val startScripts = - tasks.named("startScripts") { - classpath = files(provider { shadowJar.get().archiveFileName }) - } - -tasks.register("prepareDockerDist") { - into(project.layout.buildDirectory.dir("docker-dist")) - from(startScripts) { into("bin") } - from(shadowJar) { into("lib") } - doFirst { delete(project.layout.buildDirectory.dir("regtest-dist")) } +/** + * Adds the JPMS options required for Spark to run on Java 17, taken from the + * `DEFAULT_MODULE_OPTIONS` constant in `org.apache.spark.launcher.JavaModuleOptions`. + */ +fun JavaForkOptions.addSparkJvmOptions() { + jvmArgs = + (jvmArgs ?: emptyList()) + + listOf( + // Spark 3.3+ + "-XX:+IgnoreUnrecognizedVMOptions", + "--add-opens=java.base/java.lang=ALL-UNNAMED", + "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED", + "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED", + "--add-opens=java.base/java.io=ALL-UNNAMED", + "--add-opens=java.base/java.net=ALL-UNNAMED", + "--add-opens=java.base/java.nio=ALL-UNNAMED", + "--add-opens=java.base/java.util=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED", + "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", + "--add-opens=java.base/sun.nio.cs=ALL-UNNAMED", + "--add-opens=java.base/sun.security.action=ALL-UNNAMED", + "--add-opens=java.base/sun.util.calendar=ALL-UNNAMED", + "--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED", + // Spark 3.4+ + "-Djdk.reflect.useDirectMethodHandle=false" + ) } - -tasks.named("build").configure { dependsOn("prepareDockerDist") } - -tasks.named("assemble").configure { dependsOn("testJar") } diff --git a/polaris-service-quarkus/src/main/docker/Dockerfile.jvm b/polaris-service/src/main/docker/Dockerfile.jvm similarity index 100% rename from polaris-service-quarkus/src/main/docker/Dockerfile.jvm rename to polaris-service/src/main/docker/Dockerfile.jvm diff --git a/polaris-service/src/main/java/org/apache/polaris/service/BootstrapRealmsCommand.java b/polaris-service/src/main/java/org/apache/polaris/service/BootstrapRealmsCommand.java deleted file mode 100644 index 3f282b042..000000000 --- a/polaris-service/src/main/java/org/apache/polaris/service/BootstrapRealmsCommand.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service; - -import io.dropwizard.core.cli.ConfiguredCommand; -import io.dropwizard.core.setup.Bootstrap; -import java.util.Map; -import net.sourceforge.argparse4j.inf.Namespace; -import org.apache.polaris.core.PolarisConfigurationStore; -import org.apache.polaris.core.auth.PolarisSecretsManager.PrincipalSecretsResult; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.service.config.ConfigurationStoreAware; -import org.apache.polaris.service.config.PolarisApplicationConfig; -import org.apache.polaris.service.context.CallContextResolver; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Command for bootstrapping root level service principals for each realm. This command will invoke - * a default implementation which generates random user id and secret. These credentials will be - * printed out to the log and standard output (stdout). - */ -public class BootstrapRealmsCommand extends ConfiguredCommand { - private static final Logger LOGGER = LoggerFactory.getLogger(BootstrapRealmsCommand.class); - - public BootstrapRealmsCommand() { - super("bootstrap", "bootstraps principal credentials for all realms and prints them to log"); - } - - @Override - protected void run( - Bootstrap bootstrap, - Namespace namespace, - PolarisApplicationConfig configuration) { - MetaStoreManagerFactory metaStoreManagerFactory = configuration.getMetaStoreManagerFactory(); - - PolarisConfigurationStore configurationStore = configuration.getConfigurationStore(); - if (metaStoreManagerFactory instanceof ConfigurationStoreAware) { - ((ConfigurationStoreAware) metaStoreManagerFactory).setConfigurationStore(configurationStore); - } - CallContextResolver callContextResolver = configuration.getCallContextResolver(); - callContextResolver.setMetaStoreManagerFactory(metaStoreManagerFactory); - if (callContextResolver instanceof ConfigurationStoreAware csa) { - csa.setConfigurationStore(configurationStore); - } - - // Execute the bootstrap - Map results = - metaStoreManagerFactory.bootstrapRealms(configuration.getDefaultRealms()); - - // Log any errors: - boolean success = true; - for (Map.Entry result : results.entrySet()) { - if (!result.getValue().isSuccess()) { - LOGGER.error( - "Bootstrapping `{}` failed: {}", - result.getKey(), - result.getValue().getReturnStatus().toString()); - success = false; - } - } - - if (success) { - LOGGER.info("Bootstrap completed successfully."); - } else { - LOGGER.error("Bootstrap encountered errors during operation."); - } - } -} diff --git a/polaris-service/src/main/java/org/apache/polaris/service/PolarisApplication.java b/polaris-service/src/main/java/org/apache/polaris/service/PolarisApplication.java deleted file mode 100644 index 908496da0..000000000 --- a/polaris-service/src/main/java/org/apache/polaris/service/PolarisApplication.java +++ /dev/null @@ -1,415 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service; - -import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.Objects.requireNonNull; -import static org.apache.polaris.service.config.PolarisApplicationConfig.REQUEST_BODY_BYTES_NO_LIMIT; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.PropertyAccessor; -import com.fasterxml.jackson.core.StreamReadConstraints; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.PropertyNamingStrategies; -import io.dropwizard.auth.AuthDynamicFeature; -import io.dropwizard.auth.AuthFilter; -import io.dropwizard.auth.oauth.OAuthCredentialAuthFilter; -import io.dropwizard.configuration.EnvironmentVariableSubstitutor; -import io.dropwizard.configuration.SubstitutingSourceProvider; -import io.dropwizard.core.Application; -import io.dropwizard.core.setup.Bootstrap; -import io.dropwizard.core.setup.Environment; -import io.micrometer.prometheusmetrics.PrometheusConfig; -import io.micrometer.prometheusmetrics.PrometheusMeterRegistry; -import io.opentelemetry.api.OpenTelemetry; -import io.opentelemetry.api.baggage.propagation.W3CBaggagePropagator; -import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; -import io.opentelemetry.context.propagation.ContextPropagators; -import io.opentelemetry.context.propagation.TextMapPropagator; -import io.opentelemetry.exporter.logging.LoggingSpanExporter; -import io.opentelemetry.sdk.OpenTelemetrySdk; -import io.opentelemetry.sdk.resources.Resource; -import io.opentelemetry.sdk.trace.SdkTracerProvider; -import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; -import io.opentelemetry.semconv.ServiceAttributes; -import io.prometheus.metrics.exporter.servlet.jakarta.PrometheusMetricsServlet; -import jakarta.servlet.DispatcherType; -import jakarta.servlet.Filter; -import jakarta.servlet.FilterChain; -import jakarta.servlet.FilterRegistration; -import jakarta.servlet.ServletException; -import jakarta.servlet.ServletRequest; -import jakarta.servlet.ServletResponse; -import jakarta.servlet.http.HttpServletRequest; -import java.io.Closeable; -import java.io.IOException; -import java.io.InputStream; -import java.net.URL; -import java.util.Collections; -import java.util.EnumSet; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.Executors; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.apache.iceberg.rest.RESTSerializers; -import org.apache.polaris.core.PolarisConfigurationStore; -import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; -import org.apache.polaris.core.auth.PolarisAuthorizer; -import org.apache.polaris.core.auth.PolarisAuthorizerImpl; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.core.monitor.MetricRegistryAware; -import org.apache.polaris.core.monitor.PolarisMetricRegistry; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.service.admin.PolarisServiceImpl; -import org.apache.polaris.service.admin.api.PolarisCatalogsApi; -import org.apache.polaris.service.admin.api.PolarisPrincipalRolesApi; -import org.apache.polaris.service.admin.api.PolarisPrincipalsApi; -import org.apache.polaris.service.auth.DiscoverableAuthenticator; -import org.apache.polaris.service.catalog.IcebergCatalogAdapter; -import org.apache.polaris.service.catalog.api.IcebergRestCatalogApi; -import org.apache.polaris.service.catalog.api.IcebergRestConfigurationApi; -import org.apache.polaris.service.catalog.api.IcebergRestOAuth2Api; -import org.apache.polaris.service.catalog.io.FileIOFactory; -import org.apache.polaris.service.config.ConfigurationStoreAware; -import org.apache.polaris.service.config.HasMetaStoreManagerFactory; -import org.apache.polaris.service.config.OAuth2ApiService; -import org.apache.polaris.service.config.PolarisApplicationConfig; -import org.apache.polaris.service.config.RealmEntityManagerFactory; -import org.apache.polaris.service.config.Serializers; -import org.apache.polaris.service.config.TaskHandlerConfiguration; -import org.apache.polaris.service.context.CallContextCatalogFactory; -import org.apache.polaris.service.context.CallContextResolver; -import org.apache.polaris.service.context.PolarisCallContextCatalogFactory; -import org.apache.polaris.service.context.RealmContextResolver; -import org.apache.polaris.service.exception.IcebergExceptionMapper; -import org.apache.polaris.service.exception.IcebergJerseyViolationExceptionMapper; -import org.apache.polaris.service.exception.IcebergJsonProcessingExceptionMapper; -import org.apache.polaris.service.exception.PolarisExceptionMapper; -import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; -import org.apache.polaris.service.ratelimiter.RateLimiterFilter; -import org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl; -import org.apache.polaris.service.task.ManifestFileCleanupTaskHandler; -import org.apache.polaris.service.task.TableCleanupTaskHandler; -import org.apache.polaris.service.task.TaskExecutorImpl; -import org.apache.polaris.service.task.TaskFileIOSupplier; -import org.apache.polaris.service.throttling.StreamReadConstraintsExceptionMapper; -import org.apache.polaris.service.tracing.OpenTelemetryAware; -import org.apache.polaris.service.tracing.TracingFilter; -import org.eclipse.jetty.servlets.CrossOriginFilter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.slf4j.MDC; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.services.sts.StsClient; -import software.amazon.awssdk.services.sts.StsClientBuilder; - -public class PolarisApplication extends Application { - private static final Logger LOGGER = LoggerFactory.getLogger(PolarisApplication.class); - - public static void main(final String[] args) throws Exception { - new PolarisApplication().run(args); - printAsciiArt(); - } - - private static void printAsciiArt() throws IOException { - URL url = PolarisApplication.class.getResource("banner.txt"); - try (InputStream in = - requireNonNull(url, "banner.txt not found on classpath") - .openConnection() - .getInputStream()) { - System.out.println(new String(in.readAllBytes(), UTF_8)); - } - } - - @Override - public void initialize(Bootstrap bootstrap) { - // Enable variable substitution with environment variables - EnvironmentVariableSubstitutor substitutor = new EnvironmentVariableSubstitutor(false); - SubstitutingSourceProvider provider = - new SubstitutingSourceProvider(bootstrap.getConfigurationSourceProvider(), substitutor); - bootstrap.setConfigurationSourceProvider(provider); - - bootstrap.addCommand(new BootstrapRealmsCommand()); - bootstrap.addCommand(new PurgeRealmsCommand()); - } - - @Override - public void run(PolarisApplicationConfig configuration, Environment environment) { - MetaStoreManagerFactory metaStoreManagerFactory = configuration.getMetaStoreManagerFactory(); - - metaStoreManagerFactory.setStorageIntegrationProvider( - new PolarisStorageIntegrationProviderImpl( - () -> { - StsClientBuilder stsClientBuilder = StsClient.builder(); - AwsCredentialsProvider awsCredentialsProvider = configuration.credentialsProvider(); - if (awsCredentialsProvider != null) { - stsClientBuilder.credentialsProvider(awsCredentialsProvider); - } - return stsClientBuilder.build(); - }, - configuration.getGcpCredentialsProvider())); - - PolarisMetricRegistry polarisMetricRegistry = - new PolarisMetricRegistry(new PrometheusMeterRegistry(PrometheusConfig.DEFAULT)); - metaStoreManagerFactory.setMetricRegistry(polarisMetricRegistry); - - OpenTelemetry openTelemetry = setupTracing(); - if (metaStoreManagerFactory instanceof OpenTelemetryAware otAware) { - otAware.setOpenTelemetry(openTelemetry); - } - PolarisConfigurationStore configurationStore = configuration.getConfigurationStore(); - if (metaStoreManagerFactory instanceof ConfigurationStoreAware) { - ((ConfigurationStoreAware) metaStoreManagerFactory).setConfigurationStore(configurationStore); - } - RealmEntityManagerFactory entityManagerFactory = - new RealmEntityManagerFactory(metaStoreManagerFactory); - CallContextResolver callContextResolver = configuration.getCallContextResolver(); - callContextResolver.setMetaStoreManagerFactory(metaStoreManagerFactory); - if (callContextResolver instanceof ConfigurationStoreAware csa) { - csa.setConfigurationStore(configurationStore); - } - - RealmContextResolver realmContextResolver = configuration.getRealmContextResolver(); - realmContextResolver.setMetaStoreManagerFactory(metaStoreManagerFactory); - environment - .servlets() - .addFilter( - "realmContext", new ContextResolverFilter(realmContextResolver, callContextResolver)) - .addMappingForUrlPatterns(EnumSet.of(DispatcherType.REQUEST), true, "/*"); - - FileIOFactory fileIOFactory = configuration.getFileIOFactory(); - if (fileIOFactory instanceof MetricRegistryAware mrAware) { - mrAware.setMetricRegistry(polarisMetricRegistry); - } - if (fileIOFactory instanceof OpenTelemetryAware otAware) { - otAware.setOpenTelemetry(openTelemetry); - } - if (fileIOFactory instanceof ConfigurationStoreAware csAware) { - csAware.setConfigurationStore(configurationStore); - } - - TaskHandlerConfiguration taskConfig = configuration.getTaskHandler(); - TaskExecutorImpl taskExecutor = - new TaskExecutorImpl(taskConfig.executorService(), metaStoreManagerFactory); - TaskFileIOSupplier fileIOSupplier = - new TaskFileIOSupplier(metaStoreManagerFactory, fileIOFactory); - taskExecutor.addTaskHandler( - new TableCleanupTaskHandler(taskExecutor, metaStoreManagerFactory, fileIOSupplier)); - taskExecutor.addTaskHandler( - new ManifestFileCleanupTaskHandler( - fileIOSupplier, Executors.newVirtualThreadPerTaskExecutor())); - - LOGGER.info( - "Initializing PolarisCallContextCatalogFactory for metaStoreManagerType {}", - metaStoreManagerFactory); - CallContextCatalogFactory catalogFactory = - new PolarisCallContextCatalogFactory( - entityManagerFactory, metaStoreManagerFactory, taskExecutor, fileIOFactory); - - PolarisAuthorizer authorizer = new PolarisAuthorizerImpl(configurationStore); - IcebergCatalogAdapter catalogAdapter = - new IcebergCatalogAdapter( - catalogFactory, entityManagerFactory, metaStoreManagerFactory, authorizer); - environment.jersey().register(new IcebergRestCatalogApi(catalogAdapter)); - environment.jersey().register(new IcebergRestConfigurationApi(catalogAdapter)); - - FilterRegistration.Dynamic corsRegistration = - environment.servlets().addFilter("CORS", CrossOriginFilter.class); - corsRegistration.setInitParameter( - CrossOriginFilter.ALLOWED_ORIGINS_PARAM, - String.join(",", configuration.getCorsConfiguration().getAllowedOrigins())); - corsRegistration.setInitParameter( - CrossOriginFilter.ALLOWED_TIMING_ORIGINS_PARAM, - String.join(",", configuration.getCorsConfiguration().getAllowedTimingOrigins())); - corsRegistration.setInitParameter( - CrossOriginFilter.ALLOWED_METHODS_PARAM, - String.join(",", configuration.getCorsConfiguration().getAllowedMethods())); - corsRegistration.setInitParameter( - CrossOriginFilter.ALLOWED_HEADERS_PARAM, - String.join(",", configuration.getCorsConfiguration().getAllowedHeaders())); - corsRegistration.setInitParameter( - CrossOriginFilter.ALLOW_CREDENTIALS_PARAM, - String.join(",", configuration.getCorsConfiguration().getAllowCredentials())); - corsRegistration.setInitParameter( - CrossOriginFilter.PREFLIGHT_MAX_AGE_PARAM, - Objects.toString(configuration.getCorsConfiguration().getPreflightMaxAge())); - corsRegistration.setInitParameter( - CrossOriginFilter.ALLOW_CREDENTIALS_PARAM, - configuration.getCorsConfiguration().getAllowCredentials()); - corsRegistration.addMappingForUrlPatterns(EnumSet.of(DispatcherType.REQUEST), true, "/*"); - environment - .servlets() - .addFilter("tracing", new TracingFilter(openTelemetry)) - .addMappingForUrlPatterns(EnumSet.of(DispatcherType.REQUEST), true, "/*"); - - if (configuration.getRateLimiter() != null) { - environment.jersey().register(new RateLimiterFilter(configuration.getRateLimiter())); - } - - DiscoverableAuthenticator authenticator = - configuration.getPolarisAuthenticator(); - authenticator.setMetaStoreManagerFactory(metaStoreManagerFactory); - AuthFilter oauthCredentialAuthFilter = - new OAuthCredentialAuthFilter.Builder() - .setAuthenticator(authenticator) - .setPrefix("Bearer") - .buildAuthFilter(); - environment.jersey().register(new AuthDynamicFeature(oauthCredentialAuthFilter)); - environment.healthChecks().register("polaris", new PolarisHealthCheck()); - OAuth2ApiService oauth2Service = configuration.getOauth2Service(); - if (oauth2Service instanceof HasMetaStoreManagerFactory emfAware) { - emfAware.setMetaStoreManagerFactory(metaStoreManagerFactory); - } - environment.jersey().register(new IcebergRestOAuth2Api(oauth2Service)); - environment.jersey().register(new IcebergExceptionMapper()); - environment.jersey().register(new PolarisExceptionMapper()); - PolarisServiceImpl polarisService = - new PolarisServiceImpl(entityManagerFactory, metaStoreManagerFactory, authorizer); - environment.jersey().register(new PolarisCatalogsApi(polarisService)); - environment.jersey().register(new PolarisPrincipalsApi(polarisService)); - environment.jersey().register(new PolarisPrincipalRolesApi(polarisService)); - ObjectMapper objectMapper = environment.getObjectMapper(); - objectMapper.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); - objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - objectMapper.setPropertyNamingStrategy(new PropertyNamingStrategies.KebabCaseStrategy()); - - long maxRequestBodyBytes = configuration.getMaxRequestBodyBytes(); - if (maxRequestBodyBytes != REQUEST_BODY_BYTES_NO_LIMIT) { - objectMapper - .getFactory() - .setStreamReadConstraints( - StreamReadConstraints.builder().maxDocumentLength(maxRequestBodyBytes).build()); - LOGGER.info("Limiting request body size to {} bytes", maxRequestBodyBytes); - } - - environment.jersey().register(new StreamReadConstraintsExceptionMapper()); - RESTSerializers.registerAll(objectMapper); - Serializers.registerSerializers(objectMapper); - environment.jersey().register(new IcebergJsonProcessingExceptionMapper()); - environment.jersey().register(new IcebergJerseyViolationExceptionMapper()); - environment.jersey().register(new TimedApplicationEventListener(polarisMetricRegistry)); - - polarisMetricRegistry.init( - IcebergRestCatalogApi.class, - IcebergRestConfigurationApi.class, - IcebergRestOAuth2Api.class, - PolarisCatalogsApi.class, - PolarisPrincipalsApi.class, - PolarisPrincipalRolesApi.class); - - environment - .admin() - .addServlet( - "metrics", - new PrometheusMetricsServlet( - ((PrometheusMeterRegistry) polarisMetricRegistry.getMeterRegistry()) - .getPrometheusRegistry())) - .addMapping("/metrics"); - - // For in-memory metastore we need to bootstrap Service and Service principal at startup (for - // default realm) - // We can not utilize dropwizard Bootstrap command as command and server will be running two - // different processes - // and in-memory state will be lost b/w invocation of bootstrap command and running a server - if (metaStoreManagerFactory instanceof InMemoryPolarisMetaStoreManagerFactory) { - metaStoreManagerFactory.getOrCreateMetaStoreManager(configuration::getDefaultRealm); - } - - LOGGER.info("Server started successfully."); - } - - private static OpenTelemetry setupTracing() { - Resource resource = - Resource.getDefault().toBuilder() - .put(ServiceAttributes.SERVICE_NAME, "polaris") - .put(ServiceAttributes.SERVICE_VERSION, "0.1.0") - .build(); - SdkTracerProvider sdkTracerProvider = - SdkTracerProvider.builder() - .addSpanProcessor(SimpleSpanProcessor.create(LoggingSpanExporter.create())) - .setResource(resource) - .build(); - return OpenTelemetrySdk.builder() - .setTracerProvider(sdkTracerProvider) - .setPropagators( - ContextPropagators.create( - TextMapPropagator.composite( - W3CTraceContextPropagator.getInstance(), W3CBaggagePropagator.getInstance()))) - .build(); - } - - /** Resolves and sets ThreadLocal CallContext/RealmContext based on the request contents. */ - private static class ContextResolverFilter implements Filter { - private final RealmContextResolver realmContextResolver; - private final CallContextResolver callContextResolver; - - public ContextResolverFilter( - RealmContextResolver realmContextResolver, CallContextResolver callContextResolver) { - this.realmContextResolver = realmContextResolver; - this.callContextResolver = callContextResolver; - } - - @Override - public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) - throws IOException, ServletException { - HttpServletRequest httpRequest = (HttpServletRequest) request; - Stream headerNames = Collections.list(httpRequest.getHeaderNames()).stream(); - Map headers = - headerNames.collect(Collectors.toMap(Function.identity(), httpRequest::getHeader)); - RealmContext currentRealmContext = - realmContextResolver.resolveRealmContext( - httpRequest.getRequestURL().toString(), - httpRequest.getMethod(), - httpRequest.getRequestURI().substring(1), - request.getParameterMap().entrySet().stream() - .collect( - Collectors.toMap(Map.Entry::getKey, (e) -> ((String[]) e.getValue())[0])), - headers); - CallContext currentCallContext = - callContextResolver.resolveCallContext( - currentRealmContext, - httpRequest.getMethod(), - httpRequest.getRequestURI().substring(1), - request.getParameterMap().entrySet().stream() - .collect( - Collectors.toMap(Map.Entry::getKey, (e) -> ((String[]) e.getValue())[0])), - headers); - CallContext.setCurrentContext(currentCallContext); - try (MDC.MDCCloseable ignored1 = - MDC.putCloseable("realm", currentRealmContext.getRealmIdentifier()); - MDC.MDCCloseable ignored2 = - MDC.putCloseable("request_id", httpRequest.getHeader("request_id"))) { - chain.doFilter(request, response); - } finally { - Object contextCatalog = - currentCallContext - .contextVariables() - .get(CallContext.REQUEST_PATH_CATALOG_INSTANCE_KEY); - if (contextCatalog instanceof Closeable closeableCatalog) { - closeableCatalog.close(); - } - currentCallContext.close(); - } - } - } -} diff --git a/polaris-service/src/main/java/org/apache/polaris/service/PolarisHealthCheck.java b/polaris-service/src/main/java/org/apache/polaris/service/PolarisHealthCheck.java deleted file mode 100644 index 6f7a897af..000000000 --- a/polaris-service/src/main/java/org/apache/polaris/service/PolarisHealthCheck.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service; - -import com.codahale.metrics.health.HealthCheck; - -/** Default {@link HealthCheck} implementation. */ -public class PolarisHealthCheck extends HealthCheck { - @Override - protected Result check() throws Exception { - return Result.healthy(); - } -} diff --git a/polaris-service/src/main/java/org/apache/polaris/service/PurgeRealmsCommand.java b/polaris-service/src/main/java/org/apache/polaris/service/PurgeRealmsCommand.java deleted file mode 100644 index 238a82cbb..000000000 --- a/polaris-service/src/main/java/org/apache/polaris/service/PurgeRealmsCommand.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service; - -import io.dropwizard.core.cli.ConfiguredCommand; -import io.dropwizard.core.setup.Bootstrap; -import net.sourceforge.argparse4j.inf.Namespace; -import org.apache.polaris.core.PolarisConfigurationStore; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.service.config.ConfigurationStoreAware; -import org.apache.polaris.service.config.PolarisApplicationConfig; -import org.apache.polaris.service.context.CallContextResolver; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** Command for purging all metadata associated with a realm */ -public class PurgeRealmsCommand extends ConfiguredCommand { - private static final Logger LOGGER = LoggerFactory.getLogger(PurgeRealmsCommand.class); - - public PurgeRealmsCommand() { - super("purge", "purge principal credentials for all realms and prints them to log"); - } - - @Override - protected void run( - Bootstrap bootstrap, - Namespace namespace, - PolarisApplicationConfig configuration) - throws Exception { - MetaStoreManagerFactory metaStoreManagerFactory = configuration.getMetaStoreManagerFactory(); - - PolarisConfigurationStore configurationStore = configuration.getConfigurationStore(); - if (metaStoreManagerFactory instanceof ConfigurationStoreAware) { - ((ConfigurationStoreAware) metaStoreManagerFactory).setConfigurationStore(configurationStore); - } - CallContextResolver callContextResolver = configuration.getCallContextResolver(); - callContextResolver.setMetaStoreManagerFactory(metaStoreManagerFactory); - if (callContextResolver instanceof ConfigurationStoreAware csa) { - csa.setConfigurationStore(configurationStore); - } - - metaStoreManagerFactory.purgeRealms(configuration.getDefaultRealms()); - - LOGGER.info("Purge completed successfully."); - } -} diff --git a/polaris-service/src/main/java/org/apache/polaris/service/TimedApplicationEventListener.java b/polaris-service/src/main/java/org/apache/polaris/service/TimedApplicationEventListener.java deleted file mode 100644 index deaa0bba2..000000000 --- a/polaris-service/src/main/java/org/apache/polaris/service/TimedApplicationEventListener.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service; - -import static org.apache.polaris.core.monitor.PolarisMetricRegistry.TAG_RESP_CODE; - -import com.google.common.base.Stopwatch; -import io.micrometer.core.instrument.Tag; -import java.lang.reflect.Method; -import java.util.List; -import java.util.concurrent.TimeUnit; -import javax.ws.rs.ext.Provider; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.monitor.PolarisMetricRegistry; -import org.apache.polaris.core.resource.TimedApi; -import org.glassfish.jersey.server.monitoring.ApplicationEvent; -import org.glassfish.jersey.server.monitoring.ApplicationEventListener; -import org.glassfish.jersey.server.monitoring.RequestEvent; -import org.glassfish.jersey.server.monitoring.RequestEventListener; -import org.jetbrains.annotations.VisibleForTesting; - -/** - * An ApplicationEventListener that supports timing and error counting of Jersey resource methods - * annotated by {@link TimedApi}. It uses the {@link PolarisMetricRegistry} for metric collection - * and properly times the resource on success and increments the error counter on failure. - */ -@Provider -public class TimedApplicationEventListener implements ApplicationEventListener { - - /** - * Each API will increment a common counter (SINGLETON_METRIC_NAME) but have its API name tagged - * (TAG_API_NAME). - */ - public static final String SINGLETON_METRIC_NAME = "polaris.api"; - - public static final String TAG_API_NAME = "api_name"; - - // The PolarisMetricRegistry instance used for recording metrics and error counters. - private final PolarisMetricRegistry polarisMetricRegistry; - - public TimedApplicationEventListener(PolarisMetricRegistry polarisMetricRegistry) { - this.polarisMetricRegistry = polarisMetricRegistry; - } - - @VisibleForTesting - public PolarisMetricRegistry getMetricRegistry() { - return polarisMetricRegistry; - } - - @Override - public void onEvent(ApplicationEvent event) {} - - @Override - public RequestEventListener onRequest(RequestEvent event) { - return new TimedRequestEventListener(); - } - - /** - * A RequestEventListener implementation that handles timing of resource method execution and - * increments error counters on failures. The lifetime of the listener is tied to a single HTTP - * request. - */ - private class TimedRequestEventListener implements RequestEventListener { - private String metric; - private Stopwatch sw; - - /** Handles various types of RequestEvents to start timing, stop timing, and record metrics. */ - @Override - public void onEvent(RequestEvent event) { - String realmId = CallContext.getCurrentContext().getRealmContext().getRealmIdentifier(); - if (event.getType() == RequestEvent.Type.REQUEST_MATCHED) { - Method method = - event.getUriInfo().getMatchedResourceMethod().getInvocable().getHandlingMethod(); - if (method.isAnnotationPresent(TimedApi.class)) { - TimedApi timedApi = method.getAnnotation(TimedApi.class); - metric = timedApi.value(); - - // Increment both the counter with the API name in the metric name and a common metric - polarisMetricRegistry.incrementCounter(metric, realmId); - polarisMetricRegistry.incrementCounter( - SINGLETON_METRIC_NAME, realmId, Tag.of(TAG_API_NAME, metric)); - } - } else if (event.getType() == RequestEvent.Type.RESOURCE_METHOD_START && metric != null) { - sw = Stopwatch.createStarted(); - } else if (event.getType() == RequestEvent.Type.FINISHED && metric != null) { - if (event.isSuccess()) { - sw.stop(); - polarisMetricRegistry.recordTimer(metric, sw.elapsed(TimeUnit.MILLISECONDS), realmId); - } else { - int statusCode = event.getContainerResponse().getStatus(); - - // Increment both the counter with the API name in the metric name and a common metric - polarisMetricRegistry.incrementErrorCounter(metric, statusCode, realmId); - polarisMetricRegistry.incrementErrorCounter( - SINGLETON_METRIC_NAME, - realmId, - List.of( - Tag.of(TAG_API_NAME, metric), Tag.of(TAG_RESP_CODE, String.valueOf(statusCode)))); - } - } - } - } -} diff --git a/polaris-service/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java b/polaris-service/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java index 98a85fed1..a80460018 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java @@ -18,7 +18,8 @@ */ package org.apache.polaris.service.admin; -import jakarta.inject.Inject; +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -84,8 +85,6 @@ import org.apache.polaris.core.storage.StorageLocation; import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo; import org.apache.polaris.core.storage.azure.AzureStorageConfigurationInfo; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -111,7 +110,6 @@ public class PolarisAdminService { // Initialized in the authorize methods. private PolarisResolutionManifest resolutionManifest = null; - @Inject public PolarisAdminService( CallContext callContext, PolarisEntityManager entityManager, @@ -600,7 +598,7 @@ public void deleteCatalog(String name) { } } - public @NotNull CatalogEntity getCatalog(String name) { + public @Nonnull CatalogEntity getCatalog(String name) { PolarisAuthorizableOperation op = PolarisAuthorizableOperation.GET_CATALOG; authorizeBasicTopLevelEntityOperationOrThrow(op, name, PolarisEntityType.CATALOG); @@ -660,7 +658,7 @@ private void validateUpdateCatalogDiffOrThrow( } } - public @NotNull CatalogEntity updateCatalog(String name, UpdateCatalogRequest updateRequest) { + public @Nonnull CatalogEntity updateCatalog(String name, UpdateCatalogRequest updateRequest) { PolarisAuthorizableOperation op = PolarisAuthorizableOperation.UPDATE_CATALOG; authorizeBasicTopLevelEntityOperationOrThrow(op, name, PolarisEntityType.CATALOG); @@ -796,7 +794,7 @@ public void deletePrincipal(String name) { } } - public @NotNull PrincipalEntity getPrincipal(String name) { + public @Nonnull PrincipalEntity getPrincipal(String name) { PolarisAuthorizableOperation op = PolarisAuthorizableOperation.GET_PRINCIPAL; authorizeBasicTopLevelEntityOperationOrThrow(op, name, PolarisEntityType.PRINCIPAL); @@ -804,7 +802,7 @@ public void deletePrincipal(String name) { .orElseThrow(() -> new NotFoundException("Principal %s not found", name)); } - public @NotNull PrincipalEntity updatePrincipal( + public @Nonnull PrincipalEntity updatePrincipal( String name, UpdatePrincipalRequest updateRequest) { PolarisAuthorizableOperation op = PolarisAuthorizableOperation.UPDATE_PRINCIPAL; authorizeBasicTopLevelEntityOperationOrThrow(op, name, PolarisEntityType.PRINCIPAL); @@ -837,7 +835,7 @@ public void deletePrincipal(String name) { return returnedEntity; } - private @NotNull PrincipalWithCredentials rotateOrResetCredentialsHelper( + private @Nonnull PrincipalWithCredentials rotateOrResetCredentialsHelper( String principalName, boolean shouldReset) { PrincipalEntity currentPrincipalEntity = findPrincipalByName(principalName) @@ -876,14 +874,14 @@ public void deletePrincipal(String name) { newSecrets.getPrincipalClientId(), newSecrets.getMainSecret())); } - public @NotNull PrincipalWithCredentials rotateCredentials(String principalName) { + public @Nonnull PrincipalWithCredentials rotateCredentials(String principalName) { PolarisAuthorizableOperation op = PolarisAuthorizableOperation.ROTATE_CREDENTIALS; authorizeBasicTopLevelEntityOperationOrThrow(op, principalName, PolarisEntityType.PRINCIPAL); return rotateOrResetCredentialsHelper(principalName, false); } - public @NotNull PrincipalWithCredentials resetCredentials(String principalName) { + public @Nonnull PrincipalWithCredentials resetCredentials(String principalName) { PolarisAuthorizableOperation op = PolarisAuthorizableOperation.RESET_CREDENTIALS; authorizeBasicTopLevelEntityOperationOrThrow(op, principalName, PolarisEntityType.PRINCIPAL); @@ -958,7 +956,7 @@ public void deletePrincipalRole(String name) { } } - public @NotNull PrincipalRoleEntity getPrincipalRole(String name) { + public @Nonnull PrincipalRoleEntity getPrincipalRole(String name) { PolarisAuthorizableOperation op = PolarisAuthorizableOperation.GET_PRINCIPAL_ROLE; authorizeBasicTopLevelEntityOperationOrThrow(op, name, PolarisEntityType.PRINCIPAL_ROLE); @@ -966,7 +964,7 @@ public void deletePrincipalRole(String name) { .orElseThrow(() -> new NotFoundException("PrincipalRole %s not found", name)); } - public @NotNull PrincipalRoleEntity updatePrincipalRole( + public @Nonnull PrincipalRoleEntity updatePrincipalRole( String name, UpdatePrincipalRoleRequest updateRequest) { PolarisAuthorizableOperation op = PolarisAuthorizableOperation.UPDATE_PRINCIPAL_ROLE; authorizeBasicTopLevelEntityOperationOrThrow(op, name, PolarisEntityType.PRINCIPAL_ROLE); @@ -1079,7 +1077,7 @@ public void deleteCatalogRole(String catalogName, String name) { } } - public @NotNull CatalogRoleEntity getCatalogRole(String catalogName, String name) { + public @Nonnull CatalogRoleEntity getCatalogRole(String catalogName, String name) { PolarisAuthorizableOperation op = PolarisAuthorizableOperation.GET_CATALOG_ROLE; authorizeBasicCatalogRoleOperationOrThrow(op, catalogName, name); @@ -1087,7 +1085,7 @@ public void deleteCatalogRole(String catalogName, String name) { .orElseThrow(() -> new NotFoundException("CatalogRole %s not found", name)); } - public @NotNull CatalogRoleEntity updateCatalogRole( + public @Nonnull CatalogRoleEntity updateCatalogRole( String catalogName, String name, UpdateCatalogRoleRequest updateRequest) { PolarisAuthorizableOperation op = PolarisAuthorizableOperation.UPDATE_CATALOG_ROLE; authorizeBasicCatalogRoleOperationOrThrow(op, catalogName, name); @@ -1275,7 +1273,7 @@ public List listAssigneePrincipalsForPrincipalRole(String princip * @return list of grantees or securables matching the filter */ private List buildEntitiesFromGrantResults( - @NotNull LoadGrantsResult grantList, + @Nonnull LoadGrantsResult grantList, boolean grantees, @Nullable Function grantFilter) { Map granteeMap = grantList.getEntitiesAsMap(); diff --git a/polaris-service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java b/polaris-service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java index 148d4944c..ac59e67d5 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java @@ -18,6 +18,8 @@ */ package org.apache.polaris.service.admin; +import jakarta.enterprise.context.RequestScoped; +import jakarta.inject.Inject; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.SecurityContext; import java.util.List; @@ -73,6 +75,7 @@ import org.slf4j.LoggerFactory; /** Concrete implementation of the Polaris API services */ +@RequestScoped public class PolarisServiceImpl implements PolarisCatalogsApiService, PolarisPrincipalsApiService, @@ -81,18 +84,21 @@ public class PolarisServiceImpl private final RealmEntityManagerFactory entityManagerFactory; private final PolarisAuthorizer polarisAuthorizer; private final MetaStoreManagerFactory metaStoreManagerFactory; + private final CallContext callContext; + @Inject public PolarisServiceImpl( RealmEntityManagerFactory entityManagerFactory, MetaStoreManagerFactory metaStoreManagerFactory, - PolarisAuthorizer polarisAuthorizer) { + PolarisAuthorizer polarisAuthorizer, + CallContext callContext) { this.entityManagerFactory = entityManagerFactory; this.metaStoreManagerFactory = metaStoreManagerFactory; this.polarisAuthorizer = polarisAuthorizer; + this.callContext = callContext; } private PolarisAdminService newAdminService(SecurityContext securityContext) { - CallContext callContext = CallContext.getCurrentContext(); AuthenticatedPolarisPrincipal authenticatedPrincipal = (AuthenticatedPolarisPrincipal) securityContext.getUserPrincipal(); if (authenticatedPrincipal == null) { @@ -121,7 +127,6 @@ public Response createCatalog(CreateCatalogRequest request, SecurityContext secu } private void validateStorageConfig(StorageConfigInfo storageConfigInfo) { - CallContext callContext = CallContext.getCurrentContext(); PolarisCallContext polarisCallContext = callContext.getPolarisCallContext(); List allowedStorageTypes = polarisCallContext diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/Authenticator.java b/polaris-service/src/main/java/org/apache/polaris/service/auth/Authenticator.java similarity index 100% rename from polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/Authenticator.java rename to polaris-service/src/main/java/org/apache/polaris/service/auth/Authenticator.java diff --git a/polaris-service/src/main/java/org/apache/polaris/service/auth/BasePolarisAuthenticator.java b/polaris-service/src/main/java/org/apache/polaris/service/auth/BasePolarisAuthenticator.java index c06f17d86..ac51a2179 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/auth/BasePolarisAuthenticator.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/auth/BasePolarisAuthenticator.java @@ -38,23 +38,21 @@ import org.slf4j.LoggerFactory; /** - * Base implementation of {@link DiscoverableAuthenticator} constructs a {@link - * AuthenticatedPolarisPrincipal} from the token parsed by subclasses. The {@link - * AuthenticatedPolarisPrincipal} is read from the {@link PolarisMetaStoreManager} for the current - * {@link RealmContext}. If the token defines a non-empty set of scopes, only the principal roles - * specified in the scopes will be active for the current principal. Only the grants assigned to - * these roles will be active in the current request. + * Base implementation of {@link Authenticator} constructs a {@link AuthenticatedPolarisPrincipal} + * from the token parsed by subclasses. The {@link AuthenticatedPolarisPrincipal} is read from the + * {@link PolarisMetaStoreManager} for the current {@link RealmContext}. If the token defines a + * non-empty set of scopes, only the principal roles specified in the scopes will be active for the + * current principal. Only the grants assigned to these roles will be active in the current request. */ public abstract class BasePolarisAuthenticator - implements DiscoverableAuthenticator { + implements Authenticator { public static final String PRINCIPAL_ROLE_ALL = "PRINCIPAL_ROLE:ALL"; public static final String PRINCIPAL_ROLE_PREFIX = "PRINCIPAL_ROLE:"; private static final Logger LOGGER = LoggerFactory.getLogger(BasePolarisAuthenticator.class); - protected MetaStoreManagerFactory metaStoreManagerFactory; + protected final MetaStoreManagerFactory metaStoreManagerFactory; - @Override - public void setMetaStoreManagerFactory(MetaStoreManagerFactory metaStoreManagerFactory) { + protected BasePolarisAuthenticator(MetaStoreManagerFactory metaStoreManagerFactory) { this.metaStoreManagerFactory = metaStoreManagerFactory; } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/auth/DefaultOAuth2ApiService.java b/polaris-service/src/main/java/org/apache/polaris/service/auth/DefaultOAuth2ApiService.java index 57a9bc74d..2e30f2071 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/auth/DefaultOAuth2ApiService.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/auth/DefaultOAuth2ApiService.java @@ -20,31 +20,40 @@ import static java.nio.charset.StandardCharsets.UTF_8; -import com.fasterxml.jackson.annotation.JsonTypeName; +import io.quarkus.arc.lookup.LookupIfProperty; +import jakarta.enterprise.context.RequestScoped; +import jakarta.inject.Inject; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.SecurityContext; import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.hdfs.web.oauth2.OAuth2Constants; import org.apache.iceberg.rest.auth.OAuth2Properties; import org.apache.iceberg.rest.responses.OAuthTokenResponse; +import org.apache.polaris.core.config.RuntimeCandidate; import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.service.config.HasMetaStoreManagerFactory; -import org.apache.polaris.service.config.OAuth2ApiService; +import org.apache.polaris.service.catalog.api.IcebergRestOAuth2ApiService; import org.apache.polaris.service.types.TokenType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Default implementation of the {@link OAuth2ApiService} that generates a JWT token for the client - * if the client secret matches. + * Default implementation of the {@link IcebergRestOAuth2ApiService} that generates a JWT token for + * the client if the client secret matches. */ -@JsonTypeName("default") -public class DefaultOAuth2ApiService implements OAuth2ApiService, HasMetaStoreManagerFactory { +@RequestScoped +@RuntimeCandidate +@LookupIfProperty(name = "polaris.authentication.oauth2-service.type", stringValue = "default") +public class DefaultOAuth2ApiService implements IcebergRestOAuth2ApiService { private static final Logger LOGGER = LoggerFactory.getLogger(DefaultOAuth2ApiService.class); - private TokenBrokerFactory tokenBrokerFactory; + private final TokenBrokerFactory tokenBrokerFactory; + private final CallContext callContext; - public DefaultOAuth2ApiService() {} + @Inject + public DefaultOAuth2ApiService(TokenBrokerFactory tokenBrokerFactory, CallContext callContext) { + this.tokenBrokerFactory = tokenBrokerFactory; + this.callContext = callContext; + CallContext.setCurrentContext(callContext); + } @Override public Response getToken( @@ -60,8 +69,7 @@ public Response getToken( TokenType actorTokenType, SecurityContext securityContext) { - TokenBroker tokenBroker = - tokenBrokerFactory.apply(CallContext.getCurrentContext().getRealmContext()); + TokenBroker tokenBroker = tokenBrokerFactory.apply(callContext.getRealmContext()); if (!tokenBroker.supportsGrantType(grantType)) { return OAuthUtils.getResponseFromError(OAuthTokenErrorResponse.Error.unsupported_grant_type); } @@ -120,16 +128,4 @@ public Response getToken( .build()) .build(); } - - @Override - public void setMetaStoreManagerFactory(MetaStoreManagerFactory metaStoreManagerFactory) { - if (tokenBrokerFactory instanceof HasMetaStoreManagerFactory hemf) { - hemf.setMetaStoreManagerFactory(metaStoreManagerFactory); - } - } - - @Override - public void setTokenBroker(TokenBrokerFactory tokenBrokerFactory) { - this.tokenBrokerFactory = tokenBrokerFactory; - } } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/auth/DefaultPolarisAuthenticator.java b/polaris-service/src/main/java/org/apache/polaris/service/auth/DefaultPolarisAuthenticator.java index b57572d3b..e95e72613 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/auth/DefaultPolarisAuthenticator.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/auth/DefaultPolarisAuthenticator.java @@ -18,15 +18,33 @@ */ package org.apache.polaris.service.auth; -import com.fasterxml.jackson.annotation.JsonProperty; +import io.quarkus.arc.lookup.LookupIfProperty; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; import java.util.Optional; import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; +import org.apache.polaris.core.config.RuntimeCandidate; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.service.config.HasMetaStoreManagerFactory; +@ApplicationScoped +@RuntimeCandidate +@LookupIfProperty(name = "polaris.authentication.type", stringValue = "default") public class DefaultPolarisAuthenticator extends BasePolarisAuthenticator { - private TokenBrokerFactory tokenBrokerFactory; + + private final TokenBrokerFactory tokenBrokerFactory; + + // Required for CDI + public DefaultPolarisAuthenticator() { + this(null, null); + } + + @Inject + public DefaultPolarisAuthenticator( + MetaStoreManagerFactory metaStoreManagerFactory, TokenBrokerFactory tokenBrokerFactory) { + super(metaStoreManagerFactory); + this.tokenBrokerFactory = tokenBrokerFactory; + } @Override public Optional authenticate(String credentials) { @@ -35,18 +53,4 @@ public Optional authenticate(String credentials) DecodedToken decodedToken = handler.verify(credentials); return getPrincipal(decodedToken); } - - @Override - public void setMetaStoreManagerFactory(MetaStoreManagerFactory metaStoreManagerFactory) { - super.setMetaStoreManagerFactory(metaStoreManagerFactory); - if (tokenBrokerFactory instanceof HasMetaStoreManagerFactory) { - ((HasMetaStoreManagerFactory) tokenBrokerFactory) - .setMetaStoreManagerFactory(metaStoreManagerFactory); - } - } - - @JsonProperty("tokenBroker") - public void setTokenBroker(TokenBrokerFactory tokenBrokerFactory) { - this.tokenBrokerFactory = tokenBrokerFactory; - } } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/auth/DiscoverableAuthenticator.java b/polaris-service/src/main/java/org/apache/polaris/service/auth/DiscoverableAuthenticator.java deleted file mode 100644 index a6c2129b4..000000000 --- a/polaris-service/src/main/java/org/apache/polaris/service/auth/DiscoverableAuthenticator.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.auth; - -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import io.dropwizard.auth.Authenticator; -import io.dropwizard.jackson.Discoverable; -import java.security.Principal; -import org.apache.polaris.service.config.HasMetaStoreManagerFactory; - -/** - * Extension of the {@link Authenticator} interface that extends {@link Discoverable} so - * implementations can be discovered using the mechanisms described in the - * manual. The default implementation is {@link TestInlineBearerTokenPolarisAuthenticator}. - * - * @param - * @param

- */ -@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, property = "class") -public interface DiscoverableAuthenticator - extends Authenticator, Discoverable, HasMetaStoreManagerFactory {} diff --git a/polaris-service/src/main/java/org/apache/polaris/service/auth/JWTRSAKeyPairFactory.java b/polaris-service/src/main/java/org/apache/polaris/service/auth/JWTRSAKeyPairFactory.java index 5e3441f47..41557d333 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/auth/JWTRSAKeyPairFactory.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/auth/JWTRSAKeyPairFactory.java @@ -18,17 +18,29 @@ */ package org.apache.polaris.service.auth; -import com.fasterxml.jackson.annotation.JsonTypeName; +import io.quarkus.arc.lookup.LookupIfProperty; +import jakarta.enterprise.context.RequestScoped; +import java.time.Duration; +import org.apache.polaris.core.config.RuntimeCandidate; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.service.config.HasMetaStoreManagerFactory; +import org.eclipse.microprofile.config.inject.ConfigProperty; -@JsonTypeName("rsa-key-pair") -public class JWTRSAKeyPairFactory implements TokenBrokerFactory, HasMetaStoreManagerFactory { - private int maxTokenGenerationInSeconds = 3600; - private MetaStoreManagerFactory metaStoreManagerFactory; +@RequestScoped +@RuntimeCandidate +@LookupIfProperty( + name = "polaris.authentication.token-broker-factory.type", + stringValue = "rsa-key-pair") +public class JWTRSAKeyPairFactory implements TokenBrokerFactory { - public void setMaxTokenGenerationInSeconds(int maxTokenGenerationInSeconds) { + private final MetaStoreManagerFactory metaStoreManagerFactory; + private final Duration maxTokenGenerationInSeconds; + + public JWTRSAKeyPairFactory( + MetaStoreManagerFactory metaStoreManagerFactory, + @ConfigProperty(name = "polaris.authentication.token-broker-factory.max-token-generation") + Duration maxTokenGenerationInSeconds) { + this.metaStoreManagerFactory = metaStoreManagerFactory; this.maxTokenGenerationInSeconds = maxTokenGenerationInSeconds; } @@ -36,11 +48,6 @@ public void setMaxTokenGenerationInSeconds(int maxTokenGenerationInSeconds) { public TokenBroker apply(RealmContext realmContext) { return new JWTRSAKeyPair( metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext), - maxTokenGenerationInSeconds); - } - - @Override - public void setMetaStoreManagerFactory(MetaStoreManagerFactory metaStoreManagerFactory) { - this.metaStoreManagerFactory = metaStoreManagerFactory; + (int) maxTokenGenerationInSeconds.toSeconds()); } } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/auth/JWTSymmetricKeyFactory.java b/polaris-service/src/main/java/org/apache/polaris/service/auth/JWTSymmetricKeyFactory.java index 297103f8f..8324a7768 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/auth/JWTSymmetricKeyFactory.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/auth/JWTSymmetricKeyFactory.java @@ -18,21 +18,47 @@ */ package org.apache.polaris.service.auth; -import com.fasterxml.jackson.annotation.JsonTypeName; +import io.quarkus.arc.lookup.LookupIfProperty; +import jakarta.enterprise.context.RequestScoped; import java.io.IOException; import java.nio.file.Files; -import java.nio.file.Paths; +import java.nio.file.Path; +import java.time.Duration; +import java.util.Optional; import java.util.function.Supplier; +import org.apache.polaris.core.config.RuntimeCandidate; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.service.config.HasMetaStoreManagerFactory; +import org.eclipse.microprofile.config.inject.ConfigProperty; + +@RequestScoped +@RuntimeCandidate +@LookupIfProperty( + name = "polaris.authentication.token-broker-factory.type", + stringValue = "symmetric-key") +public class JWTSymmetricKeyFactory implements TokenBrokerFactory { -@JsonTypeName("symmetric-key") -public class JWTSymmetricKeyFactory implements TokenBrokerFactory, HasMetaStoreManagerFactory { private MetaStoreManagerFactory metaStoreManagerFactory; - private int maxTokenGenerationInSeconds = 3600; - private String file; - private String secret; + private final Duration maxTokenGenerationInSeconds; + private final Path file; + private final String secret; + + public JWTSymmetricKeyFactory( + MetaStoreManagerFactory metaStoreManagerFactory, + @ConfigProperty(name = "polaris.authentication.token-broker-factory.max-token-generation") + Duration maxTokenGenerationInSeconds, + @ConfigProperty(name = "polaris.authentication.token-broker-factory.symmetric-key.secret") + Optional secret, + @ConfigProperty(name = "polaris.authentication.token-broker-factory.symmetric-key.file") + Optional file) { + this.metaStoreManagerFactory = metaStoreManagerFactory; + this.maxTokenGenerationInSeconds = maxTokenGenerationInSeconds; + this.secret = secret.orElse(null); + this.file = file.orElse(null); + if (this.file == null && this.secret == null) { + throw new IllegalStateException("Either file or secret must be set"); + } + } @Override public TokenBroker apply(RealmContext realmContext) { @@ -42,34 +68,17 @@ public TokenBroker apply(RealmContext realmContext) { Supplier secretSupplier = secret != null ? () -> secret : readSecretFromDisk(); return new JWTSymmetricKeyBroker( metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext), - maxTokenGenerationInSeconds, + (int) maxTokenGenerationInSeconds.toSeconds(), secretSupplier); } private Supplier readSecretFromDisk() { return () -> { try { - return Files.readString(Paths.get(file)); + return Files.readString(file); } catch (IOException e) { throw new RuntimeException("Failed to read secret from file: " + file, e); } }; } - - public void setMaxTokenGenerationInSeconds(int maxTokenGenerationInSeconds) { - this.maxTokenGenerationInSeconds = maxTokenGenerationInSeconds; - } - - public void setFile(String file) { - this.file = file; - } - - public void setSecret(String secret) { - this.secret = secret; - } - - @Override - public void setMetaStoreManagerFactory(MetaStoreManagerFactory metaStoreManagerFactory) { - this.metaStoreManagerFactory = metaStoreManagerFactory; - } } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/auth/KeyProvider.java b/polaris-service/src/main/java/org/apache/polaris/service/auth/KeyProvider.java index bdd07e9bf..a28b9a059 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/auth/KeyProvider.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/auth/KeyProvider.java @@ -18,13 +18,10 @@ */ package org.apache.polaris.service.auth; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import io.dropwizard.jackson.Discoverable; import java.security.PrivateKey; import java.security.PublicKey; -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") -public interface KeyProvider extends Discoverable { +public interface KeyProvider { PublicKey getPublicKey(); PrivateKey getPrivateKey(); diff --git a/polaris-service/src/main/java/org/apache/polaris/service/auth/LocalRSAKeyProvider.java b/polaris-service/src/main/java/org/apache/polaris/service/auth/LocalRSAKeyProvider.java index 666af1f2a..cd8dfd16e 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/auth/LocalRSAKeyProvider.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/auth/LocalRSAKeyProvider.java @@ -18,10 +18,13 @@ */ package org.apache.polaris.service.auth; +import io.quarkus.arc.lookup.LookupIfProperty; +import jakarta.enterprise.context.RequestScoped; import java.io.IOException; import java.security.PrivateKey; import java.security.PublicKey; import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.config.RuntimeCandidate; import org.apache.polaris.core.context.CallContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,6 +33,9 @@ * Class that can load public / private keys stored on localhost. Meant to be a simple * implementation for now where a PEM file is loaded off disk. */ +@RequestScoped +@RuntimeCandidate +@LookupIfProperty(name = "polaris.authentication.key-provider.type", stringValue = "local-rsa") public class LocalRSAKeyProvider implements KeyProvider { private static final String LOCAL_PRIVATE_KEY_LOCATION_KEY = "LOCAL_PRIVATE_KEY_LOCATION_KEY"; diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/OAuthCredentialAuthFilter.java b/polaris-service/src/main/java/org/apache/polaris/service/auth/OAuthCredentialAuthFilter.java similarity index 100% rename from polaris-service-quarkus/src/main/java/org/apache/polaris/service/auth/OAuthCredentialAuthFilter.java rename to polaris-service/src/main/java/org/apache/polaris/service/auth/OAuthCredentialAuthFilter.java diff --git a/polaris-service/src/main/java/org/apache/polaris/service/auth/TestInlineBearerTokenPolarisAuthenticator.java b/polaris-service/src/main/java/org/apache/polaris/service/auth/TestInlineBearerTokenPolarisAuthenticator.java index b37a077d0..61dd85eae 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/auth/TestInlineBearerTokenPolarisAuthenticator.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/auth/TestInlineBearerTokenPolarisAuthenticator.java @@ -19,7 +19,10 @@ package org.apache.polaris.service.auth; import com.google.common.base.Splitter; -import io.dropwizard.auth.AuthenticationException; +import io.quarkus.arc.lookup.LookupIfProperty; +import io.quarkus.security.AuthenticationFailedException; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -27,15 +30,17 @@ import java.util.stream.Collectors; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; +import org.apache.polaris.core.config.RuntimeCandidate; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.PolarisPrincipalSecrets; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * {@link io.dropwizard.auth.Authenticator} that parses a token as a sequence of key/value pairs. - * Specifically, we expect to find + * Authenticator that parses a token as a sequence of key/value pairs. Specifically, we expect to + * find * *

    *
  • principal - the clientId of the principal @@ -45,13 +50,27 @@ * This class does not expect a client to be either present or correct. Lookup is delegated to the * {@link PolarisMetaStoreManager} for the current realm. */ +@ApplicationScoped +@RuntimeCandidate +@LookupIfProperty(name = "polaris.authentication.type", stringValue = "test") public class TestInlineBearerTokenPolarisAuthenticator extends BasePolarisAuthenticator { private static final Logger LOGGER = LoggerFactory.getLogger(TestInlineBearerTokenPolarisAuthenticator.class); + // Required for CDI + public TestInlineBearerTokenPolarisAuthenticator() { + this(null); + } + + @Inject + public TestInlineBearerTokenPolarisAuthenticator( + MetaStoreManagerFactory metaStoreManagerFactory) { + super(metaStoreManagerFactory); + } + @Override public Optional authenticate(String credentials) - throws AuthenticationException { + throws AuthenticationFailedException { Map properties = extractPrincipal(credentials); PolarisMetaStoreManager metaStoreManager = metaStoreManagerFactory.getOrCreateMetaStoreManager( diff --git a/polaris-service/src/main/java/org/apache/polaris/service/auth/TestOAuth2ApiService.java b/polaris-service/src/main/java/org/apache/polaris/service/auth/TestOAuth2ApiService.java index a000489f1..8544bb2b4 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/auth/TestOAuth2ApiService.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/auth/TestOAuth2ApiService.java @@ -18,7 +18,9 @@ */ package org.apache.polaris.service.auth; -import com.fasterxml.jackson.annotation.JsonTypeName; +import io.quarkus.arc.lookup.LookupIfProperty; +import jakarta.enterprise.context.RequestScoped; +import jakarta.inject.Inject; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.SecurityContext; import java.util.HashMap; @@ -27,22 +29,29 @@ import org.apache.iceberg.exceptions.NotAuthorizedException; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.auth.PolarisSecretsManager.PrincipalSecretsResult; +import org.apache.polaris.core.config.RuntimeCandidate; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.service.config.HasMetaStoreManagerFactory; -import org.apache.polaris.service.config.OAuth2ApiService; +import org.apache.polaris.service.catalog.api.IcebergRestOAuth2ApiService; import org.apache.polaris.service.types.TokenType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@JsonTypeName("test") -public class TestOAuth2ApiService implements OAuth2ApiService, HasMetaStoreManagerFactory { +@RequestScoped +@RuntimeCandidate +@LookupIfProperty(name = "polaris.authentication.oauth2-service.type", stringValue = "test") +public class TestOAuth2ApiService implements IcebergRestOAuth2ApiService { private static final Logger LOGGER = LoggerFactory.getLogger(TestOAuth2ApiService.class); - private MetaStoreManagerFactory metaStoreManagerFactory; + private final MetaStoreManagerFactory metaStoreManagerFactory; + + @Inject + public TestOAuth2ApiService(MetaStoreManagerFactory metaStoreManagerFactory) { + this.metaStoreManagerFactory = metaStoreManagerFactory; + } @Override public Response getToken( @@ -107,12 +116,4 @@ private String getPrincipalName(String clientId) { return principalResult.getEntity().getName(); } } - - @Override - public void setMetaStoreManagerFactory(MetaStoreManagerFactory metaStoreManagerFactory) { - this.metaStoreManagerFactory = metaStoreManagerFactory; - } - - @Override - public void setTokenBroker(TokenBrokerFactory tokenBrokerFactory) {} } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/auth/TokenBroker.java b/polaris-service/src/main/java/org/apache/polaris/service/auth/TokenBroker.java index 16e27ec5e..f5fea376b 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/auth/TokenBroker.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/auth/TokenBroker.java @@ -18,15 +18,14 @@ */ package org.apache.polaris.service.auth; +import jakarta.annotation.Nonnull; import java.util.Optional; import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.auth.PolarisSecretsManager.PrincipalSecretsResult; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PrincipalEntity; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.service.types.TokenType; -import org.jetbrains.annotations.NotNull; /** Generic token class intended to be extended by different token types */ public interface TokenBroker { @@ -43,16 +42,17 @@ TokenResponse generateFromToken( DecodedToken verify(String token); - static @NotNull Optional findPrincipalEntity( + static @Nonnull Optional findPrincipalEntity( PolarisMetaStoreManager metaStoreManager, String clientId, String clientSecret) { // Validate the principal is present and secrets match PolarisCallContext polarisCallContext = CallContext.getCurrentContext().getPolarisCallContext(); - PrincipalSecretsResult principalSecrets = + PolarisMetaStoreManager.PrincipalSecretsResult principalSecrets = metaStoreManager.loadPrincipalSecrets(polarisCallContext, clientId); if (!principalSecrets.isSuccess()) { return Optional.empty(); } - if (!principalSecrets.getPrincipalSecrets().matchesSecret(clientSecret)) { + if (!principalSecrets.getPrincipalSecrets().getMainSecret().equals(clientSecret) + && !principalSecrets.getPrincipalSecrets().getSecondarySecret().equals(clientSecret)) { return Optional.empty(); } PolarisMetaStoreManager.EntityResult result = diff --git a/polaris-service/src/main/java/org/apache/polaris/service/auth/TokenBrokerFactory.java b/polaris-service/src/main/java/org/apache/polaris/service/auth/TokenBrokerFactory.java index abb15c9db..131f3ed64 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/auth/TokenBrokerFactory.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/auth/TokenBrokerFactory.java @@ -18,8 +18,6 @@ */ package org.apache.polaris.service.auth; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import io.dropwizard.jackson.Discoverable; import java.util.function.Function; import org.apache.polaris.core.context.RealmContext; @@ -27,5 +25,4 @@ * Factory that creates a {@link TokenBroker} for generating and parsing. The {@link TokenBroker} is * created based on the realm context. */ -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") -public interface TokenBrokerFactory extends Function, Discoverable {} +public interface TokenBrokerFactory extends Function {} diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index 4911b5cfd..6b6677da0 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -23,6 +23,7 @@ import com.google.common.base.Objects; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; +import jakarta.annotation.Nonnull; import java.io.Closeable; import java.io.IOException; import java.util.Arrays; @@ -94,20 +95,13 @@ import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifestCatalogView; import org.apache.polaris.core.persistence.resolver.ResolverPath; import org.apache.polaris.core.persistence.resolver.ResolverStatus; -import org.apache.polaris.core.storage.InMemoryStorageIntegration; -import org.apache.polaris.core.storage.PolarisCredentialVendor; -import org.apache.polaris.core.storage.PolarisStorageActions; -import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; -import org.apache.polaris.core.storage.PolarisStorageIntegration; -import org.apache.polaris.core.storage.StorageLocation; +import org.apache.polaris.core.storage.*; import org.apache.polaris.core.storage.aws.PolarisS3FileIOClientFactory; import org.apache.polaris.service.catalog.io.FileIOFactory; import org.apache.polaris.service.exception.IcebergExceptionMapper; import org.apache.polaris.service.task.TaskExecutor; import org.apache.polaris.service.types.NotificationRequest; import org.apache.polaris.service.types.NotificationType; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.TestOnly; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.core.exception.SdkException; @@ -212,7 +206,7 @@ public String name() { return catalogName; } - @TestOnly + @VisibleForTesting FileIO getIo() { return catalogFileIO; } @@ -553,7 +547,7 @@ private String resolveNamespaceLocation(Namespace namespace, Map } } - private static @NotNull String resolveLocationForPath(List parentPath) { + private static @Nonnull String resolveLocationForPath(List parentPath) { // always take the first object. If it has the base-location, stop there AtomicBoolean foundBaseLocation = new AtomicBoolean(false); return parentPath.reversed().stream() @@ -845,7 +839,7 @@ public String transformTableLikeLocation(String specifiedTableLikeLocation) { return specifiedTableLikeLocation; } - private @NotNull Optional findStorageInfo(TableIdentifier tableIdentifier) { + private @Nonnull Optional findStorageInfo(TableIdentifier tableIdentifier) { PolarisResolvedPathWrapper resolvedTableEntities = resolvedEntityView.getResolvedPath(tableIdentifier, PolarisEntitySubType.TABLE); @@ -1422,7 +1416,7 @@ private void validateMetadataFileInTableDir( } } - private static @NotNull Optional findStorageInfoFromHierarchy( + private static @Nonnull Optional findStorageInfoFromHierarchy( PolarisResolvedPathWrapper resolvedStorageEntity) { Optional storageInfoEntity = resolvedStorageEntity.getRawFullPath().reversed().stream() @@ -1830,7 +1824,7 @@ private void updateTableLike(TableIdentifier identifier, PolarisEntity entity) { } @SuppressWarnings("FormatStringAnnotation") - private @NotNull PolarisMetaStoreManager.DropEntityResult dropTableLike( + private @Nonnull PolarisMetaStoreManager.DropEntityResult dropTableLike( PolarisEntitySubType subType, TableIdentifier identifier, Map storageProperties, diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java index 94ea65243..0b84fd268 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java @@ -22,6 +22,8 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; +import jakarta.enterprise.context.RequestScoped; +import jakarta.inject.Inject; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.SecurityContext; import java.net.URLEncoder; @@ -68,6 +70,7 @@ * org.apache.iceberg.rest.CatalogHandlers} after finding the appropriate {@link Catalog} for the * current {@link RealmContext}. */ +@RequestScoped public class IcebergCatalogAdapter implements IcebergRestCatalogApiService, IcebergRestConfigurationApiService { @@ -76,6 +79,7 @@ public class IcebergCatalogAdapter private final RealmEntityManagerFactory entityManagerFactory; private final PolarisAuthorizer polarisAuthorizer; + @Inject public IcebergCatalogAdapter( CallContextCatalogFactory catalogFactory, RealmEntityManagerFactory entityManagerFactory, @@ -296,7 +300,7 @@ public Response registerTable( public Response renameTable( String prefix, RenameTableRequest renameTableRequest, SecurityContext securityContext) { newHandlerWrapper(securityContext, prefix).renameTable(renameTableRequest); - return Response.ok(javax.ws.rs.core.Response.Status.NO_CONTENT).build(); + return Response.ok(Response.Status.NO_CONTENT).build(); } @Override diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java index cf1ee6f87..7687ea310 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java @@ -18,14 +18,18 @@ */ package org.apache.polaris.service.catalog.io; -import com.fasterxml.jackson.annotation.JsonTypeName; +import io.quarkus.arc.lookup.LookupIfProperty; +import jakarta.enterprise.context.ApplicationScoped; import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.CatalogUtil; import org.apache.iceberg.io.FileIO; +import org.apache.polaris.core.config.RuntimeCandidate; /** A simple FileIOFactory implementation that defers all the work to the Iceberg SDK */ -@JsonTypeName("default") +@ApplicationScoped +@RuntimeCandidate +@LookupIfProperty(name = "polaris.io.file-io-factory.type", stringValue = "default") public class DefaultFileIOFactory implements FileIOFactory { @Override public FileIO loadFileIO(String impl, Map properties) { diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/io/FileIOFactory.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/io/FileIOFactory.java index 206aaeaa0..ca3c08511 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/io/FileIOFactory.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/io/FileIOFactory.java @@ -18,16 +18,10 @@ */ package org.apache.polaris.service.catalog.io; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import io.dropwizard.jackson.Discoverable; import java.util.Map; import org.apache.iceberg.io.FileIO; /** Interface for providing a way to construct FileIO objects, such as for reading/writing S3. */ -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - include = JsonTypeInfo.As.PROPERTY, - property = "factoryType") -public interface FileIOFactory extends Discoverable { +public interface FileIOFactory { FileIO loadFileIO(String impl, Map properties); } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/io/WasbTranslatingFileIOFactory.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/io/WasbTranslatingFileIOFactory.java index 0483a5de4..a96063b8d 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/io/WasbTranslatingFileIOFactory.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/io/WasbTranslatingFileIOFactory.java @@ -18,19 +18,22 @@ */ package org.apache.polaris.service.catalog.io; -import com.fasterxml.jackson.annotation.JsonTypeName; +import io.quarkus.arc.lookup.LookupIfProperty; +import jakarta.enterprise.context.ApplicationScoped; import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.CatalogUtil; import org.apache.iceberg.io.FileIO; +import org.apache.polaris.core.config.RuntimeCandidate; /** A {@link FileIOFactory} that translates WASB paths to ABFS ones */ -@JsonTypeName("wasb") +@ApplicationScoped +@RuntimeCandidate +@LookupIfProperty(name = "polaris.io.file-io-factory.type", stringValue = "wasb") public class WasbTranslatingFileIOFactory implements FileIOFactory { @Override public FileIO loadFileIO(String ioImpl, Map properties) { - WasbTranslatingFileIO wrapped = - new WasbTranslatingFileIO(CatalogUtil.loadFileIO(ioImpl, properties, new Configuration())); - return wrapped; + return new WasbTranslatingFileIO( + CatalogUtil.loadFileIO(ioImpl, properties, new Configuration())); } } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/config/ConfigurationStoreAware.java b/polaris-service/src/main/java/org/apache/polaris/service/config/ConfigurationStoreAware.java deleted file mode 100644 index 645f1b001..000000000 --- a/polaris-service/src/main/java/org/apache/polaris/service/config/ConfigurationStoreAware.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.config; - -import org.apache.polaris.core.PolarisConfigurationStore; - -/** Interface allows injection of a {@link PolarisConfigurationStore} */ -public interface ConfigurationStoreAware { - - void setConfigurationStore(PolarisConfigurationStore configurationStore); -} diff --git a/polaris-service/src/main/java/org/apache/polaris/service/config/CorsConfiguration.java b/polaris-service/src/main/java/org/apache/polaris/service/config/CorsConfiguration.java deleted file mode 100644 index f1154ea7d..000000000 --- a/polaris-service/src/main/java/org/apache/polaris/service/config/CorsConfiguration.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.config; - -import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.List; - -public class CorsConfiguration { - private List allowedOrigins = List.of("*"); - private List allowedTimingOrigins = List.of("*"); - private List allowedMethods = List.of("*"); - private List allowedHeaders = List.of("*"); - private List exposedHeaders = List.of("*"); - private Integer preflightMaxAge = 600; - private String allowCredentials = "true"; - - public List getAllowedOrigins() { - return allowedOrigins; - } - - @JsonProperty("allowed-origins") - public void setAllowedOrigins(List allowedOrigins) { - this.allowedOrigins = allowedOrigins; - } - - public void setAllowedTimingOrigins(List allowedTimingOrigins) { - this.allowedTimingOrigins = allowedTimingOrigins; - } - - @JsonProperty("allowed-timing-origins") - public List getAllowedTimingOrigins() { - return allowedTimingOrigins; - } - - public List getAllowedMethods() { - return allowedMethods; - } - - @JsonProperty("allowed-methods") - public void setAllowedMethods(List allowedMethods) { - this.allowedMethods = allowedMethods; - } - - public List getAllowedHeaders() { - return allowedHeaders; - } - - @JsonProperty("allowed-headers") - public void setAllowedHeaders(List allowedHeaders) { - this.allowedHeaders = allowedHeaders; - } - - public List getExposedHeaders() { - return exposedHeaders; - } - - @JsonProperty("exposed-headers") - public void setExposedHeaders(List exposedHeaders) { - this.exposedHeaders = exposedHeaders; - } - - public Integer getPreflightMaxAge() { - return preflightMaxAge; - } - - @JsonProperty("preflight-max-age") - public void setPreflightMaxAge(Integer preflightMaxAge) { - this.preflightMaxAge = preflightMaxAge; - } - - public String getAllowCredentials() { - return allowCredentials; - } - - @JsonProperty("allowed-credentials") - public void setAllowCredentials(String allowCredentials) { - this.allowCredentials = allowCredentials; - } -} diff --git a/polaris-service/src/main/java/org/apache/polaris/service/config/DefaultConfigurationStore.java b/polaris-service/src/main/java/org/apache/polaris/service/config/DefaultConfigurationStore.java index 7f04bbd50..63496ef20 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/config/DefaultConfigurationStore.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/config/DefaultConfigurationStore.java @@ -18,33 +18,81 @@ */ package org.apache.polaris.service.config; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.annotation.Nullable; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import java.util.Map; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisConfigurationStore; -import org.apache.polaris.core.context.CallContext; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; +import org.eclipse.microprofile.config.inject.ConfigProperty; +@ApplicationScoped public class DefaultConfigurationStore implements PolarisConfigurationStore { - private final Map config; - private final Map> realmConfig; - public DefaultConfigurationStore(Map config) { - this.config = config; - this.realmConfig = Map.of(); - } + private final Map properties; + // FIXME the whole PolarisConfigurationStore + PolarisConfiguration needs to be refactored + // to become a proper Quarkus configuration object + @Inject public DefaultConfigurationStore( - Map config, Map> realmConfig) { - this.config = config; - this.realmConfig = realmConfig; + ObjectMapper objectMapper, + @ConfigProperty(name = "polaris.config.feature-configurations") + Map properties) { + this(convertMap(objectMapper, properties)); + } + + public DefaultConfigurationStore(Map properties) { + this.properties = Map.copyOf(properties); + } + + private static Map convertMap( + ObjectMapper objectMapper, Map properties) { + Map m = new HashMap<>(); + for (String configName : properties.keySet()) { + String json = properties.get(configName); + try { + JsonNode node = objectMapper.readTree(json); + m.put(configName, configValue(node)); + } catch (JsonProcessingException e) { + throw new RuntimeException( + "Invalid JSON value for feature configuration: " + configName, e); + } + } + return m; + } + + private static Object configValue(JsonNode node) { + return switch (node.getNodeType()) { + case BOOLEAN -> node.asBoolean(); + case STRING -> node.asText(); + case NUMBER -> + switch (node.numberType()) { + case INT, LONG -> node.asLong(); + case FLOAT, DOUBLE -> node.asDouble(); + default -> + throw new IllegalArgumentException("Unsupported number type: " + node.numberType()); + }; + case ARRAY -> { + List list = new ArrayList<>(); + node.elements().forEachRemaining(n -> list.add(configValue(n))); + yield List.copyOf(list); + } + default -> + throw new IllegalArgumentException( + "Unsupported feature configuration JSON type: " + node.getNodeType()); + }; } - @SuppressWarnings("unchecked") @Override - public @Nullable T getConfiguration(@NotNull PolarisCallContext ctx, String configName) { - String realm = CallContext.getCurrentContext().getRealmContext().getRealmIdentifier(); - return (T) - realmConfig.getOrDefault(realm, Map.of()).getOrDefault(configName, config.get(configName)); + public @Nullable T getConfiguration(PolarisCallContext ctx, String configName) { + @SuppressWarnings("unchecked") + T o = (T) properties.get(configName); + return o; } } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/config/HasMetaStoreManagerFactory.java b/polaris-service/src/main/java/org/apache/polaris/service/config/HasMetaStoreManagerFactory.java deleted file mode 100644 index d2c8cc05b..000000000 --- a/polaris-service/src/main/java/org/apache/polaris/service/config/HasMetaStoreManagerFactory.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.config; - -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; - -public interface HasMetaStoreManagerFactory { - void setMetaStoreManagerFactory(MetaStoreManagerFactory metaStoreManagerFactory); -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/config/JacksonConfig.java b/polaris-service/src/main/java/org/apache/polaris/service/config/JacksonConfig.java similarity index 100% rename from polaris-service-quarkus/src/main/java/org/apache/polaris/service/config/JacksonConfig.java rename to polaris-service/src/main/java/org/apache/polaris/service/config/JacksonConfig.java diff --git a/polaris-service/src/main/java/org/apache/polaris/service/config/OAuth2ApiService.java b/polaris-service/src/main/java/org/apache/polaris/service/config/OAuth2ApiService.java deleted file mode 100644 index 81c219afa..000000000 --- a/polaris-service/src/main/java/org/apache/polaris/service/config/OAuth2ApiService.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.config; - -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import io.dropwizard.jackson.Discoverable; -import org.apache.polaris.service.auth.TokenBrokerFactory; -import org.apache.polaris.service.catalog.api.IcebergRestOAuth2ApiService; - -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") -public interface OAuth2ApiService extends Discoverable, IcebergRestOAuth2ApiService { - void setTokenBroker(TokenBrokerFactory tokenBrokerFactory); -} diff --git a/polaris-service/src/main/java/org/apache/polaris/service/config/PolarisApplicationConfig.java b/polaris-service/src/main/java/org/apache/polaris/service/config/PolarisApplicationConfig.java deleted file mode 100644 index 7d5fd8fdf..000000000 --- a/polaris-service/src/main/java/org/apache/polaris/service/config/PolarisApplicationConfig.java +++ /dev/null @@ -1,281 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.config; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.auth.oauth2.AccessToken; -import com.google.auth.oauth2.GoogleCredentials; -import com.google.common.base.Preconditions; -import io.dropwizard.core.Configuration; -import java.io.IOException; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.function.Supplier; -import javax.annotation.Nullable; -import org.apache.commons.lang3.StringUtils; -import org.apache.polaris.core.PolarisConfigurationStore; -import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.service.auth.DiscoverableAuthenticator; -import org.apache.polaris.service.catalog.io.FileIOFactory; -import org.apache.polaris.service.context.CallContextResolver; -import org.apache.polaris.service.context.RealmContextResolver; -import org.apache.polaris.service.ratelimiter.RateLimiter; -import org.slf4j.LoggerFactory; -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; - -/** - * Configuration specific to a Polaris REST Service. Place these entries in a YML file for them to - * be picked up, i.e. `iceberg-rest-server.yml` - */ -public class PolarisApplicationConfig extends Configuration { - private MetaStoreManagerFactory metaStoreManagerFactory; - private String defaultRealm = "default-realm"; - private RealmContextResolver realmContextResolver; - private CallContextResolver callContextResolver; - private DiscoverableAuthenticator polarisAuthenticator; - private CorsConfiguration corsConfiguration = new CorsConfiguration(); - private TaskHandlerConfiguration taskHandler = new TaskHandlerConfiguration(); - private Map globalFeatureConfiguration = Map.of(); - private Map> realmConfiguration = Map.of(); - private List defaultRealms; - private String awsAccessKey; - private String awsSecretKey; - private FileIOFactory fileIOFactory; - private RateLimiter rateLimiter; - - private AccessToken gcpAccessToken; - - public static final long REQUEST_BODY_BYTES_NO_LIMIT = -1; - private long maxRequestBodyBytes = REQUEST_BODY_BYTES_NO_LIMIT; - - @JsonProperty("metaStoreManager") - public void setMetaStoreManagerFactory(MetaStoreManagerFactory metaStoreManagerFactory) { - this.metaStoreManagerFactory = metaStoreManagerFactory; - } - - @JsonProperty("metaStoreManager") - public MetaStoreManagerFactory getMetaStoreManagerFactory() { - return metaStoreManagerFactory; - } - - @JsonProperty("io") - public void setFileIOFactory(FileIOFactory fileIOFactory) { - this.fileIOFactory = fileIOFactory; - } - - @JsonProperty("io") - public FileIOFactory getFileIOFactory() { - return fileIOFactory; - } - - @JsonProperty("authenticator") - public void setPolarisAuthenticator( - DiscoverableAuthenticator polarisAuthenticator) { - this.polarisAuthenticator = polarisAuthenticator; - } - - public DiscoverableAuthenticator - getPolarisAuthenticator() { - return polarisAuthenticator; - } - - public RealmContextResolver getRealmContextResolver() { - realmContextResolver.setDefaultRealm(this.defaultRealm); - return realmContextResolver; - } - - public void setRealmContextResolver(RealmContextResolver realmContextResolver) { - this.realmContextResolver = realmContextResolver; - } - - public CallContextResolver getCallContextResolver() { - return callContextResolver; - } - - @JsonProperty("callContextResolver") - public void setCallContextResolver(CallContextResolver callContextResolver) { - this.callContextResolver = callContextResolver; - } - - private OAuth2ApiService oauth2Service; - - @JsonProperty("oauth2") - public void setOauth2Service(OAuth2ApiService oauth2Service) { - this.oauth2Service = oauth2Service; - } - - public OAuth2ApiService getOauth2Service() { - return oauth2Service; - } - - public String getDefaultRealm() { - return defaultRealm; - } - - @JsonProperty("defaultRealm") - public void setDefaultRealm(String defaultRealm) { - this.defaultRealm = defaultRealm; - realmContextResolver.setDefaultRealm(defaultRealm); - } - - @JsonProperty("cors") - public CorsConfiguration getCorsConfiguration() { - return corsConfiguration; - } - - @JsonProperty("cors") - public void setCorsConfiguration(CorsConfiguration corsConfiguration) { - this.corsConfiguration = corsConfiguration; - } - - @JsonProperty("rateLimiter") - public RateLimiter getRateLimiter() { - return rateLimiter; - } - - @JsonProperty("rateLimiter") - public void setRateLimiter(@Nullable RateLimiter rateLimiter) { - this.rateLimiter = rateLimiter; - } - - public void setTaskHandler(TaskHandlerConfiguration taskHandler) { - this.taskHandler = taskHandler; - } - - public TaskHandlerConfiguration getTaskHandler() { - return taskHandler; - } - - @JsonProperty("featureConfiguration") - public void setFeatureConfiguration(Map featureConfiguration) { - this.globalFeatureConfiguration = featureConfiguration; - } - - @JsonProperty("realmFeatureConfiguration") - public void setRealmFeatureConfiguration(Map> realmConfiguration) { - this.realmConfiguration = realmConfiguration; - } - - @JsonProperty("maxRequestBodyBytes") - public void setMaxRequestBodyBytes(long maxRequestBodyBytes) { - // The underlying library that we use to implement the limit treats all values <= 0 as the - // same, so we block all but -1 to prevent ambiguity. - Preconditions.checkArgument( - maxRequestBodyBytes == -1 || maxRequestBodyBytes > 0, - "maxRequestBodyBytes must be a positive integer or %s to specify no limit.", - REQUEST_BODY_BYTES_NO_LIMIT); - - this.maxRequestBodyBytes = maxRequestBodyBytes; - } - - public long getMaxRequestBodyBytes() { - return maxRequestBodyBytes; - } - - public PolarisConfigurationStore getConfigurationStore() { - return new DefaultConfigurationStore(globalFeatureConfiguration, realmConfiguration); - } - - public List getDefaultRealms() { - return defaultRealms; - } - - public AwsCredentialsProvider credentialsProvider() { - if (StringUtils.isNotBlank(awsAccessKey) && StringUtils.isNotBlank(awsSecretKey)) { - LoggerFactory.getLogger(PolarisApplicationConfig.class) - .warn("Using hard-coded AWS credentials - this is not recommended for production"); - return StaticCredentialsProvider.create( - AwsBasicCredentials.create(awsAccessKey, awsSecretKey)); - } - return null; - } - - public void setAwsAccessKey(String awsAccessKey) { - this.awsAccessKey = awsAccessKey; - } - - public void setAwsSecretKey(String awsSecretKey) { - this.awsSecretKey = awsSecretKey; - } - - public void setDefaultRealms(List defaultRealms) { - this.defaultRealms = defaultRealms; - } - - public Supplier getGcpCredentialsProvider() { - return () -> - Optional.ofNullable(gcpAccessToken) - .map(GoogleCredentials::create) - .orElseGet( - () -> { - try { - return GoogleCredentials.getApplicationDefault(); - } catch (IOException e) { - throw new RuntimeException("Failed to get GCP credentials", e); - } - }); - } - - @JsonProperty("gcp_credentials") - void setGcpCredentials(GcpAccessToken token) { - this.gcpAccessToken = - new AccessToken( - token.getAccessToken(), - new Date(System.currentTimeMillis() + token.getExpiresIn() * 1000)); - } - - /** - * A static AccessToken representation used to store a static token and expiration date. This - * should strictly be used for testing. - */ - static class GcpAccessToken { - private String accessToken; - private long expiresIn; - - public GcpAccessToken() {} - - public GcpAccessToken(String accessToken, long expiresIn) { - this.accessToken = accessToken; - this.expiresIn = expiresIn; - } - - public String getAccessToken() { - return accessToken; - } - - @JsonProperty("access_token") - public void setAccessToken(String accessToken) { - this.accessToken = accessToken; - } - - public long getExpiresIn() { - return expiresIn; - } - - @JsonProperty("expires_in") - public void setExpiresIn(long expiresIn) { - this.expiresIn = expiresIn; - } - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/config/PolarisQuarkusInfrastructure.java b/polaris-service/src/main/java/org/apache/polaris/service/config/PolarisQuarkusInfrastructure.java similarity index 99% rename from polaris-service-quarkus/src/main/java/org/apache/polaris/service/config/PolarisQuarkusInfrastructure.java rename to polaris-service/src/main/java/org/apache/polaris/service/config/PolarisQuarkusInfrastructure.java index df5aa477c..4d5db88db 100644 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/config/PolarisQuarkusInfrastructure.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/config/PolarisQuarkusInfrastructure.java @@ -32,6 +32,7 @@ import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; import org.apache.polaris.core.auth.PolarisAuthorizer; import org.apache.polaris.core.auth.PolarisAuthorizerImpl; +import org.apache.polaris.core.config.RuntimeCandidate; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; diff --git a/polaris-service/src/main/java/org/apache/polaris/service/config/RealmEntityManagerFactory.java b/polaris-service/src/main/java/org/apache/polaris/service/config/RealmEntityManagerFactory.java index b896a13eb..afa1da612 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/config/RealmEntityManagerFactory.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/config/RealmEntityManagerFactory.java @@ -18,6 +18,8 @@ */ package org.apache.polaris.service.config; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import org.apache.polaris.core.context.RealmContext; @@ -26,19 +28,18 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** Gets or creates PolarisEntityManager instances based on config values and RealmContext. */ +@ApplicationScoped public class RealmEntityManagerFactory { - private static final Logger LOGGER = LoggerFactory.getLogger(RealmEntityManagerFactory.class); + + private static final Logger LOGGER = + LoggerFactory.getLogger(RealmEntityManagerFactory.class.getName()); + private final MetaStoreManagerFactory metaStoreManagerFactory; // Key: realmIdentifier private final Map cachedEntityManagers = new ConcurrentHashMap<>(); - // Subclasses for test injection. - protected RealmEntityManagerFactory() { - this.metaStoreManagerFactory = null; - } - + @Inject public RealmEntityManagerFactory(MetaStoreManagerFactory metaStoreManagerFactory) { this.metaStoreManagerFactory = metaStoreManagerFactory; } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/config/TaskHandlerConfiguration.java b/polaris-service/src/main/java/org/apache/polaris/service/config/TaskHandlerConfiguration.java index fee41e71b..bc8cab508 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/config/TaskHandlerConfiguration.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/config/TaskHandlerConfiguration.java @@ -19,24 +19,25 @@ package org.apache.polaris.service.config; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import jakarta.enterprise.context.ApplicationScoped; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ThreadFactory; +import org.eclipse.microprofile.config.inject.ConfigProperty; +@ApplicationScoped public class TaskHandlerConfiguration { - private int poolSize = 10; - private boolean fixedSize = true; - private String threadNamePattern = "taskHandler-%d"; - public void setPoolSize(int poolSize) { - this.poolSize = poolSize; - } + private final int poolSize; + private final boolean fixedSize; + private final String threadNamePattern; - public void setFixedSize(boolean fixedSize) { + public TaskHandlerConfiguration( + @ConfigProperty(name = "polaris.tasks.pool-size") int poolSize, + @ConfigProperty(name = "polaris.tasks.fixed-size") boolean fixedSize, + @ConfigProperty(name = "polaris.tasks.thread-name-pattern") String threadNamePattern) { + this.poolSize = poolSize; this.fixedSize = fixedSize; - } - - public void setThreadNamePattern(String threadNamePattern) { this.threadNamePattern = threadNamePattern; } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/context/CallContextResolver.java b/polaris-service/src/main/java/org/apache/polaris/service/context/CallContextResolver.java index 6221e73f0..71b21d0f3 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/context/CallContextResolver.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/context/CallContextResolver.java @@ -18,16 +18,12 @@ */ package org.apache.polaris.service.context; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import io.dropwizard.jackson.Discoverable; import java.util.Map; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.service.config.HasMetaStoreManagerFactory; /** Uses the resolved RealmContext to further resolve elements of the CallContext. */ -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") -public interface CallContextResolver extends HasMetaStoreManagerFactory, Discoverable { +public interface CallContextResolver { CallContext resolveCallContext( RealmContext realmContext, String method, diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/context/DefaultCallContextResolver.java b/polaris-service/src/main/java/org/apache/polaris/service/context/DefaultCallContextResolver.java similarity index 98% rename from polaris-service-quarkus/src/main/java/org/apache/polaris/service/context/DefaultCallContextResolver.java rename to polaris-service/src/main/java/org/apache/polaris/service/context/DefaultCallContextResolver.java index 7b8038397..0a8211bc4 100644 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/context/DefaultCallContextResolver.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/context/DefaultCallContextResolver.java @@ -29,11 +29,11 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisConfigurationStore; import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.config.RuntimeCandidate; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisMetaStoreSession; -import org.apache.polaris.service.config.RuntimeCandidate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/polaris-service/src/main/java/org/apache/polaris/service/context/DefaultContextResolver.java b/polaris-service/src/main/java/org/apache/polaris/service/context/DefaultContextResolver.java deleted file mode 100644 index 520d8fdcd..000000000 --- a/polaris-service/src/main/java/org/apache/polaris/service/context/DefaultContextResolver.java +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.context; - -import com.fasterxml.jackson.annotation.JsonTypeName; -import com.google.common.base.Splitter; -import java.time.Clock; -import java.time.ZoneId; -import java.util.HashMap; -import java.util.Map; -import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.PolarisConfigurationStore; -import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; -import org.apache.polaris.core.PolarisDiagnostics; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.core.persistence.PolarisMetaStoreSession; -import org.apache.polaris.service.config.ConfigurationStoreAware; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * For local/dev testing, this resolver simply expects a custom bearer-token format that is a - * semicolon-separated list of colon-separated key/value pairs that constitute the realm properties. - * - *

    Example: principal:data-engineer;password:test;realm:acct123 - */ -@JsonTypeName("default") -public class DefaultContextResolver - implements RealmContextResolver, CallContextResolver, ConfigurationStoreAware { - private static final Logger LOGGER = LoggerFactory.getLogger(DefaultContextResolver.class); - - public static final String REALM_PROPERTY_KEY = "realm"; - - public static final String PRINCIPAL_PROPERTY_KEY = "principal"; - public static final String PRINCIPAL_PROPERTY_DEFAULT_VALUE = "default-principal"; - - private MetaStoreManagerFactory metaStoreManagerFactory; - private PolarisConfigurationStore configurationStore; - private String defaultRealm = "default-realm"; - - /** - * During CallContext resolution that might depend on RealmContext, the {@code - * entityManagerFactory} will be used to resolve elements of the CallContext which require - * additional information from an underlying entity store. - */ - @Override - public void setMetaStoreManagerFactory(MetaStoreManagerFactory metaStoreManagerFactory) { - this.metaStoreManagerFactory = metaStoreManagerFactory; - } - - @Override - public RealmContext resolveRealmContext( - String requestURL, - String method, - String path, - Map queryParams, - Map headers) { - // Since this default resolver is strictly for use in test/dev environments, we'll consider - // it safe to log all contents. Any "real" resolver used in a prod environment should make - // sure to only log non-sensitive contents. - LOGGER.debug( - "Resolving RealmContext for method: {}, path: {}, queryParams: {}, headers: {}", - method, - path, - queryParams, - headers); - final Map parsedProperties = parseBearerTokenAsKvPairs(headers); - - if (!parsedProperties.containsKey(REALM_PROPERTY_KEY) - && headers.containsKey(REALM_PROPERTY_KEY)) { - parsedProperties.put(REALM_PROPERTY_KEY, headers.get(REALM_PROPERTY_KEY)); - } - - if (!parsedProperties.containsKey(REALM_PROPERTY_KEY)) { - LOGGER.warn( - "Failed to parse {} from headers; using {}", REALM_PROPERTY_KEY, getDefaultRealm()); - parsedProperties.put(REALM_PROPERTY_KEY, getDefaultRealm()); - } - return () -> parsedProperties.get(REALM_PROPERTY_KEY); - } - - @Override - public void setDefaultRealm(String defaultRealm) { - this.defaultRealm = defaultRealm; - } - - @Override - public String getDefaultRealm() { - return this.defaultRealm; - } - - @Override - public CallContext resolveCallContext( - final RealmContext realmContext, - String method, - String path, - Map queryParams, - Map headers) { - LOGGER - .atDebug() - .addKeyValue("realmContext", realmContext.getRealmIdentifier()) - .addKeyValue("method", method) - .addKeyValue("path", path) - .addKeyValue("queryParams", queryParams) - .addKeyValue("headers", headers) - .log("Resolving CallContext"); - final Map parsedProperties = parseBearerTokenAsKvPairs(headers); - - if (!parsedProperties.containsKey(PRINCIPAL_PROPERTY_KEY)) { - LOGGER.warn( - "Failed to parse {} from headers ({}); using {}", - PRINCIPAL_PROPERTY_KEY, - headers, - PRINCIPAL_PROPERTY_DEFAULT_VALUE); - parsedProperties.put(PRINCIPAL_PROPERTY_KEY, PRINCIPAL_PROPERTY_DEFAULT_VALUE); - } - - PolarisDiagnostics diagServices = new PolarisDefaultDiagServiceImpl(); - PolarisMetaStoreSession metaStoreSession = - metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); - PolarisCallContext polarisContext = - new PolarisCallContext( - metaStoreSession, - diagServices, - configurationStore, - Clock.system(ZoneId.systemDefault())); - return CallContext.of(realmContext, polarisContext); - } - - /** - * Returns kv pairs parsed from the "Authorization: Bearer k1:v1;k2:v2;k3:v3" header if it exists; - * if missing, returns empty map. - */ - private static Map parseBearerTokenAsKvPairs(Map headers) { - Map parsedProperties = new HashMap<>(); - if (headers != null) { - String authHeader = headers.get("Authorization"); - if (authHeader != null) { - String[] parts = authHeader.split(" "); - if (parts.length == 2 && "Bearer".equalsIgnoreCase(parts[0])) { - if (parts[1].matches("[\\w\\d=_+-]+:[\\w\\d=+_-]+(?:;[\\w\\d=+_-]+:[\\w\\d=+_-]+)*")) { - parsedProperties.putAll( - Splitter.on(';').trimResults().withKeyValueSeparator(':').split(parts[1])); - } - } - } - } - return parsedProperties; - } - - @Override - public void setConfigurationStore(PolarisConfigurationStore configurationStore) { - this.configurationStore = configurationStore; - } -} diff --git a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/context/DefaultRealmContextResolver.java b/polaris-service/src/main/java/org/apache/polaris/service/context/DefaultRealmContextResolver.java similarity index 98% rename from polaris-service-quarkus/src/main/java/org/apache/polaris/service/context/DefaultRealmContextResolver.java rename to polaris-service/src/main/java/org/apache/polaris/service/context/DefaultRealmContextResolver.java index 813c3aeb6..2fc1d1ba7 100644 --- a/polaris-service-quarkus/src/main/java/org/apache/polaris/service/context/DefaultRealmContextResolver.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/context/DefaultRealmContextResolver.java @@ -24,8 +24,8 @@ import jakarta.inject.Inject; import java.util.HashMap; import java.util.Map; +import org.apache.polaris.core.config.RuntimeCandidate; import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.service.config.RuntimeCandidate; import org.eclipse.microprofile.config.inject.ConfigProperty; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/polaris-service/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java b/polaris-service/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java index 83d659a83..a23802431 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java @@ -18,6 +18,8 @@ */ package org.apache.polaris.service.context; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; @@ -38,6 +40,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@ApplicationScoped public class PolarisCallContextCatalogFactory implements CallContextCatalogFactory { private static final Logger LOGGER = LoggerFactory.getLogger(PolarisCallContextCatalogFactory.class); @@ -50,6 +53,7 @@ public class PolarisCallContextCatalogFactory implements CallContextCatalogFacto private final FileIOFactory fileIOFactory; private final MetaStoreManagerFactory metaStoreManagerFactory; + @Inject public PolarisCallContextCatalogFactory( RealmEntityManagerFactory entityManagerFactory, MetaStoreManagerFactory metaStoreManagerFactory, diff --git a/polaris-service/src/main/java/org/apache/polaris/service/context/RealmContextResolver.java b/polaris-service/src/main/java/org/apache/polaris/service/context/RealmContextResolver.java index 681a10b60..fedc1fde2 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/context/RealmContextResolver.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/context/RealmContextResolver.java @@ -18,14 +18,10 @@ */ package org.apache.polaris.service.context; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import io.dropwizard.jackson.Discoverable; import java.util.Map; import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.service.config.HasMetaStoreManagerFactory; -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") -public interface RealmContextResolver extends Discoverable, HasMetaStoreManagerFactory { +public interface RealmContextResolver { RealmContext resolveRealmContext( String requestURL, @@ -34,7 +30,5 @@ RealmContext resolveRealmContext( Map queryParams, Map headers); - void setDefaultRealm(String defaultRealm); - String getDefaultRealm(); } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/exception/IcebergExceptionMapper.java b/polaris-service/src/main/java/org/apache/polaris/service/exception/IcebergExceptionMapper.java index 6be8d9f42..22032c30d 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/exception/IcebergExceptionMapper.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/exception/IcebergExceptionMapper.java @@ -25,6 +25,7 @@ import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.ext.ExceptionMapper; +import jakarta.ws.rs.ext.Provider; import java.util.Arrays; import java.util.Collection; import java.util.Locale; @@ -51,11 +52,11 @@ import org.apache.iceberg.exceptions.UnprocessableEntityException; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.rest.responses.ErrorResponse; -import org.jetbrains.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.services.s3.model.S3Exception; +@Provider public class IcebergExceptionMapper implements ExceptionMapper { private static final Logger LOGGER = LoggerFactory.getLogger(IcebergExceptionMapper.class); @@ -138,7 +139,6 @@ public static boolean containsAnyAccessDeniedHint(String message) { return ACCESS_DENIED_HINTS.stream().anyMatch(messageLower::contains); } - @VisibleForTesting public static Collection getAccessDeniedHints() { return ImmutableSet.copyOf(ACCESS_DENIED_HINTS); } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/exception/IcebergJerseyViolationExceptionMapper.java b/polaris-service/src/main/java/org/apache/polaris/service/exception/IcebergJerseyViolationExceptionMapper.java index e26e89a2b..d5ec76841 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/exception/IcebergJerseyViolationExceptionMapper.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/exception/IcebergJerseyViolationExceptionMapper.java @@ -18,22 +18,19 @@ */ package org.apache.polaris.service.exception; -import io.dropwizard.jersey.validation.JerseyViolationException; +import jakarta.validation.ConstraintViolationException; import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.ext.ExceptionMapper; import jakarta.ws.rs.ext.Provider; import org.apache.iceberg.rest.responses.ErrorResponse; -/** - * Override of the default JerseyViolationExceptionMapper to provide an Iceberg ErrorResponse with - * the exception details. - */ +/** See {@code io.dropwizard.jersey.validation.JerseyViolationException} */ @Provider public class IcebergJerseyViolationExceptionMapper - implements ExceptionMapper { + implements ExceptionMapper { @Override - public Response toResponse(JerseyViolationException exception) { + public Response toResponse(ConstraintViolationException exception) { final String message = "Invalid value: " + exception.getMessage(); ErrorResponse icebergErrorResponse = ErrorResponse.builder() diff --git a/polaris-service/src/main/java/org/apache/polaris/service/exception/IcebergJsonProcessingExceptionMapper.java b/polaris-service/src/main/java/org/apache/polaris/service/exception/IcebergJsonProcessingExceptionMapper.java index 929453ae1..77782398c 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/exception/IcebergJsonProcessingExceptionMapper.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/exception/IcebergJsonProcessingExceptionMapper.java @@ -18,25 +18,40 @@ */ package org.apache.polaris.service.exception; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonGenerationException; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.exc.InvalidDefinitionException; import com.fasterxml.jackson.databind.exc.ValueInstantiationException; -import io.dropwizard.jersey.errors.LoggingExceptionMapper; +import jakarta.annotation.Nullable; import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.Response.Status; +import jakarta.ws.rs.ext.ExceptionMapper; import jakarta.ws.rs.ext.Provider; +import java.util.Locale; +import java.util.concurrent.ThreadLocalRandom; import org.apache.iceberg.rest.responses.ErrorResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -/** - * Override of the default JsonProcessingExceptionMapper to provide an Iceberg ErrorResponse with - * the exception details. This code mostly comes from Dropwizard's {@link - * io.dropwizard.jersey.jackson.JsonProcessingExceptionMapper} - */ +/** See Dropwizard's {@code io.dropwizard.jersey.jackson.JsonProcessingExceptionMapper} */ @Provider public final class IcebergJsonProcessingExceptionMapper - extends LoggingExceptionMapper { + implements ExceptionMapper { + + private static final Logger LOGGER = + LoggerFactory.getLogger(IcebergJsonProcessingExceptionMapper.class); + + @JsonInclude(Include.NON_NULL) + public record ErrorMessage( + @JsonProperty("code") int code, + @JsonProperty("message") @Nullable String message, + @JsonProperty("details") @Nullable String details) {} + @Override public Response toResponse(JsonProcessingException exception) { /* @@ -44,13 +59,23 @@ public Response toResponse(JsonProcessingException exception) { */ if (exception instanceof JsonGenerationException || exception instanceof InvalidDefinitionException) { - return super.toResponse(exception); // LoggingExceptionMapper will log exception + long id = ThreadLocalRandom.current().nextLong(); + LOGGER.error(String.format(Locale.ROOT, "Error handling a request: %016x", id), exception); + String message = + String.format( + Locale.ROOT, + "There was an error processing your request. It has been logged (ID %016x).", + id); + return Response.status(Status.INTERNAL_SERVER_ERROR.getStatusCode()) + .type(MediaType.APPLICATION_JSON_TYPE) + .entity(new ErrorMessage(500, message, null)) + .build(); } /* * Otherwise, it's those pesky users. */ - logger.info("Unable to process JSON: {}", exception.getMessage()); + LOGGER.info("Unable to process JSON: {}", exception.getMessage()); String messagePrefix = switch (exception) { diff --git a/polaris-service/src/main/java/org/apache/polaris/service/logging/PolarisJsonLayoutFactory.java b/polaris-service/src/main/java/org/apache/polaris/service/logging/PolarisJsonLayoutFactory.java deleted file mode 100644 index 476c5a997..000000000 --- a/polaris-service/src/main/java/org/apache/polaris/service/logging/PolarisJsonLayoutFactory.java +++ /dev/null @@ -1,242 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.logging; - -import ch.qos.logback.classic.LoggerContext; -import ch.qos.logback.classic.pattern.ExtendedThrowableProxyConverter; -import ch.qos.logback.classic.pattern.RootCauseFirstThrowableProxyConverter; -import ch.qos.logback.classic.pattern.ThrowableHandlingConverter; -import ch.qos.logback.classic.spi.ILoggingEvent; -import ch.qos.logback.core.LayoutBase; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import com.google.common.collect.ImmutableMap; -import io.dropwizard.logging.json.AbstractJsonLayoutBaseFactory; -import io.dropwizard.logging.json.EventAttribute; -import io.dropwizard.logging.json.layout.EventJsonLayout; -import io.dropwizard.logging.json.layout.ExceptionFormat; -import io.dropwizard.logging.json.layout.JsonFormatter; -import io.dropwizard.logging.json.layout.TimestampFormatter; -import java.util.ArrayList; -import java.util.Collections; -import java.util.EnumSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TimeZone; -import java.util.stream.Collectors; -import org.checkerframework.checker.nullness.qual.Nullable; - -/** - * Basically a direct copy of {@link io.dropwizard.logging.json.EventJsonLayoutBaseFactory} that - * adds support for {@link ILoggingEvent#getKeyValuePairs()} in the output. By default, additional - * key/value pairs are included as the `params` field of the json output, but they can optionally be - * flattened into the log event output. - * - *

    To use this appender, change the appender type to `polaris` - * loggers: - * org.apache.iceberg.rest: DEBUG - * org.apache.iceberg.polaris: DEBUG - * appenders: - * - type: console - * threshold: ALL - * layout: - * type: polaris - * flattenKeyValues: false - * includeKeyValues: true - * - */ -@JsonTypeName("polaris") -public class PolarisJsonLayoutFactory extends AbstractJsonLayoutBaseFactory { - private EnumSet includes = - EnumSet.of( - EventAttribute.LEVEL, - EventAttribute.THREAD_NAME, - EventAttribute.MDC, - EventAttribute.MARKER, - EventAttribute.LOGGER_NAME, - EventAttribute.MESSAGE, - EventAttribute.EXCEPTION, - EventAttribute.TIMESTAMP); - - private Set includesMdcKeys = Collections.emptySet(); - private boolean flattenMdc = false; - private boolean includeKeyValues = true; - private boolean flattenKeyValues = false; - - @Nullable private ExceptionFormat exceptionFormat; - - @JsonProperty - public EnumSet getIncludes() { - return includes; - } - - @JsonProperty - public void setIncludes(EnumSet includes) { - this.includes = includes; - } - - @JsonProperty - public Set getIncludesMdcKeys() { - return includesMdcKeys; - } - - @JsonProperty - public void setIncludesMdcKeys(Set includesMdcKeys) { - this.includesMdcKeys = includesMdcKeys; - } - - @JsonProperty - public boolean isFlattenMdc() { - return flattenMdc; - } - - @JsonProperty - public void setFlattenMdc(boolean flattenMdc) { - this.flattenMdc = flattenMdc; - } - - @JsonProperty - public boolean isIncludeKeyValues() { - return includeKeyValues; - } - - @JsonProperty - public void setIncludeKeyValues(boolean includeKeyValues) { - this.includeKeyValues = includeKeyValues; - } - - @JsonProperty - public boolean isFlattenKeyValues() { - return flattenKeyValues; - } - - @JsonProperty - public void setFlattenKeyValues(boolean flattenKeyValues) { - this.flattenKeyValues = flattenKeyValues; - } - - /** - * @since 2.0 - */ - @JsonProperty("exception") - public void setExceptionFormat(ExceptionFormat exceptionFormat) { - this.exceptionFormat = exceptionFormat; - } - - /** - * @since 2.0 - */ - @JsonProperty("exception") - @Nullable - public ExceptionFormat getExceptionFormat() { - return exceptionFormat; - } - - @Override - public LayoutBase build(LoggerContext context, TimeZone timeZone) { - final PolarisJsonLayout jsonLayout = - new PolarisJsonLayout( - createDropwizardJsonFormatter(), - createTimestampFormatter(timeZone), - createThrowableProxyConverter(context), - includes, - getCustomFieldNames(), - getAdditionalFields(), - includesMdcKeys, - flattenMdc, - includeKeyValues, - flattenKeyValues); - jsonLayout.setContext(context); - return jsonLayout; - } - - public static class PolarisJsonLayout extends EventJsonLayout { - private final boolean includeKeyValues; - private final boolean flattenKeyValues; - - public PolarisJsonLayout( - JsonFormatter jsonFormatter, - TimestampFormatter timestampFormatter, - ThrowableHandlingConverter throwableProxyConverter, - Set includes, - Map customFieldNames, - Map additionalFields, - Set includesMdcKeys, - boolean flattenMdc, - boolean includeKeyValues, - boolean flattenKeyValues) { - super( - jsonFormatter, - timestampFormatter, - throwableProxyConverter, - includes, - customFieldNames, - additionalFields, - includesMdcKeys, - flattenMdc); - this.includeKeyValues = includeKeyValues; - this.flattenKeyValues = flattenKeyValues; - } - - @Override - protected Map toJsonMap(ILoggingEvent event) { - Map jsonMap = super.toJsonMap(event); - if (!includeKeyValues) { - return jsonMap; - } - Map keyValueMap = - event.getKeyValuePairs() == null - ? Map.of() - : event.getKeyValuePairs().stream() - .collect(Collectors.toMap(kv -> kv.key, kv -> kv.value)); - ImmutableMap.Builder builder = - ImmutableMap.builder().putAll(jsonMap); - if (flattenKeyValues) { - builder.putAll(keyValueMap); - } else { - builder.put("params", keyValueMap); - } - return builder.build(); - } - } - - protected ThrowableHandlingConverter createThrowableProxyConverter(LoggerContext context) { - if (exceptionFormat == null) { - return new RootCauseFirstThrowableProxyConverter(); - } - - ThrowableHandlingConverter throwableHandlingConverter; - if (exceptionFormat.isRootFirst()) { - throwableHandlingConverter = new RootCauseFirstThrowableProxyConverter(); - } else { - throwableHandlingConverter = new ExtendedThrowableProxyConverter(); - } - - List options = new ArrayList<>(); - // depth must be added first - options.add(exceptionFormat.getDepth()); - options.addAll(exceptionFormat.getEvaluators()); - - throwableHandlingConverter.setOptionList(options); - throwableHandlingConverter.setContext(context); - - return throwableHandlingConverter; - } -} diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java index a268fd6a6..d154e530f 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java @@ -18,7 +18,11 @@ */ package org.apache.polaris.service.persistence; -import com.fasterxml.jackson.annotation.JsonTypeName; +import io.quarkus.arc.lookup.LookupIfProperty; +import io.quarkus.runtime.Startup; +import jakarta.annotation.Nonnull; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; import java.util.Collections; import java.util.HashSet; import java.util.Map; @@ -26,27 +30,44 @@ import java.util.function.Supplier; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.auth.PolarisSecretsManager.PrincipalSecretsResult; +import org.apache.polaris.core.config.RuntimeCandidate; import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.core.persistence.LocalPolarisMetaStoreManagerFactory; -import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.PolarisMetaStoreSession; -import org.apache.polaris.core.persistence.PolarisTreeMapMetaStoreSessionImpl; -import org.apache.polaris.core.persistence.PolarisTreeMapStore; -import org.jetbrains.annotations.NotNull; +import org.apache.polaris.core.persistence.*; +import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; +import org.apache.polaris.service.context.RealmContextResolver; -@JsonTypeName("in-memory") +@ApplicationScoped +@RuntimeCandidate +@LookupIfProperty(name = "polaris.persistence.metastore-manager.type", stringValue = "in-memory") public class InMemoryPolarisMetaStoreManagerFactory extends LocalPolarisMetaStoreManagerFactory { - final Set bootstrappedRealms = new HashSet<>(); + + private final Set bootstrappedRealms = new HashSet<>(); + private final RealmContextResolver realmContextResolver; + + @Inject + public InMemoryPolarisMetaStoreManagerFactory( + PolarisStorageIntegrationProvider storageIntegration, + RealmContextResolver realmContextResolver) { + this.storageIntegration = storageIntegration; + this.realmContextResolver = realmContextResolver; + } + + @Startup + public void init() { + // For in-memory metastore we need to bootstrap Service and Service principal at startup + // (for default realm) + getOrCreateMetaStoreManager(realmContextResolver::getDefaultRealm); + } @Override - protected PolarisTreeMapStore createBackingStore(@NotNull PolarisDiagnostics diagnostics) { + protected PolarisTreeMapStore createBackingStore(@Nonnull PolarisDiagnostics diagnostics) { return new PolarisTreeMapStore(diagnostics); } @Override protected PolarisMetaStoreSession createMetaStoreSession( - @NotNull PolarisTreeMapStore store, @NotNull RealmContext realmContext) { + @Nonnull PolarisTreeMapStore store, @Nonnull RealmContext realmContext) { return new PolarisTreeMapMetaStoreSessionImpl(store, storageIntegration); } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/NoOpRateLimiter.java b/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/NoOpRateLimiter.java index 33dffd7b2..f0cce0708 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/NoOpRateLimiter.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/NoOpRateLimiter.java @@ -18,10 +18,14 @@ */ package org.apache.polaris.service.ratelimiter; -import com.fasterxml.jackson.annotation.JsonTypeName; +import io.quarkus.arc.lookup.LookupIfProperty; +import jakarta.enterprise.context.ApplicationScoped; +import org.apache.polaris.core.config.RuntimeCandidate; /** Rate limiter that always allows the request */ -@JsonTypeName("no-op") +@ApplicationScoped +@RuntimeCandidate +@LookupIfProperty(name = "polaris.rate-limiter.type", stringValue = "no-op") public class NoOpRateLimiter implements RateLimiter { @Override public boolean tryAcquire() { diff --git a/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiter.java b/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiter.java index 5d102b628..be2017d32 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiter.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiter.java @@ -18,12 +18,8 @@ */ package org.apache.polaris.service.ratelimiter; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import io.dropwizard.jackson.Discoverable; - /** Interface for rate limiting requests */ -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") -public interface RateLimiter extends Discoverable { +public interface RateLimiter { /** * This signifies that a request is being made. That is, the rate limiter should count the request * at this point. diff --git a/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiterFilter.java b/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiterFilter.java index 034717c4c..725aa8734 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiterFilter.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiterFilter.java @@ -18,30 +18,34 @@ */ package org.apache.polaris.service.ratelimiter; +import jakarta.annotation.Priority; +import jakarta.inject.Inject; +import jakarta.ws.rs.Priorities; import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.container.ContainerRequestFilter; import jakarta.ws.rs.core.Response; -import java.io.IOException; -import javax.ws.rs.ext.Provider; +import jakarta.ws.rs.ext.Provider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** Request filter that returns a 429 Too Many Requests if the rate limiter says so */ +@Priority(Priorities.AUTHORIZATION + 1) @Provider public class RateLimiterFilter implements ContainerRequestFilter { private static final Logger LOGGER = LoggerFactory.getLogger(RateLimiterFilter.class); private final RateLimiter rateLimiter; + @Inject public RateLimiterFilter(RateLimiter rateLimiter) { this.rateLimiter = rateLimiter; } /** Returns a 429 if the rate limiter says so. Otherwise, forwards the request along. */ @Override - public void filter(ContainerRequestContext ctx) throws IOException { + public void filter(ContainerRequestContext containerRequestContext) { if (!rateLimiter.tryAcquire()) { - ctx.abortWith(Response.status(Response.Status.TOO_MANY_REQUESTS).build()); + containerRequestContext.abortWith(Response.status(Response.Status.TOO_MANY_REQUESTS).build()); LOGGER.atDebug().log("Rate limiting request"); } } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/RealmTokenBucketRateLimiter.java b/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/RealmTokenBucketRateLimiter.java index bc45153f8..c1162ce16 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/RealmTokenBucketRateLimiter.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/RealmTokenBucketRateLimiter.java @@ -18,34 +18,41 @@ */ package org.apache.polaris.service.ratelimiter; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; +import io.quarkus.arc.lookup.LookupIfProperty; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; import java.time.Clock; +import java.time.Duration; import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; +import org.apache.polaris.core.config.RuntimeCandidate; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; -import org.jetbrains.annotations.VisibleForTesting; +import org.eclipse.microprofile.config.inject.ConfigProperty; /** * Rate limiter that maps the request's realm identifier to its own TokenBucketRateLimiter, with its * own capacity. */ -@JsonTypeName("realm-token-bucket") +@ApplicationScoped +@RuntimeCandidate +@LookupIfProperty(name = "polaris.rate-limiter.type", stringValue = "realm-token-bucket") public class RealmTokenBucketRateLimiter implements RateLimiter { private final long requestsPerSecond; - private final long windowSeconds; + private final Duration window; private final Map perRealmLimiters; + private final Clock clock; - @VisibleForTesting - @JsonCreator + @Inject public RealmTokenBucketRateLimiter( - @JsonProperty("requestsPerSecond") final long requestsPerSecond, - @JsonProperty("windowSeconds") final long windowSeconds) { + @ConfigProperty(name = "polaris.rate-limiter.realm-token-bucket.requests-per-second") + long requestsPerSecond, + @ConfigProperty(name = "polaris.rate-limiter.realm-token-bucket.window") Duration window, + Clock clock) { this.requestsPerSecond = requestsPerSecond; - this.windowSeconds = windowSeconds; + this.window = window; + this.clock = clock; this.perRealmLimiters = new ConcurrentHashMap<>(); } @@ -69,13 +76,8 @@ public boolean tryAcquire() { (k) -> new TokenBucketRateLimiter( requestsPerSecond, - Math.multiplyExact(requestsPerSecond, windowSeconds), - getClock())) + Math.multiplyExact(requestsPerSecond, window.getSeconds()), + clock)) .tryAcquire(); } - - @VisibleForTesting - protected Clock getClock() { - return Clock.systemUTC(); - } } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/TokenBucketRateLimiter.java b/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/TokenBucketRateLimiter.java index 39a8aea87..65e635f48 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/TokenBucketRateLimiter.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/TokenBucketRateLimiter.java @@ -18,12 +18,14 @@ */ package org.apache.polaris.service.ratelimiter; +import jakarta.enterprise.inject.Vetoed; import java.time.InstantSource; /** * Token bucket implementation of a Polaris RateLimiter. Acquires tokens at a fixed rate and has a * maximum amount of tokens. Each successful "tryAcquire" costs 1 token. */ +@Vetoed public class TokenBucketRateLimiter implements RateLimiter { private final double tokensPerMilli; private final long maxTokens; diff --git a/polaris-service/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java b/polaris-service/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java index b3252a299..52917dddf 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java @@ -20,11 +20,17 @@ import com.google.api.client.http.javanet.NetHttpTransport; import com.google.auth.http.HttpTransportFactory; +import com.google.auth.oauth2.AccessToken; import com.google.auth.oauth2.GoogleCredentials; import com.google.cloud.ServiceOptions; -import java.util.EnumMap; -import java.util.Map; -import java.util.Set; +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.*; import java.util.function.Supplier; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.storage.PolarisCredentialProperty; @@ -35,15 +41,61 @@ import org.apache.polaris.core.storage.aws.AwsCredentialsStorageIntegration; import org.apache.polaris.core.storage.azure.AzureCredentialsStorageIntegration; import org.apache.polaris.core.storage.gcp.GcpCredentialsStorageIntegration; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; +import org.eclipse.microprofile.config.inject.ConfigProperty; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.services.sts.StsClientBuilder; +@ApplicationScoped public class PolarisStorageIntegrationProviderImpl implements PolarisStorageIntegrationProvider { + private static final Logger LOGGER = + LoggerFactory.getLogger(PolarisStorageIntegrationProviderImpl.class); + private final Supplier stsClientSupplier; private final Supplier gcpCredsProvider; + @Inject + public PolarisStorageIntegrationProviderImpl( + @ConfigProperty(name = "polaris.storage.aws.awsAccessKey") Optional awsAccessKey, + @ConfigProperty(name = "polaris.storage.aws.awsSecretKey") Optional awsSecretKey, + @ConfigProperty(name = "polaris.storage.gcp.token") Optional gcpAccessToken, + @ConfigProperty(name = "polaris.storage.gcp.lifespan") Optional lifespan) { + // TODO clean up this constructor, use bean injection with qualifier for configuration for each + // provider (AWS, GCP, ...) + this( + () -> { + StsClientBuilder stsClientBuilder = StsClient.builder(); + if (!awsAccessKey.get().isBlank() && !awsSecretKey.get().isBlank()) { + LOGGER.warn( + "Using hard-coded AWS credentials - this is not recommended for production"); + StaticCredentialsProvider awsCredentialsProvider = + StaticCredentialsProvider.create( + AwsBasicCredentials.create(awsAccessKey.get(), awsSecretKey.get())); + stsClientBuilder.credentialsProvider(awsCredentialsProvider); + } + return stsClientBuilder.build(); + }, + () -> { + if (gcpAccessToken.get().isBlank()) { + try { + return GoogleCredentials.getApplicationDefault(); + } catch (IOException e) { + throw new RuntimeException("Failed to get GCP credentials", e); + } + } else { + AccessToken accessToken = + new AccessToken( + gcpAccessToken.get(), + new Date(Instant.now().plus(lifespan.get()).toEpochMilli())); + return GoogleCredentials.create(accessToken); + } + }); + } + public PolarisStorageIntegrationProviderImpl( Supplier stsClientSupplier, Supplier gcpCredsProvider) { this.stsClientSupplier = stsClientSupplier; @@ -82,20 +134,20 @@ PolarisStorageIntegration getStorageIntegrationForConfig( new PolarisStorageIntegration<>("file") { @Override public EnumMap getSubscopedCreds( - @NotNull PolarisDiagnostics diagnostics, - @NotNull T storageConfig, + @Nonnull PolarisDiagnostics diagnostics, + @Nonnull T storageConfig, boolean allowListOperation, - @NotNull Set allowedReadLocations, - @NotNull Set allowedWriteLocations) { + @Nonnull Set allowedReadLocations, + @Nonnull Set allowedWriteLocations) { return new EnumMap<>(PolarisCredentialProperty.class); } @Override - public @NotNull Map> + public @Nonnull Map> validateAccessToLocations( - @NotNull T storageConfig, - @NotNull Set actions, - @NotNull Set locations) { + @Nonnull T storageConfig, + @Nonnull Set actions, + @Nonnull Set locations) { return Map.of(); } }; diff --git a/polaris-service/src/main/java/org/apache/polaris/service/task/TaskExecutorImpl.java b/polaris-service/src/main/java/org/apache/polaris/service/task/TaskExecutorImpl.java index e3a227f4e..748bcadd2 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/task/TaskExecutorImpl.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/task/TaskExecutorImpl.java @@ -18,12 +18,17 @@ */ package org.apache.polaris.service.task; +import io.quarkus.runtime.Startup; +import jakarta.annotation.Nonnull; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.PolarisBaseEntity; @@ -32,25 +37,35 @@ import org.apache.polaris.core.entity.TaskEntity; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.jetbrains.annotations.NotNull; +import org.apache.polaris.service.config.TaskHandlerConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * Given a list of registered {@link TaskHandler}s, execute tasks asynchronously with the provided - * {@link CallContext}. - */ +@ApplicationScoped public class TaskExecutorImpl implements TaskExecutor { private static final Logger LOGGER = LoggerFactory.getLogger(TaskExecutorImpl.class); public static final long TASK_RETRY_DELAY = 1000; private final ExecutorService executorService; private final MetaStoreManagerFactory metaStoreManagerFactory; + private final TaskFileIOSupplier fileIOSupplier; private final List taskHandlers = new ArrayList<>(); + @Inject public TaskExecutorImpl( - ExecutorService executorService, MetaStoreManagerFactory metaStoreManagerFactory) { - this.executorService = executorService; + TaskHandlerConfiguration taskHandlerConfiguration, + MetaStoreManagerFactory metaStoreManagerFactory, + TaskFileIOSupplier fileIOSupplier) { + this.executorService = taskHandlerConfiguration.executorService(); this.metaStoreManagerFactory = metaStoreManagerFactory; + this.fileIOSupplier = fileIOSupplier; + } + + @Startup + public void init() { + addTaskHandler(new TableCleanupTaskHandler(this, metaStoreManagerFactory, fileIOSupplier)); + addTaskHandler( + new ManifestFileCleanupTaskHandler( + fileIOSupplier, Executors.newVirtualThreadPerTaskExecutor())); } /** @@ -72,7 +87,7 @@ public void addTaskHandlerContext(long taskEntityId, CallContext callContext) { tryHandleTask(taskEntityId, clone, null, 1); } - private @NotNull CompletableFuture tryHandleTask( + private @Nonnull CompletableFuture tryHandleTask( long taskEntityId, CallContext clone, Throwable e, int attempt) { if (attempt > 3) { return CompletableFuture.failedFuture(e); diff --git a/polaris-service/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java b/polaris-service/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java index c84eebd9f..e63c2e987 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java @@ -18,6 +18,8 @@ */ package org.apache.polaris.service.task; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; import java.util.HashMap; import java.util.Map; import java.util.Set; @@ -31,10 +33,12 @@ import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.service.catalog.io.FileIOFactory; +@ApplicationScoped public class TaskFileIOSupplier implements Function { private final MetaStoreManagerFactory metaStoreManagerFactory; private final FileIOFactory fileIOFactory; + @Inject public TaskFileIOSupplier( MetaStoreManagerFactory metaStoreManagerFactory, FileIOFactory fileIOFactory) { this.metaStoreManagerFactory = metaStoreManagerFactory; diff --git a/polaris-service/src/main/java/org/apache/polaris/service/throttling/RequestThrottlingErrorResponse.java b/polaris-service/src/main/java/org/apache/polaris/service/throttling/RequestThrottlingErrorResponse.java deleted file mode 100644 index 137875341..000000000 --- a/polaris-service/src/main/java/org/apache/polaris/service/throttling/RequestThrottlingErrorResponse.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.throttling; - -import com.fasterxml.jackson.annotation.JsonProperty; - -/** - * Response object for errors caused by DoS-prevention throttling mechanisms, such as request size - * limits - */ -public record RequestThrottlingErrorResponse( - @JsonProperty("error_type") RequestThrottlingErrorType errorType) { - public enum RequestThrottlingErrorType { - REQUEST_TOO_LARGE, - ; - } -} diff --git a/polaris-service/src/main/java/org/apache/polaris/service/throttling/StreamReadConstraintsExceptionMapper.java b/polaris-service/src/main/java/org/apache/polaris/service/throttling/StreamReadConstraintsExceptionMapper.java deleted file mode 100644 index 27a4f07e3..000000000 --- a/polaris-service/src/main/java/org/apache/polaris/service/throttling/StreamReadConstraintsExceptionMapper.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.throttling; - -import static org.apache.polaris.service.throttling.RequestThrottlingErrorResponse.RequestThrottlingErrorType.REQUEST_TOO_LARGE; - -import com.fasterxml.jackson.core.exc.StreamConstraintsException; -import jakarta.ws.rs.core.MediaType; -import jakarta.ws.rs.core.Response; -import jakarta.ws.rs.ext.ExceptionMapper; - -/** - * Handles exceptions during the request that are a result of stream constraints such as the request - * being too large - */ -public class StreamReadConstraintsExceptionMapper - implements ExceptionMapper { - - @Override - public Response toResponse(StreamConstraintsException exception) { - return Response.status(Response.Status.BAD_REQUEST) - .type(MediaType.APPLICATION_JSON_TYPE) - .entity(new RequestThrottlingErrorResponse(REQUEST_TOO_LARGE)) - .build(); - } -} diff --git a/polaris-service/src/main/java/org/apache/polaris/service/tracing/HeadersMapAccessor.java b/polaris-service/src/main/java/org/apache/polaris/service/tracing/HeadersMapAccessor.java deleted file mode 100644 index 44f52ffb0..000000000 --- a/polaris-service/src/main/java/org/apache/polaris/service/tracing/HeadersMapAccessor.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.tracing; - -import io.opentelemetry.context.propagation.TextMapGetter; -import io.opentelemetry.context.propagation.TextMapSetter; -import jakarta.servlet.http.HttpServletRequest; -import java.net.http.HttpRequest; -import java.util.Spliterator; -import java.util.Spliterators; -import java.util.stream.StreamSupport; -import org.jetbrains.annotations.Nullable; - -/** - * Implementation of {@link TextMapSetter} and {@link TextMapGetter} that can handle an {@link - * HttpServletRequest} for extracting headers and sets headers on a {@link HttpRequest.Builder}. - */ -public class HeadersMapAccessor - implements TextMapSetter, TextMapGetter { - @Override - public Iterable keys(HttpServletRequest carrier) { - return StreamSupport.stream( - Spliterators.spliteratorUnknownSize( - carrier.getHeaderNames().asIterator(), Spliterator.IMMUTABLE), - false) - .toList(); - } - - @Nullable - @Override - public String get(@Nullable HttpServletRequest carrier, String key) { - return carrier == null ? null : carrier.getHeader(key); - } - - @Override - public void set(@Nullable HttpRequest.Builder carrier, String key, String value) { - if (carrier != null) { - carrier.setHeader(key, value); - } - } -} diff --git a/polaris-service/src/main/java/org/apache/polaris/service/tracing/OpenTelemetryAware.java b/polaris-service/src/main/java/org/apache/polaris/service/tracing/OpenTelemetryAware.java deleted file mode 100644 index 07a24e29c..000000000 --- a/polaris-service/src/main/java/org/apache/polaris/service/tracing/OpenTelemetryAware.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.tracing; - -import io.opentelemetry.api.OpenTelemetry; - -/** Allows setting a configured instance of {@link OpenTelemetry} */ -public interface OpenTelemetryAware { - void setOpenTelemetry(OpenTelemetry openTelemetry); -} diff --git a/polaris-service/src/main/java/org/apache/polaris/service/tracing/TracingFilter.java b/polaris-service/src/main/java/org/apache/polaris/service/tracing/TracingFilter.java deleted file mode 100644 index b3cefe1cb..000000000 --- a/polaris-service/src/main/java/org/apache/polaris/service/tracing/TracingFilter.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.tracing; - -import io.opentelemetry.api.OpenTelemetry; -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.api.trace.SpanKind; -import io.opentelemetry.api.trace.Tracer; -import io.opentelemetry.context.Context; -import io.opentelemetry.context.Scope; -import io.opentelemetry.semconv.HttpAttributes; -import io.opentelemetry.semconv.ServerAttributes; -import io.opentelemetry.semconv.UrlAttributes; -import jakarta.annotation.Priority; -import jakarta.servlet.Filter; -import jakarta.servlet.FilterChain; -import jakarta.servlet.ServletException; -import jakarta.servlet.ServletRequest; -import jakarta.servlet.ServletResponse; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.ws.rs.Priorities; -import java.io.IOException; -import org.apache.polaris.core.context.CallContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.slf4j.MDC; - -/** - * Servlet {@link Filter} that starts an OpenTracing {@link Span}, propagating the calling context - * from HTTP headers, if present. "spanId" and "traceId" are added to the logging MDC so that all - * logs recorded in the request will contain the current span and trace id. Downstream HTTP calls - * should use the OpenTelemetry {@link io.opentelemetry.context.propagation.ContextPropagators} to - * include the current trace id in the request headers. - */ -@Priority(Priorities.AUTHENTICATION - 1) -public class TracingFilter implements Filter { - private static final Logger LOGGER = LoggerFactory.getLogger(TracingFilter.class); - private final OpenTelemetry openTelemetry; - - public TracingFilter(OpenTelemetry openTelemetry) { - this.openTelemetry = openTelemetry; - } - - @Override - public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) - throws IOException, ServletException { - HttpServletRequest httpRequest = (HttpServletRequest) request; - Context extractedContext = - openTelemetry - .getPropagators() - .getTextMapPropagator() - .extract(Context.current(), httpRequest, new HeadersMapAccessor()); - try (Scope scope = extractedContext.makeCurrent()) { - Tracer tracer = openTelemetry.getTracer(httpRequest.getPathInfo()); - Span span = - tracer - .spanBuilder(httpRequest.getMethod() + " " + httpRequest.getPathInfo()) - .setSpanKind(SpanKind.SERVER) - .setAttribute( - "realm", CallContext.getCurrentContext().getRealmContext().getRealmIdentifier()) - .startSpan(); - - try (Scope ignored = span.makeCurrent(); - MDC.MDCCloseable spanId = MDC.putCloseable("spanId", span.getSpanContext().getSpanId()); - MDC.MDCCloseable traceId = - MDC.putCloseable("traceId", span.getSpanContext().getTraceId())) { - LOGGER - .atInfo() - .addKeyValue("spanId", span.getSpanContext().getSpanId()) - .addKeyValue("traceId", span.getSpanContext().getTraceId()) - .addKeyValue("parentContext", extractedContext) - .log("Started span with parent"); - span.setAttribute(HttpAttributes.HTTP_REQUEST_METHOD, httpRequest.getMethod()); - span.setAttribute(ServerAttributes.SERVER_ADDRESS, httpRequest.getServerName()); - span.setAttribute(UrlAttributes.URL_SCHEME, httpRequest.getScheme()); - span.setAttribute(UrlAttributes.URL_PATH, httpRequest.getPathInfo()); - - chain.doFilter(request, response); - } finally { - span.end(); - } - } - } -} diff --git a/polaris-service/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable b/polaris-service/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable deleted file mode 100644 index 95d1f8ec7..000000000 --- a/polaris-service/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable +++ /dev/null @@ -1,27 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -org.apache.polaris.service.auth.DiscoverableAuthenticator -org.apache.polaris.core.persistence.MetaStoreManagerFactory -org.apache.polaris.service.config.OAuth2ApiService -org.apache.polaris.service.context.RealmContextResolver -org.apache.polaris.service.context.CallContextResolver -org.apache.polaris.service.auth.TokenBrokerFactory -org.apache.polaris.service.catalog.io.FileIOFactory -org.apache.polaris.service.ratelimiter.RateLimiter diff --git a/polaris-service/src/main/resources/META-INF/services/io.dropwizard.logging.common.layout.DiscoverableLayoutFactory b/polaris-service/src/main/resources/META-INF/services/io.dropwizard.logging.common.layout.DiscoverableLayoutFactory deleted file mode 100644 index aa766ac70..000000000 --- a/polaris-service/src/main/resources/META-INF/services/io.dropwizard.logging.common.layout.DiscoverableLayoutFactory +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -org.apache.polaris.service.logging.PolarisJsonLayoutFactory \ No newline at end of file diff --git a/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.core.persistence.MetaStoreManagerFactory b/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.core.persistence.MetaStoreManagerFactory deleted file mode 100644 index 85ae92caf..000000000 --- a/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.core.persistence.MetaStoreManagerFactory +++ /dev/null @@ -1,21 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory -org.apache.polaris.extension.persistence.impl.eclipselink.EclipseLinkPolarisMetaStoreManagerFactory diff --git a/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.auth.TokenBrokerFactory b/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.auth.TokenBrokerFactory deleted file mode 100644 index 422b154c7..000000000 --- a/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.auth.TokenBrokerFactory +++ /dev/null @@ -1,21 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -org.apache.polaris.service.auth.JWTRSAKeyPairFactory -org.apache.polaris.service.auth.JWTSymmetricKeyFactory \ No newline at end of file diff --git a/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.catalog.io.FileIOFactory b/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.catalog.io.FileIOFactory deleted file mode 100644 index 6b280ad71..000000000 --- a/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.catalog.io.FileIOFactory +++ /dev/null @@ -1,21 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -org.apache.polaris.service.catalog.io.DefaultFileIOFactory -org.apache.polaris.service.catalog.io.WasbTranslatingFileIOFactory diff --git a/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.config.OAuth2ApiService b/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.config.OAuth2ApiService deleted file mode 100644 index 3c8f0e254..000000000 --- a/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.config.OAuth2ApiService +++ /dev/null @@ -1,21 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -org.apache.polaris.service.auth.TestOAuth2ApiService -org.apache.polaris.service.auth.DefaultOAuth2ApiService \ No newline at end of file diff --git a/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.context.CallContextResolver b/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.context.CallContextResolver deleted file mode 100644 index 1ac9dbea2..000000000 --- a/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.context.CallContextResolver +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -org.apache.polaris.service.context.DefaultContextResolver \ No newline at end of file diff --git a/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.context.RealmContextResolver b/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.context.RealmContextResolver deleted file mode 100644 index 1ac9dbea2..000000000 --- a/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.context.RealmContextResolver +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -org.apache.polaris.service.context.DefaultContextResolver \ No newline at end of file diff --git a/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.ratelimiter.RateLimiter b/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.ratelimiter.RateLimiter deleted file mode 100644 index 461bcc2db..000000000 --- a/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.ratelimiter.RateLimiter +++ /dev/null @@ -1,21 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -org.apache.polaris.service.ratelimiter.RealmTokenBucketRateLimiter -org.apache.polaris.service.ratelimiter.NoOpRateLimiter diff --git a/polaris-service-quarkus/src/main/resources/application.properties b/polaris-service/src/main/resources/application.properties similarity index 100% rename from polaris-service-quarkus/src/main/resources/application.properties rename to polaris-service/src/main/resources/application.properties diff --git a/polaris-service/src/main/resources/log4j.properties b/polaris-service/src/main/resources/log4j.properties deleted file mode 100644 index 37c1696bd..000000000 --- a/polaris-service/src/main/resources/log4j.properties +++ /dev/null @@ -1,24 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -log4j.rootLogger=INFO, stdout -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.Target=System.out -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd'T'HH:mm:ss.SSS} %-5p [%c] - %m%n diff --git a/polaris-service/src/main/resources/org/apache/polaris/service/banner.txt b/polaris-service/src/main/resources/org/apache/polaris/service/banner.txt deleted file mode 100644 index 5c615a243..000000000 --- a/polaris-service/src/main/resources/org/apache/polaris/service/banner.txt +++ /dev/null @@ -1,20 +0,0 @@ - - @@@@ @@@ @ @ @@@@ @ @@@@ @@@@ @ @@@@@ @ @ @@@ @@@@ - @ @ @ @ @ @ @ @ @ @ @@ @ @ @ @ @ @ @ @ @ @ - @@@@ @ @ @ @@@@@ @@@@ @ @@ @ @@@@@ @ @@@@@ @ @ @ @ @@@ - @ @@@ @@@@ @ @ @ @@ @ @@@@ @@@@ @ @ @ @@ @@ @@@@ @@@ @@@@ - - * - - - /////\ - //\\///T\\\ - ///\\\////\\\\\\ - //\\\\T////\\\\\\\\\ - /T\ //\\\\\T///T\\//T\\\\\\ - //\\\/////T\\////\\/////\\\\\\\ //\\ - //\\\\\\T///////////////////T\\\\\\\T\\\\\ - //\\\\/////T\//////////\///////T\\\\\T\\\\\\\\ - //\\\\\/////\\\T////////////////\\\\\\/\\\\\\\\\ -,,..,,,..,,,..,//\\\\////////\\\\\\\\\\/////////\\\\\///\\\\\\\\\\,,,..,,..,,,..,,,. -,,..,,,..,,,..,,,..,,,..,,,..,,,..,,,..,,,..,,,..,,,..,,,..,,,..,,,..,,,.,,,..,,,.., diff --git a/polaris-service-quarkus/src/main/resources/polaris-banner.txt b/polaris-service/src/main/resources/polaris-banner.txt similarity index 100% rename from polaris-service-quarkus/src/main/resources/polaris-banner.txt rename to polaris-service/src/main/resources/polaris-banner.txt diff --git a/polaris-service/src/test/java/org/apache/polaris/service/PolarisApplicationIntegrationTest.java b/polaris-service/src/test/java/org/apache/polaris/service/PolarisApplicationIntegrationTest.java index 465b34d51..01d8abf47 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/PolarisApplicationIntegrationTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/PolarisApplicationIntegrationTest.java @@ -18,27 +18,24 @@ */ package org.apache.polaris.service; -import static org.apache.polaris.service.context.DefaultContextResolver.REALM_PROPERTY_KEY; -import static org.apache.polaris.service.throttling.RequestThrottlingErrorResponse.RequestThrottlingErrorType.REQUEST_TOO_LARGE; +import static org.apache.polaris.service.auth.BasePolarisAuthenticator.PRINCIPAL_ROLE_ALL; +import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; +import io.quarkus.test.junit.QuarkusTest; +import jakarta.inject.Inject; import jakarta.ws.rs.ProcessingException; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.client.Invocation; import jakarta.ws.rs.core.Response; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.util.List; import java.util.Map; -import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.BaseTable; import org.apache.iceberg.PartitionData; @@ -63,7 +60,6 @@ import org.apache.iceberg.rest.RESTClient; import org.apache.iceberg.rest.RESTSessionCatalog; import org.apache.iceberg.rest.auth.AuthConfig; -import org.apache.iceberg.rest.auth.ImmutableAuthConfig; import org.apache.iceberg.rest.auth.OAuth2Properties; import org.apache.iceberg.rest.auth.OAuth2Util; import org.apache.iceberg.types.Types; @@ -79,13 +75,7 @@ import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.PolarisEntityConstants; -import org.apache.polaris.service.auth.BasePolarisAuthenticator; -import org.apache.polaris.service.config.PolarisApplicationConfig; -import org.apache.polaris.service.test.PolarisConnectionExtension; -import org.apache.polaris.service.test.PolarisRealm; -import org.apache.polaris.service.test.SnowmanCredentialsExtension; -import org.apache.polaris.service.test.TestEnvironmentExtension; -import org.apache.polaris.service.throttling.RequestThrottlingErrorResponse; +import org.apache.polaris.service.test.PolarisIntegrationTestHelper; import org.assertj.core.api.Assertions; import org.assertj.core.api.InstanceOfAssertFactories; import org.junit.jupiter.api.AfterAll; @@ -93,74 +83,50 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; -import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.io.TempDir; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; - -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class, - SnowmanCredentialsExtension.class -}) + +@QuarkusTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) public class PolarisApplicationIntegrationTest { + private static final Logger LOGGER = LoggerFactory.getLogger(PolarisApplicationIntegrationTest.class); public static final String PRINCIPAL_ROLE_NAME = "admin"; - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config( - "server.adminConnectors[0].port", "0")); // Bind to random port to support parallelism - - private static String userToken; - private static SnowmanCredentialsExtension.SnowmanCredentials snowmanCredentials; - private static Path testDir; - private static String realm; - - @BeforeAll - public static void setup( - PolarisConnectionExtension.PolarisToken userToken, - SnowmanCredentialsExtension.SnowmanCredentials snowmanCredentials, - @PolarisRealm String polarisRealm) - throws IOException { - realm = polarisRealm; - testDir = Path.of("build/test_data/iceberg/" + realm); - FileUtils.deleteQuietly(testDir.toFile()); - Files.createDirectories(testDir); - PolarisApplicationIntegrationTest.userToken = userToken.token(); - PolarisApplicationIntegrationTest.snowmanCredentials = snowmanCredentials; + @Inject PolarisIntegrationTestHelper testHelper; + @BeforeAll + public void setUp(TestInfo testInfo) { + testHelper.setUp(testInfo); PrincipalRole principalRole = new PrincipalRole(PRINCIPAL_ROLE_NAME); try (Response createPrResponse = - EXT.client() + testHelper + .client .target( String.format( - "http://localhost:%d/api/management/v1/principal-roles", EXT.getLocalPort())) + "http://localhost:%d/api/management/v1/principal-roles", testHelper.localPort)) .request("application/json") - .header("Authorization", "Bearer " + userToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + testHelper.adminToken) + .header(REALM_PROPERTY_KEY, testHelper.realm) .post(Entity.json(principalRole))) { assertThat(createPrResponse) .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } try (Response assignPrResponse = - EXT.client() + testHelper + .client .target( String.format( - "http://localhost:%d/api/management/v1/principals/%s/principal-roles", - EXT.getLocalPort(), snowmanCredentials.identifier().principalName())) + "http://localhost:%d/api/management/v1/principals/snowman/principal-roles", + testHelper.localPort)) .request("application/json") - .header("Authorization", "Bearer " + PolarisApplicationIntegrationTest.userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + testHelper.adminToken) + .header(REALM_PROPERTY_KEY, testHelper.realm) .put(Entity.json(principalRole))) { assertThat(assignPrResponse) .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -168,17 +134,19 @@ public static void setup( } @AfterAll - public static void deletePrincipalRole() { - EXT.client() + public void tearDown() { + testHelper + .client .target( String.format( "http://localhost:%d/api/management/v1/principal-roles/%s", - EXT.getLocalPort(), PRINCIPAL_ROLE_NAME)) + testHelper.localPort, PRINCIPAL_ROLE_NAME)) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + testHelper.adminToken) + .header(REALM_PROPERTY_KEY, testHelper.realm) .delete() .close(); + testHelper.tearDown(); } /** @@ -188,7 +156,7 @@ public static void deletePrincipalRole() { * @param testInfo */ @BeforeEach - public void before(TestInfo testInfo) { + public void createTestCatalog(TestInfo testInfo) { testInfo .getTestMethod() .ifPresent( @@ -199,7 +167,7 @@ public void before(TestInfo testInfo) { }); } - private static void createCatalog( + private void createCatalog( String catalogName, Catalog.TypeEnum catalogType, String principalRoleName) { createCatalog( catalogName, @@ -214,7 +182,7 @@ private static void createCatalog( "s3://my-bucket/path/to/data"); } - private static void createCatalog( + private void createCatalog( String catalogName, Catalog.TypeEnum catalogType, String principalRoleName, @@ -241,39 +209,43 @@ private static void createCatalog( .setStorageConfigInfo(storageConfig) .build(); try (Response response = - EXT.client() + testHelper + .client .target( - String.format("http://localhost:%d/api/management/v1/catalogs", EXT.getLocalPort())) + String.format( + "http://localhost:%d/api/management/v1/catalogs", testHelper.localPort)) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + testHelper.adminToken) + .header(REALM_PROPERTY_KEY, testHelper.realm) .post(Entity.json(catalog))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } try (Response response = - EXT.client() + testHelper + .client .target( String.format( "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/%s", - EXT.getLocalPort(), + testHelper.localPort, catalogName, PolarisEntityConstants.getNameOfCatalogAdminRole())) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + testHelper.adminToken) + .header(REALM_PROPERTY_KEY, testHelper.realm) .get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); CatalogRole catalogRole = response.readEntity(CatalogRole.class); try (Response assignResponse = - EXT.client() + testHelper + .client .target( String.format( "http://localhost:%d/api/management/v1/principal-roles/%s/catalog-roles/%s", - EXT.getLocalPort(), principalRoleName, catalogName)) + testHelper.localPort, principalRoleName, catalogName)) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + testHelper.adminToken) + .header(REALM_PROPERTY_KEY, testHelper.realm) .put(Entity.json(catalogRole))) { assertThat(assignResponse) .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -281,21 +253,23 @@ private static void createCatalog( } } - private static RESTSessionCatalog newSessionCatalog(String catalog) { + private RESTSessionCatalog newSessionCatalog(String catalog) { RESTSessionCatalog sessionCatalog = new RESTSessionCatalog(); sessionCatalog.initialize( "polaris_catalog_test", Map.of( "uri", - "http://localhost:" + EXT.getLocalPort() + "/api/catalog", + "http://localhost:" + testHelper.localPort + "/api/catalog", OAuth2Properties.CREDENTIAL, - snowmanCredentials.clientId() + ":" + snowmanCredentials.clientSecret(), + testHelper.snowmanCredentials.clientId() + + ":" + + testHelper.snowmanCredentials.clientSecret(), OAuth2Properties.SCOPE, - BasePolarisAuthenticator.PRINCIPAL_ROLE_ALL, + PRINCIPAL_ROLE_ALL, "warehouse", catalog, "header." + REALM_PROPERTY_KEY, - realm)); + testHelper.realm)); return sessionCatalog; } @@ -484,16 +458,17 @@ public void testIcebergCreateTablesWithWritePathBlocked(TestInfo testInfo) throw } @Test - public void testIcebergRegisterTableInExternalCatalog(TestInfo testInfo) throws IOException { + public void testIcebergRegisterTableInExternalCatalog(TestInfo testInfo, @TempDir Path tempDir) + throws IOException { String catalogName = testInfo.getTestMethod().get().getName() + "External"; createCatalog( catalogName, Catalog.TypeEnum.EXTERNAL, PRINCIPAL_ROLE_NAME, FileStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.FILE) - .setAllowedLocations(List.of("file://" + testDir.toFile().getAbsolutePath())) + .setAllowedLocations(List.of("file://" + tempDir.toFile().getAbsolutePath())) .build(), - "file://" + testDir.toFile().getAbsolutePath()); + "file://" + tempDir.toFile().getAbsolutePath()); try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName); HadoopFileIO fileIo = new HadoopFileIO(new Configuration()); ) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); @@ -502,7 +477,7 @@ public void testIcebergRegisterTableInExternalCatalog(TestInfo testInfo) throws TableIdentifier tableIdentifier = TableIdentifier.of(ns, "the_table"); String location = "file://" - + testDir.toFile().getAbsolutePath() + + tempDir.toFile().getAbsolutePath() + "/" + testInfo.getTestMethod().get().getName(); String metadataLocation = location + "/metadata/000001-494949494949494949.metadata.json"; @@ -531,16 +506,17 @@ public void testIcebergRegisterTableInExternalCatalog(TestInfo testInfo) throws } @Test - public void testIcebergUpdateTableInExternalCatalog(TestInfo testInfo) throws IOException { + public void testIcebergUpdateTableInExternalCatalog(TestInfo testInfo, @TempDir Path tempDir) + throws IOException { String catalogName = testInfo.getTestMethod().get().getName() + "External"; createCatalog( catalogName, Catalog.TypeEnum.EXTERNAL, PRINCIPAL_ROLE_NAME, FileStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.FILE) - .setAllowedLocations(List.of("file://" + testDir.toFile().getAbsolutePath())) + .setAllowedLocations(List.of("file://" + tempDir.toFile().getAbsolutePath())) .build(), - "file://" + testDir.toFile().getAbsolutePath()); + "file://" + tempDir.toFile().getAbsolutePath()); try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName); HadoopFileIO fileIo = new HadoopFileIO(new Configuration()); ) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); @@ -549,7 +525,7 @@ public void testIcebergUpdateTableInExternalCatalog(TestInfo testInfo) throws IO TableIdentifier tableIdentifier = TableIdentifier.of(ns, "the_table"); String location = "file://" - + testDir.toFile().getAbsolutePath() + + tempDir.toFile().getAbsolutePath() + "/" + testInfo.getTestMethod().get().getName(); String metadataLocation = location + "/metadata/000001-494949494949494949.metadata.json"; @@ -584,16 +560,17 @@ public void testIcebergUpdateTableInExternalCatalog(TestInfo testInfo) throws IO } @Test - public void testIcebergDropTableInExternalCatalog(TestInfo testInfo) throws IOException { + public void testIcebergDropTableInExternalCatalog(TestInfo testInfo, @TempDir Path tempDir) + throws IOException { String catalogName = testInfo.getTestMethod().get().getName() + "External"; createCatalog( catalogName, Catalog.TypeEnum.EXTERNAL, PRINCIPAL_ROLE_NAME, FileStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.FILE) - .setAllowedLocations(List.of("file://" + testDir.toFile().getAbsolutePath())) + .setAllowedLocations(List.of("file://" + tempDir.toFile().getAbsolutePath())) .build(), - "file://" + testDir.toFile().getAbsolutePath()); + "file://" + tempDir.toFile().getAbsolutePath()); try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName); HadoopFileIO fileIo = new HadoopFileIO(new Configuration()); ) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); @@ -602,7 +579,7 @@ public void testIcebergDropTableInExternalCatalog(TestInfo testInfo) throws IOEx TableIdentifier tableIdentifier = TableIdentifier.of(ns, "the_table"); String location = "file://" - + testDir.toFile().getAbsolutePath() + + tempDir.toFile().getAbsolutePath() + "/" + testInfo.getTestMethod().get().getName(); String metadataLocation = location + "/metadata/000001-494949494949494949.metadata.json"; @@ -639,15 +616,17 @@ public void testWarehouseNotSpecified() throws IOException { "polaris_catalog_test", Map.of( "uri", - "http://localhost:" + EXT.getLocalPort() + "/api/catalog", + "http://localhost:" + testHelper.localPort + "/api/catalog", OAuth2Properties.CREDENTIAL, - snowmanCredentials.clientId() + ":" + snowmanCredentials.clientSecret(), + testHelper.snowmanCredentials.clientId() + + ":" + + testHelper.snowmanCredentials.clientSecret(), OAuth2Properties.SCOPE, - BasePolarisAuthenticator.PRINCIPAL_ROLE_ALL, + PRINCIPAL_ROLE_ALL, "warehouse", emptyEnvironmentVariable, "header." + REALM_PROPERTY_KEY, - realm))) + testHelper.realm))) .isInstanceOf(BadRequestException.class) .hasMessage("Malformed request: Please specify a warehouse"); } @@ -656,10 +635,11 @@ public void testWarehouseNotSpecified() throws IOException { @Test public void testRequestHeaderTooLarge() { Invocation.Builder request = - EXT.client() + testHelper + .client .target( String.format( - "http://localhost:%d/api/management/v1/principal-roles", EXT.getLocalPort())) + "http://localhost:%d/api/management/v1/principal-roles", testHelper.localPort)) .request("application/json"); // The default limit is 8KiB and each of these headers is at least 8 bytes, so 1500 definitely @@ -671,8 +651,8 @@ public void testRequestHeaderTooLarge() { try { try (Response response = request - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + testHelper.adminToken) + .header(REALM_PROPERTY_KEY, testHelper.realm) .post(Entity.json(new PrincipalRole("r")))) { assertThat(response) .returns( @@ -692,54 +672,52 @@ public void testRequestBodyTooLarge() { Entity largeRequest = Entity.json(new PrincipalRole("r".repeat(1000001))); try (Response response = - EXT.client() + testHelper + .client .target( String.format( - "http://localhost:%d/api/management/v1/principal-roles", EXT.getLocalPort())) + "http://localhost:%d/api/management/v1/principal-roles", testHelper.localPort)) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + testHelper.adminToken) + .header(REALM_PROPERTY_KEY, testHelper.realm) .post(largeRequest)) { assertThat(response) - .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus) - .matches( - r -> - r.readEntity(RequestThrottlingErrorResponse.class) - .errorType() - .equals(REQUEST_TOO_LARGE)); + .returns(Response.Status.REQUEST_ENTITY_TOO_LARGE.getStatusCode(), Response::getStatus); } } @Test public void testRefreshToken() throws IOException { String path = - String.format("http://localhost:%d/api/catalog/v1/oauth/tokens", EXT.getLocalPort()); + String.format("http://localhost:%d/api/catalog/v1/oauth/tokens", testHelper.localPort); try (RESTClient client = - HTTPClient.builder(ImmutableMap.of()) - .withHeader(REALM_PROPERTY_KEY, realm) + HTTPClient.builder(Map.of()) + .withHeader(REALM_PROPERTY_KEY, testHelper.realm) .uri(path) .build()) { String credentialString = - snowmanCredentials.clientId() + ":" + snowmanCredentials.clientSecret(); - var authConfig = - AuthConfig.builder().credential(credentialString).scope("PRINCIPAL_ROLE:ALL").build(); - ImmutableAuthConfig configSpy = spy(authConfig); - when(configSpy.expiresAtMillis()).thenReturn(0L); - assertThat(configSpy.expiresAtMillis()).isEqualTo(0L); - when(configSpy.oauth2ServerUri()).thenReturn(path); - - var parentSession = new OAuth2Util.AuthSession(Map.of(), configSpy); + testHelper.snowmanCredentials.clientId() + + ":" + + testHelper.snowmanCredentials.clientSecret(); + AuthConfig configMock = mock(AuthConfig.class); + when(configMock.credential()).thenReturn(credentialString); + when(configMock.scope()).thenReturn(PRINCIPAL_ROLE_ALL); + when(configMock.expiresAtMillis()).thenReturn(0L); + when(configMock.oauth2ServerUri()).thenReturn(path); + + var parentSession = new OAuth2Util.AuthSession(Map.of(), configMock); var session = - OAuth2Util.AuthSession.fromAccessToken(client, null, userToken, 0L, parentSession); + OAuth2Util.AuthSession.fromAccessToken( + client, null, testHelper.adminToken, 0L, parentSession); OAuth2Util.AuthSession sessionSpy = spy(session); when(sessionSpy.expiresAtMillis()).thenReturn(0L); assertThat(sessionSpy.expiresAtMillis()).isEqualTo(0L); - assertThat(sessionSpy.token()).isEqualTo(userToken); + assertThat(sessionSpy.token()).isEqualTo(testHelper.adminToken); sessionSpy.refresh(client); assertThat(sessionSpy.credential()).isNotNull(); - assertThat(sessionSpy.credential()).isNotEqualTo(userToken); + assertThat(sessionSpy.credential()).isNotEqualTo(testHelper.adminToken); } } } diff --git a/polaris-service/src/test/java/org/apache/polaris/service/TimedApplicationEventListenerTest.java b/polaris-service/src/test/java/org/apache/polaris/service/TimedApplicationEventListenerTest.java deleted file mode 100644 index d5369a7af..000000000 --- a/polaris-service/src/test/java/org/apache/polaris/service/TimedApplicationEventListenerTest.java +++ /dev/null @@ -1,234 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service; - -import static org.apache.polaris.core.monitor.PolarisMetricRegistry.*; -import static org.apache.polaris.service.TimedApplicationEventListener.SINGLETON_METRIC_NAME; -import static org.apache.polaris.service.TimedApplicationEventListener.TAG_API_NAME; -import static org.apache.polaris.service.context.DefaultContextResolver.REALM_PROPERTY_KEY; - -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; -import io.micrometer.core.instrument.Tag; -import jakarta.ws.rs.core.Response; -import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import org.apache.polaris.core.monitor.PolarisMetricRegistry; -import org.apache.polaris.core.resource.TimedApi; -import org.apache.polaris.service.admin.api.PolarisPrincipalsApi; -import org.apache.polaris.service.config.PolarisApplicationConfig; -import org.apache.polaris.service.test.PolarisConnectionExtension; -import org.apache.polaris.service.test.PolarisRealm; -import org.apache.polaris.service.test.SnowmanCredentialsExtension; -import org.apache.polaris.service.test.TestEnvironmentExtension; -import org.apache.polaris.service.test.TestMetricsUtil; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; - -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class, - SnowmanCredentialsExtension.class -}) -public class TimedApplicationEventListenerTest { - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config( - "server.adminConnectors[0].port", "0")); // Bind to random port to support parallelism - - private static final int ERROR_CODE = Response.Status.NOT_FOUND.getStatusCode(); - private static final String ENDPOINT = "api/management/v1/principals"; - private static final String API_ANNOTATION = - Arrays.stream(PolarisPrincipalsApi.class.getMethods()) - .filter(m -> m.getName().contains("getPrincipal")) - .findFirst() - .orElseThrow() - .getAnnotation(TimedApi.class) - .value(); - - private static PolarisConnectionExtension.PolarisToken userToken; - private static SnowmanCredentialsExtension.SnowmanCredentials snowmanCredentials; - private static String realm; - - @BeforeAll - public static void setup( - PolarisConnectionExtension.PolarisToken userToken, - SnowmanCredentialsExtension.SnowmanCredentials snowmanCredentials, - @PolarisRealm String realm) - throws IOException { - TimedApplicationEventListenerTest.userToken = userToken; - TimedApplicationEventListenerTest.snowmanCredentials = snowmanCredentials; - TimedApplicationEventListenerTest.realm = realm; - } - - @BeforeEach - public void clearMetrics() { - getPolarisMetricRegistry().clear(); - } - - @Test - public void testMetricsEmittedOnSuccessfulRequest() { - sendSuccessfulRequest(); - Assertions.assertTrue(getPerApiMetricCount() > 0); - Assertions.assertTrue(getPerApiRealmMetricCount() > 0); - Assertions.assertTrue(getCommonMetricCount() > 0); - Assertions.assertTrue(getCommonRealmMetricCount() > 0); - Assertions.assertEquals(0, getPerApiMetricErrorCount()); - Assertions.assertEquals(0, getPerApiRealmMetricErrorCount()); - Assertions.assertEquals(0, getCommonMetricErrorCount()); - Assertions.assertEquals(0, getCommonRealmMetricErrorCount()); - } - - @Test - public void testMetricsEmittedOnFailedRequest() { - sendFailingRequest(); - Assertions.assertTrue(getPerApiMetricCount() > 0); - Assertions.assertTrue(getPerApiRealmMetricCount() > 0); - Assertions.assertTrue(getCommonMetricCount() > 0); - Assertions.assertTrue(getCommonRealmMetricCount() > 0); - Assertions.assertTrue(getPerApiMetricErrorCount() > 0); - Assertions.assertTrue(getPerApiRealmMetricErrorCount() > 0); - Assertions.assertTrue(getCommonMetricErrorCount() > 0); - Assertions.assertTrue(getCommonRealmMetricErrorCount() > 0); - } - - private PolarisMetricRegistry getPolarisMetricRegistry() { - TimedApplicationEventListener listener = - (TimedApplicationEventListener) - EXT.getEnvironment().jersey().getResourceConfig().getSingletons().stream() - .filter( - s -> - TimedApplicationEventListener.class - .getName() - .equals(s.getClass().getName())) - .findAny() - .orElseThrow(); - return listener.getMetricRegistry(); - } - - private double getPerApiMetricCount() { - return TestMetricsUtil.getTotalCounter( - EXT, API_ANNOTATION + SUFFIX_COUNTER, Collections.emptyList()); - } - - private double getPerApiRealmMetricCount() { - return TestMetricsUtil.getTotalCounter( - EXT, - API_ANNOTATION + SUFFIX_COUNTER + SUFFIX_REALM, - List.of( - Tag.of(TAG_REALM, realm), - // spotless:off - Tag.of(TAG_REALM_DEPRECATED, realm))); - // spotless:on - } - - private double getPerApiMetricErrorCount() { - return TestMetricsUtil.getTotalCounter( - EXT, - API_ANNOTATION + SUFFIX_ERROR, - List.of( - Tag.of(TAG_RESP_CODE, String.valueOf(ERROR_CODE)), - // spotless:off - Tag.of(TAG_RESP_CODE_DEPRECATED, String.valueOf(ERROR_CODE)))); - // spotless:on - } - - private double getPerApiRealmMetricErrorCount() { - return TestMetricsUtil.getTotalCounter( - EXT, - API_ANNOTATION + SUFFIX_ERROR + SUFFIX_REALM, - List.of( - Tag.of(TAG_REALM, realm), - Tag.of(TAG_RESP_CODE, String.valueOf(ERROR_CODE)), - // spotless:off - Tag.of(TAG_REALM_DEPRECATED, realm), - Tag.of(TAG_RESP_CODE_DEPRECATED, String.valueOf(ERROR_CODE)))); - // spotless:on - } - - private double getCommonMetricCount() { - return TestMetricsUtil.getTotalCounter( - EXT, - SINGLETON_METRIC_NAME + SUFFIX_COUNTER, - Collections.singleton(Tag.of(TAG_API_NAME, API_ANNOTATION))); - } - - private double getCommonRealmMetricCount() { - return TestMetricsUtil.getTotalCounter( - EXT, - SINGLETON_METRIC_NAME + SUFFIX_COUNTER + SUFFIX_REALM, - List.of(Tag.of(TAG_API_NAME, API_ANNOTATION), Tag.of(TAG_REALM, realm))); - } - - private double getCommonMetricErrorCount() { - return TestMetricsUtil.getTotalCounter( - EXT, - SINGLETON_METRIC_NAME + SUFFIX_ERROR, - List.of( - Tag.of(TAG_API_NAME, API_ANNOTATION), - Tag.of(TAG_RESP_CODE, String.valueOf(ERROR_CODE)))); - } - - private double getCommonRealmMetricErrorCount() { - return TestMetricsUtil.getTotalCounter( - EXT, - SINGLETON_METRIC_NAME + SUFFIX_ERROR + SUFFIX_REALM, - List.of( - Tag.of(TAG_API_NAME, API_ANNOTATION), - Tag.of(TAG_REALM, realm), - Tag.of(TAG_RESP_CODE, String.valueOf(ERROR_CODE)))); - } - - private int sendRequest(String principalName) { - try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/%s/%s", EXT.getLocalPort(), ENDPOINT, principalName)) - .request("application/json") - .header("Authorization", "Bearer " + userToken.token()) - .header(REALM_PROPERTY_KEY, realm) - .get()) { - return response.getStatus(); - } - } - - private void sendSuccessfulRequest() { - Assertions.assertEquals( - Response.Status.OK.getStatusCode(), - sendRequest(snowmanCredentials.identifier().principalName())); - } - - private void sendFailingRequest() { - Assertions.assertEquals(ERROR_CODE, sendRequest("notarealprincipal")); - } -} diff --git a/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisAdminServiceAuthzTest.java b/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisAdminServiceAuthzTest.java index f695148c2..556ca7468 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisAdminServiceAuthzTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisAdminServiceAuthzTest.java @@ -18,6 +18,7 @@ */ package org.apache.polaris.service.admin; +import io.quarkus.test.junit.QuarkusTest; import java.util.List; import java.util.Map; import java.util.Set; @@ -35,6 +36,7 @@ import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; +@QuarkusTest public class PolarisAdminServiceAuthzTest extends PolarisAuthzTestBase { private PolarisAdminService newTestAdminService() { return newTestAdminService(Set.of()); diff --git a/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java b/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java index 6fc24a2b2..a66ed10da 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java @@ -23,6 +23,11 @@ import com.google.auth.oauth2.AccessToken; import com.google.auth.oauth2.GoogleCredentials; import com.google.common.collect.ImmutableMap; +import io.quarkus.test.junit.QuarkusMock; +import jakarta.annotation.Nonnull; +import jakarta.enterprise.inject.Vetoed; +import jakarta.enterprise.inject.spi.CDI; +import jakarta.inject.Inject; import java.io.IOException; import java.time.Clock; import java.util.Date; @@ -40,7 +45,6 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisConfiguration; import org.apache.polaris.core.PolarisConfigurationStore; -import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.admin.model.FileStorageConfigInfo; import org.apache.polaris.core.admin.model.PrincipalWithCredentials; @@ -60,23 +64,26 @@ import org.apache.polaris.core.entity.PolarisPrivilege; import org.apache.polaris.core.entity.PrincipalEntity; import org.apache.polaris.core.entity.PrincipalRoleEntity; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest; -import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.apache.polaris.service.catalog.BasePolarisCatalog; import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView; import org.apache.polaris.service.catalog.io.DefaultFileIOFactory; +import org.apache.polaris.service.catalog.io.FileIOFactory; import org.apache.polaris.service.config.DefaultConfigurationStore; import org.apache.polaris.service.config.RealmEntityManagerFactory; +import org.apache.polaris.service.context.CallContextCatalogFactory; import org.apache.polaris.service.context.PolarisCallContextCatalogFactory; -import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; import org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl; +import org.apache.polaris.service.task.TaskExecutor; import org.assertj.core.api.Assertions; -import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestInfo; import org.mockito.Mockito; /** Base class for shared test setup logic used by various Polaris authz-related tests. */ @@ -137,6 +144,11 @@ public abstract class PolarisAuthzTestBase { PolarisConfiguration.ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING.key, true))); + @Inject protected MetaStoreManagerFactory managerFactory; + @Inject protected RealmEntityManagerFactory realmEntityManagerFactory; + @Inject protected CallContextCatalogFactory callContextCatalogFactory; + @Inject protected PolarisDiagnostics diagServices; + protected BasePolarisCatalog baseCatalog; protected PolarisAdminService adminService; protected PolarisEntityManager entityManager; @@ -145,26 +157,38 @@ public abstract class PolarisAuthzTestBase { protected PrincipalEntity principalEntity; protected CallContext callContext; protected AuthenticatedPolarisPrincipal authenticatedRoot; - protected InMemoryPolarisMetaStoreManagerFactory metaStoreManagerFactory; - @BeforeEach - @SuppressWarnings("unchecked") - public void before() { - PolarisDiagnostics diagServices = new PolarisDefaultDiagServiceImpl(); - metaStoreManagerFactory = new InMemoryPolarisMetaStoreManagerFactory(); - metaStoreManagerFactory.setStorageIntegrationProvider( + private PolarisCallContext polarisContext; + + @BeforeAll + public static void setUpMocks() { + PolarisStorageIntegrationProviderImpl mock = new PolarisStorageIntegrationProviderImpl( - Mockito::mock, () -> GoogleCredentials.create(new AccessToken("abc", new Date())))); - RealmContext realmContext = () -> "realm"; - PolarisMetaStoreManager metaStoreManager = - metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext); + Mockito::mock, () -> GoogleCredentials.create(new AccessToken("abc", new Date()))); + QuarkusMock.installMockForType(mock, PolarisStorageIntegrationProviderImpl.class); + RealmEntityManagerFactory realmEntityManagerFactory = + CDI.current().select(RealmEntityManagerFactory.class).get(); + TaskExecutor taskExecutor = CDI.current().select(TaskExecutor.class).get(); + FileIOFactory fileIOFactory = CDI.current().select(FileIOFactory.class).get(); + MetaStoreManagerFactory metaStoreManagerFactory = + CDI.current().select(MetaStoreManagerFactory.class).get(); + TestPolarisCallContextCatalogFactory m = + new TestPolarisCallContextCatalogFactory( + realmEntityManagerFactory, metaStoreManagerFactory, taskExecutor, fileIOFactory); + QuarkusMock.installMockForType(m, PolarisCallContextCatalogFactory.class); + } + + @BeforeEach + public void before(TestInfo testInfo) { + RealmContext realmContext = testInfo::getDisplayName; + metaStoreManager = managerFactory.getOrCreateMetaStoreManager(realmContext); Map configMap = Map.of( "ALLOW_SPECIFYING_FILE_IO_IMPL", true, "ALLOW_EXTERNAL_METADATA_FILE_LOCATION", true); - PolarisCallContext polarisContext = + polarisContext = new PolarisCallContext( - metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), + managerFactory.getOrCreateSessionSupplier(realmContext).get(), diagServices, new PolarisConfigurationStore() { @Override @@ -173,18 +197,9 @@ public void before() { } }, Clock.systemDefaultZone()); - this.entityManager = new PolarisEntityManager(metaStoreManager, new StorageCredentialCache()); - this.metaStoreManager = metaStoreManager; + this.entityManager = realmEntityManagerFactory.getOrCreateEntityManager(realmContext); - callContext = - CallContext.of( - new RealmContext() { - @Override - public String getRealmIdentifier() { - return "test-realm"; - } - }, - polarisContext); + callContext = CallContext.of(realmContext, polarisContext); CallContext.setCurrentContext(callContext); PrincipalEntity rootEntity = @@ -299,17 +314,21 @@ public String getRealmIdentifier() { @AfterEach public void after() { - if (this.baseCatalog != null) { - try { - this.baseCatalog.close(); - this.baseCatalog = null; - } catch (IOException e) { - throw new RuntimeException(e); + try { + if (this.baseCatalog != null) { + try { + this.baseCatalog.close(); + this.baseCatalog = null; + } catch (IOException e) { + throw new RuntimeException(e); + } } + } finally { + metaStoreManager.purge(polarisContext); } } - protected @NotNull PrincipalEntity rotateAndRefreshPrincipal( + protected @Nonnull PrincipalEntity rotateAndRefreshPrincipal( PolarisMetaStoreManager metaStoreManager, String principalName, PrincipalWithCredentialsCredentials credentials, @@ -372,18 +391,16 @@ private void initBaseCatalog() { CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO")); } - public class TestPolarisCallContextCatalogFactory extends PolarisCallContextCatalogFactory { - public TestPolarisCallContextCatalogFactory() { - super( - new RealmEntityManagerFactory() { - @Override - public PolarisEntityManager getOrCreateEntityManager(RealmContext realmContext) { - return entityManager; - } - }, - metaStoreManagerFactory, - Mockito.mock(), - new DefaultFileIOFactory()); + @Vetoed + public static class TestPolarisCallContextCatalogFactory + extends PolarisCallContextCatalogFactory { + + public TestPolarisCallContextCatalogFactory( + RealmEntityManagerFactory entityManagerFactory, + MetaStoreManagerFactory metaStoreManagerFactory, + TaskExecutor taskExecutor, + FileIOFactory fileIOFactory) { + super(entityManagerFactory, metaStoreManagerFactory, taskExecutor, fileIOFactory); } @Override diff --git a/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingCatalogTest.java b/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingCatalogTest.java index 47be13e48..8548b8a7e 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingCatalogTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingCatalogTest.java @@ -18,75 +18,63 @@ */ package org.apache.polaris.service.admin; -import static org.apache.polaris.service.context.DefaultContextResolver.REALM_PROPERTY_KEY; +import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; import static org.assertj.core.api.Assertions.assertThat; -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; +import jakarta.inject.Inject; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.client.Invocation; import jakarta.ws.rs.core.Response; -import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.UUID; import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; import org.apache.polaris.core.admin.model.Catalog; import org.apache.polaris.core.admin.model.CatalogProperties; import org.apache.polaris.core.admin.model.CreateCatalogRequest; import org.apache.polaris.core.admin.model.StorageConfigInfo; -import org.apache.polaris.service.PolarisApplication; -import org.apache.polaris.service.config.PolarisApplicationConfig; -import org.apache.polaris.service.test.PolarisConnectionExtension; -import org.apache.polaris.service.test.PolarisRealm; -import org.apache.polaris.service.test.TestEnvironmentExtension; +import org.apache.polaris.service.test.PolarisIntegrationTestHelper; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class -}) +@QuarkusTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@TestProfile(PolarisOverlappingCatalogTest.Profile.class) public class PolarisOverlappingCatalogTest { - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - // Bind to random port to support parallelism - ConfigOverride.config("server.applicationConnectors[0].port", "0"), - ConfigOverride.config("server.adminConnectors[0].port", "0"), - // Block overlapping catalog paths: - ConfigOverride.config("featureConfiguration.ALLOW_OVERLAPPING_CATALOG_URLS", "false")); - private static String userToken; - private static String realm; + + @Inject PolarisIntegrationTestHelper testHelper; @BeforeAll - public static void setup( - PolarisConnectionExtension.PolarisToken adminToken, @PolarisRealm String polarisRealm) - throws IOException { - userToken = adminToken.token(); - realm = polarisRealm; - - // Set up the database location - PolarisConnectionExtension.createTestDir(realm); + public void setUp(TestInfo testInfo) { + testHelper.setUp(testInfo); + } + + @AfterAll + public void tearDown() { + testHelper.tearDown(); } private Response createCatalog(String prefix, String defaultBaseLocation, boolean isExternal) { return createCatalog(prefix, defaultBaseLocation, isExternal, new ArrayList()); } - private static Invocation.Builder request() { - return EXT.client() - .target(String.format("http://localhost:%d/api/management/v1/catalogs", EXT.getLocalPort())) + private Invocation.Builder request() { + return testHelper + .client + .target( + String.format("http://localhost:%d/api/management/v1/catalogs", testHelper.localPort)) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm); + .header("Authorization", "Bearer " + testHelper.adminToken) + .header(REALM_PROPERTY_KEY, testHelper.realm); } private Response createCatalog( @@ -179,4 +167,13 @@ public void testAllowedLocationOverlappingCatalogs( assertThat(createCatalog(prefix, "plays", initiallyExternal, Arrays.asList("rent", "cats"))) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); } + + public static class Profile implements QuarkusTestProfile { + + @Override + public Map getConfigOverrides() { + return Map.of( + "polaris.config.feature-configurations.ALLOW_OVERLAPPING_CATALOG_URLS", "false"); + } + } } diff --git a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingTableLaxTest.java b/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingTableLaxTest.java similarity index 100% rename from polaris-service-quarkus/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingTableLaxTest.java rename to polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingTableLaxTest.java diff --git a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingTableStrictTest.java b/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingTableStrictTest.java similarity index 100% rename from polaris-service-quarkus/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingTableStrictTest.java rename to polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingTableStrictTest.java diff --git a/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingTableTest.java b/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingTableTest.java deleted file mode 100644 index bb60f7149..000000000 --- a/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingTableTest.java +++ /dev/null @@ -1,301 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.admin; - -import static org.apache.polaris.service.admin.PolarisAuthzTestBase.SCHEMA; -import static org.apache.polaris.service.context.DefaultContextResolver.REALM_PROPERTY_KEY; -import static org.assertj.core.api.Assertions.assertThat; - -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; -import jakarta.ws.rs.client.Entity; -import jakarta.ws.rs.client.Invocation; -import jakarta.ws.rs.core.Response; -import java.util.List; -import java.util.UUID; -import java.util.stream.Stream; -import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.rest.requests.CreateNamespaceRequest; -import org.apache.iceberg.rest.requests.CreateTableRequest; -import org.apache.polaris.core.PolarisConfiguration; -import org.apache.polaris.core.admin.model.Catalog; -import org.apache.polaris.core.admin.model.CatalogProperties; -import org.apache.polaris.core.admin.model.CreateCatalogRequest; -import org.apache.polaris.core.admin.model.FileStorageConfigInfo; -import org.apache.polaris.core.admin.model.StorageConfigInfo; -import org.apache.polaris.service.PolarisApplication; -import org.apache.polaris.service.config.PolarisApplicationConfig; -import org.apache.polaris.service.test.PolarisConnectionExtension; -import org.apache.polaris.service.test.PolarisRealm; -import org.apache.polaris.service.test.TestEnvironmentExtension; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.MethodSource; - -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class -}) -public class PolarisOverlappingTableTest { - private static final DropwizardAppExtension BASE_EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - // Bind to random port to support parallelism - ConfigOverride.config("server.applicationConnectors[0].port", "0"), - ConfigOverride.config("server.adminConnectors[0].port", "0"), - // Enforce table location constraints - ConfigOverride.config("featureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION", "false"), - ConfigOverride.config("featureConfiguration.ALLOW_TABLE_LOCATION_OVERLAP", "false")); - - private static final DropwizardAppExtension LAX_EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - // Bind to random port to support parallelism - ConfigOverride.config("server.applicationConnectors[0].port", "0"), - ConfigOverride.config("server.adminConnectors[0].port", "0"), - // Relax table location constraints - ConfigOverride.config("featureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION", "true"), - ConfigOverride.config("featureConfiguration.ALLOW_TABLE_LOCATION_OVERLAP", "true")); - - private static PolarisConnectionExtension.PolarisToken adminToken; - private static String userToken; - private static String realm; - private static String namespace; - private static final String baseLocation = "file:///tmp/PolarisOverlappingTableTest"; - - private static final CatalogWrapper defaultCatalog = new CatalogWrapper("default"); - private static final CatalogWrapper laxCatalog = new CatalogWrapper("lax"); - private static final CatalogWrapper strictCatalog = new CatalogWrapper("strict"); - - /** Used to define a parameterized test config */ - protected record TestConfig( - DropwizardAppExtension extension, - CatalogWrapper catalogWrapper, - Response.Status response) { - public String catalog() { - return catalogWrapper.catalog; - } - - private String extensionName() { - return (extension - .getConfiguration() - .getConfigurationStore() - .getConfiguration(null, PolarisConfiguration.ALLOW_TABLE_LOCATION_OVERLAP)) - ? "lax" - : "strict"; - } - - /** Extract the first component of the catalog name; e.g. `default` from `default_123_xyz` */ - private String catalogShortName() { - int firstComponentEnd = catalog().indexOf('_'); - if (firstComponentEnd != -1) { - return catalog().substring(0, firstComponentEnd); - } else { - return catalog(); - } - } - - @Override - public String toString() { - return String.format( - "extension=%s, catalog=%s, status=%s", - extensionName(), catalogShortName(), response.toString()); - } - } - - /* Used to wrap a catalog name, so the TestConfig's final `catalog` field can be updated */ - protected static class CatalogWrapper { - public String catalog; - - public CatalogWrapper(String catalog) { - this.catalog = catalog; - } - - @Override - public String toString() { - return catalog; - } - } - - @BeforeEach - public void setup( - PolarisConnectionExtension.PolarisToken adminToken, @PolarisRealm String polarisRealm) { - userToken = adminToken.token(); - realm = polarisRealm; - defaultCatalog.catalog = String.format("default_catalog_%s", UUID.randomUUID().toString()); - laxCatalog.catalog = String.format("lax_catalog_%s", UUID.randomUUID().toString()); - strictCatalog.catalog = String.format("strict_catalog_%s", UUID.randomUUID().toString()); - for (var EXT : List.of(BASE_EXT, LAX_EXT)) { - for (var c : List.of(defaultCatalog, laxCatalog, strictCatalog)) { - CatalogProperties.Builder propertiesBuilder = - CatalogProperties.builder() - .setDefaultBaseLocation(String.format("%s/%s", baseLocation, c)); - if (!c.equals(defaultCatalog)) { - propertiesBuilder - .addProperty( - PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), - String.valueOf(c.equals(laxCatalog))) - .addProperty( - PolarisConfiguration.ALLOW_TABLE_LOCATION_OVERLAP.catalogConfig(), - String.valueOf(c.equals(laxCatalog))); - } - StorageConfigInfo config = - FileStorageConfigInfo.builder() - .setStorageType(StorageConfigInfo.StorageTypeEnum.FILE) - .build(); - Catalog catalogObject = - new Catalog( - Catalog.TypeEnum.INTERNAL, - c.catalog, - propertiesBuilder.build(), - 1725487592064L, - 1725487592064L, - 1, - config); - try (Response response = - request(EXT, "management/v1/catalogs") - .post(Entity.json(new CreateCatalogRequest(catalogObject)))) { - if (response.getStatus() != Response.Status.CREATED.getStatusCode()) { - throw new IllegalStateException( - "Failed to create catalog: " + response.readEntity(String.class)); - } - } - - namespace = "ns"; - CreateNamespaceRequest createNamespaceRequest = - CreateNamespaceRequest.builder().withNamespace(Namespace.of(namespace)).build(); - try (Response response = - request(EXT, String.format("catalog/v1/%s/namespaces", c)) - .post(Entity.json(createNamespaceRequest))) { - if (response.getStatus() != Response.Status.OK.getStatusCode()) { - throw new IllegalStateException( - "Failed to create namespace: " + response.readEntity(String.class)); - } - } - } - } - } - - private Response createTable( - DropwizardAppExtension extension, String catalog, String location) { - CreateTableRequest createTableRequest = - CreateTableRequest.builder() - .withName("table_" + UUID.randomUUID().toString()) - .withLocation(location) - .withSchema(SCHEMA) - .build(); - String prefix = String.format("catalog/v1/%s/namespaces/%s/tables", catalog, namespace); - try (Response response = request(extension, prefix).post(Entity.json(createTableRequest))) { - return response; - } - } - - private static Invocation.Builder request( - DropwizardAppExtension extension, String prefix) { - return extension - .client() - .target(String.format("http://localhost:%d/api/%s", extension.getLocalPort(), prefix)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm); - } - - private static Stream getTestConfigs() { - return Stream.of( - new TestConfig(BASE_EXT, defaultCatalog, Response.Status.FORBIDDEN), - new TestConfig(BASE_EXT, strictCatalog, Response.Status.FORBIDDEN), - new TestConfig(BASE_EXT, laxCatalog, Response.Status.OK), - new TestConfig(LAX_EXT, defaultCatalog, Response.Status.OK), - new TestConfig(LAX_EXT, strictCatalog, Response.Status.FORBIDDEN), - new TestConfig(LAX_EXT, laxCatalog, Response.Status.OK)); - } - - @ParameterizedTest - @MethodSource("getTestConfigs") - @DisplayName("Test restrictions on table locations") - void testTableLocationRestrictions(TestConfig config) { - // Original table - assertThat( - createTable( - config.extension, - config.catalog(), - String.format("%s/%s/%s/table_1", baseLocation, config.catalog(), namespace))) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus); - - // Unrelated path - assertThat( - createTable( - config.extension, - config.catalog(), - String.format("%s/%s/%s/table_2", baseLocation, config.catalog(), namespace))) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus); - - // Trailing slash makes this not overlap with table_1 - assertThat( - createTable( - config.extension, - config.catalog(), - String.format("%s/%s/%s/table_100", baseLocation, config.catalog(), namespace))) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus); - - // Repeat location - assertThat( - createTable( - config.extension, - config.catalog(), - String.format("%s/%s/%s/table_100", baseLocation, config.catalog(), namespace))) - .returns(config.response.getStatusCode(), Response::getStatus); - - // Parent of existing location - assertThat( - createTable( - config.extension, - config.catalog(), - String.format("%s/%s/%s", baseLocation, config.catalog(), namespace))) - .returns(config.response.getStatusCode(), Response::getStatus); - - // Child of existing location - assertThat( - createTable( - config.extension, - config.catalog(), - String.format( - "%s/%s/%s/table_100/child", baseLocation, config.catalog(), namespace))) - .returns(config.response.getStatusCode(), Response::getStatus); - - // Outside the namespace - assertThat( - createTable( - config.extension, - config.catalog(), - String.format("%s/%s", baseLocation, config.catalog()))) - .returns(config.response.getStatusCode(), Response::getStatus); - - // Outside the catalog - assertThat(createTable(config.extension, config.catalog(), String.format("%s", baseLocation))) - .returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); - } -} diff --git a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingTableTestBase.java b/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingTableTestBase.java similarity index 100% rename from polaris-service-quarkus/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingTableTestBase.java rename to polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingTableTestBase.java diff --git a/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplIntegrationTest.java b/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplIntegrationTest.java index 7ef920313..9a3d0e0b7 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplIntegrationTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplIntegrationTest.java @@ -18,37 +18,24 @@ */ package org.apache.polaris.service.admin; -import static io.dropwizard.jackson.Jackson.newObjectMapper; -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.polaris.service.context.DefaultContextResolver.REALM_PROPERTY_KEY; +import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; import static org.assertj.core.api.Assertions.assertThat; -import com.auth0.jwt.JWT; -import com.auth0.jwt.JWTCreator; -import com.auth0.jwt.algorithms.Algorithm; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; +import jakarta.inject.Inject; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.client.Invocation; import jakarta.ws.rs.core.Response; import java.io.IOException; -import java.time.Duration; -import java.time.Instant; -import java.time.temporal.ChronoUnit; import java.util.Arrays; -import java.util.Base64; import java.util.List; import java.util.Map; -import java.util.UUID; import org.apache.commons.lang3.RandomStringUtils; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.rest.RESTUtil; @@ -90,74 +77,41 @@ import org.apache.polaris.core.admin.model.UpdatePrincipalRequest; import org.apache.polaris.core.admin.model.UpdatePrincipalRoleRequest; import org.apache.polaris.core.entity.PolarisEntityConstants; -import org.apache.polaris.service.PolarisApplication; -import org.apache.polaris.service.auth.BasePolarisAuthenticator; import org.apache.polaris.service.auth.TokenUtils; -import org.apache.polaris.service.config.PolarisApplicationConfig; -import org.apache.polaris.service.test.PolarisConnectionExtension; -import org.apache.polaris.service.test.PolarisRealm; -import org.apache.polaris.service.test.TestEnvironmentExtension; +import org.apache.polaris.service.test.PolarisIntegrationTestHelper; import org.assertj.core.api.InstanceOfAssertFactories; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestInstance; import org.slf4j.LoggerFactory; -import org.testcontainers.shaded.org.awaitility.Awaitility; -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class -}) +@QuarkusTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@TestProfile(PolarisServiceImplIntegrationTest.Profile.class) public class PolarisServiceImplIntegrationTest { private static final int MAX_IDENTIFIER_LENGTH = 256; - private static final String ISSUER_KEY = "polaris"; - private static final String CLAIM_KEY_ACTIVE = "active"; - private static final String CLAIM_KEY_CLIENT_ID = "client_id"; - private static final String CLAIM_KEY_PRINCIPAL_ID = "principalId"; - private static final String CLAIM_KEY_SCOPE = "scope"; // TODO: Add a test-only hook that fully clobbers all persistence state so we can have a fresh // slate on every test case; otherwise, leftover state from one test from failures will interfere // with other test cases. - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config("server.adminConnectors[0].port", "0"), - - // disallow FILE urls for the sake of tests below - ConfigOverride.config( - "featureConfiguration.SUPPORTED_CATALOG_STORAGE_TYPES", "S3,GCS,AZURE"), - ConfigOverride.config("gcp_credentials.access_token", "abc"), - ConfigOverride.config("gcp_credentials.expires_in", "12345")); - private static String userToken; - private static String realm; - private static String clientId; + + @Inject PolarisIntegrationTestHelper testHelper; @BeforeAll - public static void setup( - PolarisConnectionExtension.PolarisToken adminToken, @PolarisRealm String polarisRealm) - throws IOException { - userToken = adminToken.token(); - realm = polarisRealm; - - Base64.Decoder decoder = Base64.getUrlDecoder(); - String[] chunks = adminToken.token().split("\\."); - String payload = new String(decoder.decode(chunks[1]), UTF_8); - JsonElement jsonElement = JsonParser.parseString(payload); - clientId = String.valueOf(((JsonObject) jsonElement).get("client_id")); - - // Set up test location - PolarisConnectionExtension.createTestDir(realm); + public void setUp(TestInfo testInfo) { + testHelper.setUp(testInfo); } - @AfterEach + @AfterAll public void tearDown() { + testHelper.tearDown(); + } + + @AfterEach + public void after() { try (Response response = newRequest("http://localhost:%d/api/management/v1/catalogs").get()) { response .readEntity(Catalogs.class) @@ -321,11 +275,11 @@ public void testListCatalogsUnauthorized() { PrincipalWithCredentials creds = response.readEntity(PrincipalWithCredentials.class); newToken = TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), + testHelper.client, + testHelper.localPort, creds.getCredentials().getClientId(), creds.getCredentials().getClientSecret(), - realm); + testHelper.realm); } try (Response response = newRequest("http://localhost:%d/api/management/v1/catalogs", newToken).get()) { @@ -363,8 +317,6 @@ public void testCreateCatalogWithInvalidName() { String goodName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH, true, true); - ObjectMapper mapper = newObjectMapper(); - Catalog catalog = PolarisCatalog.builder() .setType(Catalog.TypeEnum.INTERNAL) @@ -374,7 +326,7 @@ public void testCreateCatalogWithInvalidName() { .build(); try (Response response = newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(mapper.writeValueAsString(catalog)))) { + .post(Entity.json(testHelper.objectMapper.writeValueAsString(catalog)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } catch (JsonProcessingException e) { throw new RuntimeException(e); @@ -401,7 +353,7 @@ public void testCreateCatalogWithInvalidName() { try (Response response = newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(mapper.writeValueAsString(catalog)))) { + .post(Entity.json(testHelper.objectMapper.writeValueAsString(catalog)))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); assertThat(response.hasEntity()).isTrue(); @@ -527,7 +479,7 @@ public void testCreateCatalogWithoutProperties() { requestNode.set("catalog", catalogNode); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", userToken) + newRequest("http://localhost:%d/api/management/v1/catalogs", testHelper.adminToken) .post(Entity.json(requestNode))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); @@ -545,7 +497,7 @@ public void testCreateCatalogWithoutStorageConfig() throws JsonProcessingExcepti String catalogString = "{\"catalog\": {\"type\":\"INTERNAL\",\"name\":\"my-catalog\",\"properties\":{\"default-base-location\":\"s3://my-bucket/path/to/data\"}}}"; try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", userToken) + newRequest("http://localhost:%d/api/management/v1/catalogs", testHelper.adminToken) .post(Entity.json(catalogString))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); @@ -562,7 +514,7 @@ public void testCreateCatalogWithoutStorageConfig() throws JsonProcessingExcepti public void testCreateCatalogWithUnparsableJson() throws JsonProcessingException { String catalogString = "{\"catalog\": {{\"bad data}"; try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", userToken) + newRequest("http://localhost:%d/api/management/v1/catalogs", testHelper.adminToken) .post(Entity.json(catalogString))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); @@ -570,8 +522,7 @@ public void testCreateCatalogWithUnparsableJson() throws JsonProcessingException assertThat(error) .isNotNull() .extracting(ErrorResponse::message) - .asString() - .startsWith("Invalid JSON: Unexpected character"); + .isEqualTo("HTTP 400 Bad Request"); } } @@ -590,7 +541,7 @@ public void testCreateCatalogWithDisallowedStorageConfig() throws JsonProcessing .setStorageConfigInfo(fileStorage) .build(); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", userToken) + newRequest("http://localhost:%d/api/management/v1/catalogs", testHelper.adminToken) .post(Entity.json(new CreateCatalogRequest(catalog)))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); @@ -601,66 +552,6 @@ public void testCreateCatalogWithDisallowedStorageConfig() throws JsonProcessing } } - @Test - public void testUpdateCatalogWithoutDefaultBaseLocationInUpdate() throws JsonProcessingException { - AwsStorageConfigInfo awsConfigModel = - AwsStorageConfigInfo.builder() - .setRoleArn("arn:aws:iam::123456789012:role/my-role") - .setExternalId("externalId") - .setUserArn("userArn") - .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) - .setAllowedLocations(List.of("s3://my-old-bucket/path/to/data")) - .build(); - String catalogName = "mycatalog"; - Catalog catalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName(catalogName) - .setProperties(new CatalogProperties("s3://bucket/path/to/data")) - .setStorageConfigInfo(awsConfigModel) - .build(); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", userToken) - .post(Entity.json(new CreateCatalogRequest(catalog)))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - - // 200 successful GET after creation - Catalog fetchedCatalog = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + catalogName, userToken) - .get()) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - fetchedCatalog = response.readEntity(Catalog.class); - - assertThat(fetchedCatalog.getName()).isEqualTo(catalogName); - assertThat(fetchedCatalog.getProperties().toMap()) - .isEqualTo(Map.of("default-base-location", "s3://bucket/path/to/data")); - assertThat(fetchedCatalog.getEntityVersion()).isGreaterThan(0); - } - - // Create an UpdateCatalogRequest that only sets a new property foo=bar but omits - // default-base-location. - UpdateCatalogRequest updateRequest = - new UpdateCatalogRequest( - fetchedCatalog.getEntityVersion(), Map.of("foo", "bar"), null /* storageConfigIno */); - - // Successfully update - Catalog updatedCatalog = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + catalogName, userToken) - .put(Entity.json(updateRequest))) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - updatedCatalog = response.readEntity(Catalog.class); - - assertThat(updatedCatalog.getName()).isEqualTo(catalogName); - // Check that default-base-location is preserved in addition to adding the new property - assertThat(updatedCatalog.getProperties().toMap()) - .isEqualTo(Map.of("default-base-location", "s3://bucket/path/to/data", "foo", "bar")); - assertThat(updatedCatalog.getEntityVersion()).isGreaterThan(0); - } - } - @Test public void testUpdateCatalogWithDisallowedStorageConfig() throws JsonProcessingException { AwsStorageConfigInfo awsConfigModel = @@ -680,7 +571,7 @@ public void testUpdateCatalogWithDisallowedStorageConfig() throws JsonProcessing .setStorageConfigInfo(awsConfigModel) .build(); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", userToken) + newRequest("http://localhost:%d/api/management/v1/catalogs", testHelper.adminToken) .post(Entity.json(new CreateCatalogRequest(catalog)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } @@ -688,7 +579,9 @@ public void testUpdateCatalogWithDisallowedStorageConfig() throws JsonProcessing // 200 successful GET after creation Catalog fetchedCatalog = null; try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + catalogName, userToken) + newRequest( + "http://localhost:%d/api/management/v1/catalogs/" + catalogName, + testHelper.adminToken) .get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalog = response.readEntity(Catalog.class); @@ -711,7 +604,9 @@ public void testUpdateCatalogWithDisallowedStorageConfig() throws JsonProcessing // failure to update try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + catalogName, userToken) + newRequest( + "http://localhost:%d/api/management/v1/catalogs/" + catalogName, + testHelper.adminToken) .put(Entity.json(updateRequest))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); @@ -1000,16 +895,17 @@ public void testCreateListUpdateAndDeleteCatalog() { } } - private static Invocation.Builder newRequest(String url, String token) { - return EXT.client() - .target(String.format(url, EXT.getLocalPort())) + private Invocation.Builder newRequest(String url, String token) { + return testHelper + .client + .target(String.format(url, testHelper.localPort)) .request("application/json") .header("Authorization", "Bearer " + token) - .header(REALM_PROPERTY_KEY, realm); + .header(REALM_PROPERTY_KEY, testHelper.realm); } - private static Invocation.Builder newRequest(String url) { - return newRequest(url, userToken); + private Invocation.Builder newRequest(String url) { + return newRequest(url, testHelper.adminToken); } @Test @@ -1095,11 +991,11 @@ public void testListPrincipalsUnauthorized() { PrincipalWithCredentials creds = response.readEntity(PrincipalWithCredentials.class); newToken = TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), + testHelper.client, + testHelper.localPort, creds.getCredentials().getClientId(), creds.getCredentials().getClientSecret(), - realm); + testHelper.realm); } try (Response response = newRequest("http://localhost:%d/api/management/v1/principals", newToken).get()) { @@ -1142,7 +1038,7 @@ public void testCreatePrincipalAndRotateCredentials() { // Get a fresh token associate with the principal itself. String newPrincipalToken = TokenUtils.getTokenFromSecrets( - EXT.client(), EXT.getLocalPort(), oldClientId, oldSecret, realm); + testHelper.client, testHelper.localPort, oldClientId, oldSecret, testHelper.realm); // Any call should initially fail with error indicating that rotation is needed. try (Response response = @@ -1973,7 +1869,8 @@ public void testCatalogAdminGrantAndRevokeCatalogRoles() { createCatalog(catalog); CatalogRole catalogAdminRole = readCatalogRole(catalogName, "catalog_admin"); - grantCatalogRoleToPrincipalRole(principalRoleName, catalogName, catalogAdminRole, userToken); + grantCatalogRoleToPrincipalRole( + principalRoleName, catalogName, catalogAdminRole, testHelper.adminToken); PrincipalWithCredentials catalogAdminPrincipal = createPrincipal("principal1"); @@ -1981,11 +1878,11 @@ public void testCatalogAdminGrantAndRevokeCatalogRoles() { String catalogAdminToken = TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), + testHelper.client, + testHelper.localPort, catalogAdminPrincipal.getCredentials().getClientId(), catalogAdminPrincipal.getCredentials().getClientSecret(), - realm); + testHelper.realm); // Create a second principal role. Use the catalog admin principal to list principal roles and // grant a catalog role to the new principal role @@ -2034,7 +1931,7 @@ public void testCatalogAdminGrantAndRevokeCatalogRoles() { + catalogName + "/" + catalogRoleName, - userToken) + testHelper.adminToken) .delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -2061,7 +1958,8 @@ public void testServiceAdminCanTransferCatalogAdmin() { createCatalog(catalog); CatalogRole catalogAdminRole = readCatalogRole(catalogName, "catalog_admin"); - grantCatalogRoleToPrincipalRole(principalRoleName, catalogName, catalogAdminRole, userToken); + grantCatalogRoleToPrincipalRole( + principalRoleName, catalogName, catalogAdminRole, testHelper.adminToken); PrincipalWithCredentials catalogAdminPrincipal = createPrincipal("principal1"); @@ -2069,11 +1967,11 @@ public void testServiceAdminCanTransferCatalogAdmin() { String catalogAdminToken = TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), + testHelper.client, + testHelper.localPort, catalogAdminPrincipal.getCredentials().getClientId(), catalogAdminPrincipal.getCredentials().getClientSecret(), - realm); + testHelper.realm); // service_admin revokes the catalog_admin privilege from its principal role try { @@ -2082,7 +1980,7 @@ public void testServiceAdminCanTransferCatalogAdmin() { "http://localhost:%d/api/management/v1/principal-roles/service_admin/catalog-roles/" + catalogName + "/catalog_admin", - userToken) + testHelper.adminToken) .delete()) { assertThat(response) .returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); @@ -2144,11 +2042,12 @@ public void testCatalogAdminGrantAndRevokeCatalogRolesFromWrongCatalog() { // create a catalog role *in the second catalog* and grant it manage_content privilege String catalogRoleName = "mycr1"; - createCatalogRole(catalogName2, catalogRoleName, userToken); + createCatalogRole(catalogName2, catalogRoleName, testHelper.adminToken); // Get the catalog admin role from the *first* catalog and grant that role to the principal role CatalogRole catalogAdminRole = readCatalogRole(catalogName, "catalog_admin"); - grantCatalogRoleToPrincipalRole(principalRoleName, catalogName, catalogAdminRole, userToken); + grantCatalogRoleToPrincipalRole( + principalRoleName, catalogName, catalogAdminRole, testHelper.adminToken); // Create a principal and grant the principal role to it PrincipalWithCredentials catalogAdminPrincipal = createPrincipal("principal1"); @@ -2156,11 +2055,11 @@ public void testCatalogAdminGrantAndRevokeCatalogRolesFromWrongCatalog() { String catalogAdminToken = TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), + testHelper.client, + testHelper.localPort, catalogAdminPrincipal.getCredentials().getClientId(), catalogAdminPrincipal.getCredentials().getClientSecret(), - realm); + testHelper.realm); // Create a second principal role. String principalRoleName2 = "mypr2"; @@ -2202,7 +2101,7 @@ public void testTableManageAccessCanGrantAndRevokeFromCatalogRoles() { createCatalog(catalog); // create a valid target CatalogRole in this catalog - createCatalogRole(catalogName, "target_catalog_role", userToken); + createCatalogRole(catalogName, "target_catalog_role", testHelper.adminToken); // create a second catalog String catalogName2 = "anothertablemanagecatalog"; @@ -2218,7 +2117,7 @@ public void testTableManageAccessCanGrantAndRevokeFromCatalogRoles() { createCatalog(catalog2); // create an *invalid* target CatalogRole in second catalog - createCatalogRole(catalogName2, "invalid_target_catalog_role", userToken); + createCatalogRole(catalogName2, "invalid_target_catalog_role", testHelper.adminToken); // create the namespace "c" in *both* namespaces String namespaceName = "c"; @@ -2229,7 +2128,7 @@ public void testTableManageAccessCanGrantAndRevokeFromCatalogRoles() { // namespace level // grant that role to the PrincipalRole String catalogRoleName = "ns_manage_access_role"; - createCatalogRole(catalogName, catalogRoleName, userToken); + createCatalogRole(catalogName, catalogRoleName, testHelper.adminToken); grantPrivilegeToCatalogRole( catalogName, catalogRoleName, @@ -2237,11 +2136,11 @@ public void testTableManageAccessCanGrantAndRevokeFromCatalogRoles() { List.of(namespaceName), NamespacePrivilege.CATALOG_MANAGE_ACCESS, GrantResource.TypeEnum.NAMESPACE), - userToken, + testHelper.adminToken, Response.Status.CREATED); grantCatalogRoleToPrincipalRole( - principalRoleName, catalogName, new CatalogRole(catalogRoleName), userToken); + principalRoleName, catalogName, new CatalogRole(catalogRoleName), testHelper.adminToken); // Create a principal and grant the principal role to it PrincipalWithCredentials catalogAdminPrincipal = createPrincipal("ns_manage_access_user"); @@ -2249,11 +2148,11 @@ public void testTableManageAccessCanGrantAndRevokeFromCatalogRoles() { String manageAccessUserToken = TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), + testHelper.client, + testHelper.localPort, catalogAdminPrincipal.getCredentials().getClientId(), catalogAdminPrincipal.getCredentials().getClientSecret(), - realm); + testHelper.realm); // Use the ns_manage_access_user to grant TABLE_CREATE access to the target catalog role // This works because the user has CATALOG_MANAGE_ACCESS within the namespace and the target @@ -2316,83 +2215,11 @@ public void testTableManageAccessCanGrantAndRevokeFromCatalogRoles() { Response.Status.FORBIDDEN); } - @Test - public void testTokenExpiry() { - // TokenExpiredException - if the token has expired. - String newToken = - defaultJwt() - .withExpiresAt(Instant.now().plus(1, ChronoUnit.SECONDS)) - .sign(Algorithm.HMAC256("polaris")); - Awaitility.await("expected list of records should be produced") - .atMost(Duration.ofSeconds(2)) - .pollDelay(Duration.ofSeconds(1)) - .pollInterval(Duration.ofSeconds(1)) - .untilAsserted( - () -> { - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principals", "Bearer " + newToken) - .get()) { - assertThat(response) - .returns(Response.Status.UNAUTHORIZED.getStatusCode(), Response::getStatus); - } - }); - } - - @Test - public void testTokenInactive() { - // InvalidClaimException - if a claim contained a different value than the expected one. - String newToken = - defaultJwt().withClaim(CLAIM_KEY_ACTIVE, false).sign(Algorithm.HMAC256("polaris")); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals", "Bearer " + newToken) - .get()) { - assertThat(response) - .returns(Response.Status.UNAUTHORIZED.getStatusCode(), Response::getStatus); - } - } - - @Test - public void testTokenInvalidSignature() { - // SignatureVerificationException - if the signature is invalid. - String newToken = defaultJwt().sign(Algorithm.HMAC256("invalid_secret")); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals", "Bearer " + newToken) - .get()) { - assertThat(response) - .returns(Response.Status.UNAUTHORIZED.getStatusCode(), Response::getStatus); - } - } - - @Test - public void testTokenInvalidPrincipalId() { - String newToken = - defaultJwt().withClaim(CLAIM_KEY_PRINCIPAL_ID, 0).sign(Algorithm.HMAC256("polaris")); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals", "Bearer " + newToken) - .get()) { - assertThat(response) - .returns(Response.Status.UNAUTHORIZED.getStatusCode(), Response::getStatus); - } - } - - public static JWTCreator.Builder defaultJwt() { - Instant now = Instant.now(); - return JWT.create() - .withIssuer(ISSUER_KEY) - .withSubject(String.valueOf(1)) - .withIssuedAt(now) - .withExpiresAt(now.plus(10, ChronoUnit.SECONDS)) - .withJWTId(UUID.randomUUID().toString()) - .withClaim(CLAIM_KEY_ACTIVE, true) - .withClaim(CLAIM_KEY_CLIENT_ID, clientId) - .withClaim(CLAIM_KEY_PRINCIPAL_ID, 1) - .withClaim(CLAIM_KEY_SCOPE, BasePolarisAuthenticator.PRINCIPAL_ROLE_ALL); - } - - private static void createNamespace(String catalogName, String namespaceName) { + private void createNamespace(String catalogName, String namespaceName) { try (Response response = - newRequest("http://localhost:%d/api/catalog/v1/" + catalogName + "/namespaces", userToken) + newRequest( + "http://localhost:%d/api/catalog/v1/" + catalogName + "/namespaces", + testHelper.adminToken) .post( Entity.json( CreateNamespaceRequest.builder() @@ -2402,7 +2229,7 @@ private static void createNamespace(String catalogName, String namespaceName) { } } - private static void createCatalog(Catalog catalog) { + private void createCatalog(Catalog catalog) { try (Response response = newRequest("http://localhost:%d/api/management/v1/catalogs") .post(Entity.json(new CreateCatalogRequest(catalog)))) { @@ -2411,7 +2238,7 @@ private static void createCatalog(Catalog catalog) { } } - private static void grantPrivilegeToCatalogRole( + private void grantPrivilegeToCatalogRole( String catalogName, String catalogRoleName, GrantResource grant, @@ -2430,7 +2257,7 @@ private static void grantPrivilegeToCatalogRole( } } - private static void createCatalogRole( + private void createCatalogRole( String catalogName, String catalogRoleName, String catalogAdminToken) { try (Response response = newRequest( @@ -2441,8 +2268,7 @@ private static void createCatalogRole( } } - private static void grantPrincipalRoleToPrincipal( - String principalName, PrincipalRole principalRole) { + private void grantPrincipalRoleToPrincipal(String principalName, PrincipalRole principalRole) { try (Response response = newRequest( "http://localhost:%d/api/management/v1/principals/" @@ -2453,7 +2279,7 @@ private static void grantPrincipalRoleToPrincipal( } } - private static PrincipalWithCredentials createPrincipal(String principalName) { + private PrincipalWithCredentials createPrincipal(String principalName) { PrincipalWithCredentials catalogAdminPrincipal; try (Response response = newRequest("http://localhost:%d/api/management/v1/principals") @@ -2464,7 +2290,7 @@ private static PrincipalWithCredentials createPrincipal(String principalName) { return catalogAdminPrincipal; } - private static void grantCatalogRoleToPrincipalRole( + private void grantCatalogRoleToPrincipalRole( String principalRoleName, String catalogName, CatalogRole catalogRole, String token) { try (Response response = newRequest( @@ -2478,7 +2304,7 @@ private static void grantCatalogRoleToPrincipalRole( } } - private static CatalogRole readCatalogRole(String catalogName, String roleName) { + private CatalogRole readCatalogRole(String catalogName, String roleName) { try (Response response = newRequest( "http://localhost:%d/api/management/v1/catalogs/" @@ -2492,7 +2318,7 @@ private static CatalogRole readCatalogRole(String catalogName, String roleName) } } - private static void createPrincipalRole(PrincipalRole principalRole1) { + private void createPrincipalRole(PrincipalRole principalRole1) { try (Response response = newRequest("http://localhost:%d/api/management/v1/principal-roles") .post(Entity.json(new CreatePrincipalRoleRequest(principalRole1)))) { @@ -2500,4 +2326,15 @@ private static void createPrincipalRole(PrincipalRole principalRole1) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } } + + public static class Profile implements QuarkusTestProfile { + + @Override + public Map getConfigOverrides() { + // disallow FILE urls for the sake of tests below + return Map.of( + "polaris.config.feature-configurations.SUPPORTED_CATALOG_STORAGE_TYPES", + "[\"S3\",\"GCS\",\"AZURE\"]"); + } + } } diff --git a/polaris-service/src/test/java/org/apache/polaris/service/auth/JWTSymmetricKeyGeneratorTest.java b/polaris-service/src/test/java/org/apache/polaris/service/auth/JWTSymmetricKeyGeneratorTest.java index db7a00fff..110a99269 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/auth/JWTSymmetricKeyGeneratorTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/auth/JWTSymmetricKeyGeneratorTest.java @@ -26,14 +26,15 @@ import com.auth0.jwt.interfaces.DecodedJWT; import java.util.Map; import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.auth.PolarisSecretsManager.PrincipalSecretsResult; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisPrincipalSecrets; +import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.junit.jupiter.api.Test; import org.mockito.Mockito; @@ -65,8 +66,10 @@ public Map contextVariables() { String clientId = "test_client_id"; PolarisPrincipalSecrets principalSecrets = new PolarisPrincipalSecrets(1L, clientId, mainSecret, "otherSecret"); + PolarisEntityManager entityManager = + new PolarisEntityManager(metastoreManager, new StorageCredentialCache()); Mockito.when(metastoreManager.loadPrincipalSecrets(polarisCallContext, clientId)) - .thenReturn(new PrincipalSecretsResult(principalSecrets)); + .thenReturn(new PolarisMetaStoreManager.PrincipalSecretsResult(principalSecrets)); PolarisBaseEntity principal = new PolarisBaseEntity( 0L, diff --git a/polaris-service/src/test/java/org/apache/polaris/service/auth/TokenUtils.java b/polaris-service/src/test/java/org/apache/polaris/service/auth/TokenUtils.java index e85089e37..a4cc29803 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/auth/TokenUtils.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/auth/TokenUtils.java @@ -18,7 +18,8 @@ */ package org.apache.polaris.service.auth; -import static org.apache.polaris.service.context.DefaultContextResolver.REALM_PROPERTY_KEY; +import static org.apache.polaris.service.auth.BasePolarisAuthenticator.PRINCIPAL_ROLE_ALL; +import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; import static org.assertj.core.api.Assertions.assertThat; import jakarta.ws.rs.client.Client; @@ -34,17 +35,11 @@ public class TokenUtils { /** Get token against specified realm */ public static String getTokenFromSecrets( Client client, int port, String clientId, String clientSecret, String realm) { - return getTokenFromSecrets( - client, String.format("http://localhost:%d", port), clientId, clientSecret, realm); - } - - public static String getTokenFromSecrets( - Client client, String baseUrl, String clientId, String clientSecret, String realm) { String token; Invocation.Builder builder = client - .target(String.format("%s/api/catalog/v1/oauth/tokens", baseUrl)) + .target(String.format("http://localhost:%d/api/catalog/v1/oauth/tokens", port)) .request("application/json"); if (realm != null) { builder = builder.header(REALM_PROPERTY_KEY, realm); @@ -58,7 +53,7 @@ public static String getTokenFromSecrets( "grant_type", "client_credentials", "scope", - "PRINCIPAL_ROLE:ALL", + PRINCIPAL_ROLE_ALL, "client_id", clientId, "client_secret", diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java index 7b0270c40..27fd9d87a 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java @@ -25,11 +25,14 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterators; +import io.quarkus.test.junit.QuarkusMock; +import io.quarkus.test.junit.QuarkusTest; +import jakarta.inject.Inject; import java.io.IOException; +import java.lang.reflect.Method; import java.time.Clock; import java.util.Arrays; import java.util.EnumMap; -import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -59,7 +62,6 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisConfiguration; import org.apache.polaris.core.PolarisConfigurationStore; -import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; import org.apache.polaris.core.admin.model.StorageConfigInfo; @@ -91,26 +93,27 @@ import org.apache.polaris.service.catalog.io.FileIOFactory; import org.apache.polaris.service.catalog.io.TestFileIOFactory; import org.apache.polaris.service.exception.IcebergExceptionMapper; -import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; +import org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl; import org.apache.polaris.service.task.TableCleanupTaskHandler; import org.apache.polaris.service.task.TaskExecutor; import org.apache.polaris.service.task.TaskFileIOSupplier; import org.apache.polaris.service.types.NotificationRequest; import org.apache.polaris.service.types.NotificationType; import org.apache.polaris.service.types.TableUpdateNotification; -import org.assertj.core.api.AbstractBooleanAssert; import org.assertj.core.api.Assertions; -import org.jetbrains.annotations.Nullable; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.mockito.Mockito; import software.amazon.awssdk.services.sts.StsClient; import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; import software.amazon.awssdk.services.sts.model.AssumeRoleResponse; import software.amazon.awssdk.services.sts.model.Credentials; +@QuarkusTest public class BasePolarisCatalogTest extends CatalogTests { protected static final Namespace NS = Namespace.of("newdb"); protected static final TableIdentifier TABLE = TableIdentifier.of(NS, "table"); @@ -123,9 +126,15 @@ public class BasePolarisCatalogTest extends CatalogTests { public static final String SECRET_ACCESS_KEY = "secret_access_key"; public static final String SESSION_TOKEN = "session_token"; + @Inject MetaStoreManagerFactory managerFactory; + @Inject PolarisConfigurationStore configurationStore; + @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; + @Inject PolarisDiagnostics diagServices; + private BasePolarisCatalog catalog; private AwsStorageConfigInfo storageConfigModel; private StsClient stsClient; + private String realmName; private PolarisMetaStoreManager metaStoreManager; private PolarisCallContext polarisContext; private PolarisAdminService adminService; @@ -133,29 +142,27 @@ public class BasePolarisCatalogTest extends CatalogTests { private AuthenticatedPolarisPrincipal authenticatedRoot; private PolarisEntity catalogEntity; + @BeforeAll + public static void setUpMocks() { + PolarisStorageIntegrationProviderImpl mock = + Mockito.mock(PolarisStorageIntegrationProviderImpl.class); + QuarkusMock.installMockForType(mock, PolarisStorageIntegrationProviderImpl.class); + } + @BeforeEach @SuppressWarnings("unchecked") - public void before() { - PolarisDiagnostics diagServices = new PolarisDefaultDiagServiceImpl(); - RealmContext realmContext = () -> "realm"; - PolarisStorageIntegrationProvider storageIntegrationProvider = Mockito.mock(); - InMemoryPolarisMetaStoreManagerFactory managerFactory = - new InMemoryPolarisMetaStoreManagerFactory(); - managerFactory.setStorageIntegrationProvider(storageIntegrationProvider); + public void before(TestInfo testInfo) { + realmName = + "realm_%s_%s" + .formatted( + testInfo.getTestMethod().map(Method::getName).orElse("test"), System.nanoTime()); + RealmContext realmContext = () -> realmName; metaStoreManager = managerFactory.getOrCreateMetaStoreManager(realmContext); - Map configMap = new HashMap<>(); - configMap.put("ALLOW_SPECIFYING_FILE_IO_IMPL", true); - configMap.put("INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST", true); polarisContext = new PolarisCallContext( managerFactory.getOrCreateSessionSupplier(realmContext).get(), diagServices, - new PolarisConfigurationStore() { - @Override - public @Nullable T getConfiguration(PolarisCallContext ctx, String configName) { - return (T) configMap.get(configName); - } - }, + configurationStore, Clock.systemDefaultZone()); entityManager = new PolarisEntityManager(metaStoreManager, new StorageCredentialCache()); @@ -243,6 +250,7 @@ public void before() { @AfterEach public void after() throws IOException { catalog().close(); + metaStoreManager.purge(polarisContext); } @Override @@ -1334,8 +1342,8 @@ public void testDropTableWithPurge() { TableMetadata tableMetadata = ((BaseTable) table).operations().current(); boolean dropped = catalog.dropTable(TABLE, true); - ((AbstractBooleanAssert) - Assertions.assertThat(dropped).as("Should drop a table that does exist", new Object[0])) + Assertions.assertThat(dropped) + .as("Should drop a table that does exist", new Object[0]) .isTrue(); Assertions.assertThatPredicate(catalog::tableExists) .as("Table should not exist after drop") diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogViewTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogViewTest.java index 252cd16f1..f032aff1f 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogViewTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogViewTest.java @@ -18,14 +18,16 @@ */ package org.apache.polaris.service.catalog; -import com.google.auth.oauth2.AccessToken; -import com.google.auth.oauth2.GoogleCredentials; import com.google.common.collect.ImmutableMap; +import io.quarkus.test.junit.QuarkusMock; +import io.quarkus.test.junit.QuarkusTest; +import jakarta.inject.Inject; +import java.io.IOException; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.nio.file.Path; import java.time.Clock; -import java.util.Date; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Set; import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.catalog.Catalog; @@ -33,7 +35,6 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisConfiguration; import org.apache.polaris.core.PolarisConfigurationStore; -import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.admin.model.FileStorageConfigInfo; import org.apache.polaris.core.admin.model.StorageConfigInfo; @@ -46,52 +47,69 @@ import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PrincipalEntity; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.apache.polaris.service.admin.PolarisAdminService; import org.apache.polaris.service.catalog.io.DefaultFileIOFactory; -import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; import org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl; -import org.jetbrains.annotations.Nullable; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.io.TempDir; import org.mockito.Mockito; +@QuarkusTest public class BasePolarisCatalogViewTest extends ViewCatalogTests { public static final String CATALOG_NAME = "polaris-catalog"; + + @Inject MetaStoreManagerFactory managerFactory; + @Inject PolarisConfigurationStore configurationStore; + @Inject PolarisDiagnostics diagServices; + private BasePolarisCatalog catalog; + private String realmName; + private PolarisMetaStoreManager metaStoreManager; + private PolarisCallContext polarisContext; + + @BeforeAll + public static void setUpMocks() { + PolarisStorageIntegrationProviderImpl mock = + Mockito.mock(PolarisStorageIntegrationProviderImpl.class); + QuarkusMock.installMockForType(mock, PolarisStorageIntegrationProviderImpl.class); + } + @BeforeEach - @SuppressWarnings("unchecked") - public void before() { - PolarisDiagnostics diagServices = new PolarisDefaultDiagServiceImpl(); - RealmContext realmContext = () -> "realm"; - InMemoryPolarisMetaStoreManagerFactory managerFactory = - new InMemoryPolarisMetaStoreManagerFactory(); - managerFactory.setStorageIntegrationProvider( - new PolarisStorageIntegrationProviderImpl( - Mockito::mock, () -> GoogleCredentials.create(new AccessToken("abc", new Date())))); - PolarisMetaStoreManager metaStoreManager = - managerFactory.getOrCreateMetaStoreManager(realmContext); - Map configMap = new HashMap<>(); - configMap.put("ALLOW_WILDCARD_LOCATION", true); - configMap.put("ALLOW_SPECIFYING_FILE_IO_IMPL", true); - PolarisCallContext polarisContext = + public void setUpTempDir(@TempDir Path tempDir) throws Exception { + // see https://github.com/quarkusio/quarkus/issues/13261 + Field field = ViewCatalogTests.class.getDeclaredField("tempDir"); + field.setAccessible(true); + field.set(this, tempDir); + } + + @BeforeEach + public void before(TestInfo testInfo) { + realmName = + "realm_%s_%s" + .formatted( + testInfo.getTestMethod().map(Method::getName).orElse("test"), System.nanoTime()); + RealmContext realmContext = () -> realmName; + + metaStoreManager = managerFactory.getOrCreateMetaStoreManager(realmContext); + polarisContext = new PolarisCallContext( managerFactory.getOrCreateSessionSupplier(realmContext).get(), diagServices, - new PolarisConfigurationStore() { - @Override - public @Nullable T getConfiguration(PolarisCallContext ctx, String configName) { - return (T) configMap.get(configName); - } - }, + configurationStore, Clock.systemDefaultZone()); PolarisEntityManager entityManager = new PolarisEntityManager(metaStoreManager, new StorageCredentialCache()); - CallContext callContext = CallContext.of(null, polarisContext); + CallContext callContext = CallContext.of(realmContext, polarisContext); CallContext.setCurrentContext(callContext); PrincipalEntity rootEntity = @@ -146,6 +164,12 @@ public void before() { CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO")); } + @AfterEach + public void after() throws IOException { + catalog().close(); + metaStoreManager.purge(polarisContext); + } + @Override protected BasePolarisCatalog catalog() { return catalog; diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapperAuthzTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapperAuthzTest.java index 091bff21a..e1a608217 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapperAuthzTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapperAuthzTest.java @@ -19,6 +19,9 @@ package org.apache.polaris.service.catalog; import com.google.common.collect.ImmutableMap; +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; import java.time.Instant; import java.util.List; import java.util.Map; @@ -63,6 +66,7 @@ import org.apache.polaris.service.admin.PolarisAuthzTestBase; import org.apache.polaris.service.catalog.io.DefaultFileIOFactory; import org.apache.polaris.service.config.RealmEntityManagerFactory; +import org.apache.polaris.service.context.CallContextCatalogFactory; import org.apache.polaris.service.context.PolarisCallContextCatalogFactory; import org.apache.polaris.service.types.NotificationRequest; import org.apache.polaris.service.types.NotificationType; @@ -71,20 +75,19 @@ import org.junit.jupiter.api.Test; import org.mockito.Mockito; +@QuarkusTest +@TestProfile(PolarisCatalogHandlerWrapperAuthzTest.Profile.class) public class PolarisCatalogHandlerWrapperAuthzTest extends PolarisAuthzTestBase { private PolarisCatalogHandlerWrapper newWrapper() { return newWrapper(Set.of()); } private PolarisCatalogHandlerWrapper newWrapper(Set activatedPrincipalRoles) { - return newWrapper( - activatedPrincipalRoles, CATALOG_NAME, new TestPolarisCallContextCatalogFactory()); + return newWrapper(activatedPrincipalRoles, CATALOG_NAME, callContextCatalogFactory); } private PolarisCatalogHandlerWrapper newWrapper( - Set activatedPrincipalRoles, - String catalogName, - PolarisCallContextCatalogFactory factory) { + Set activatedPrincipalRoles, String catalogName, CallContextCatalogFactory factory) { final AuthenticatedPolarisPrincipal authenticatedPrincipal = new AuthenticatedPolarisPrincipal(principalEntity, activatedPrincipalRoles); return new PolarisCatalogHandlerWrapper( @@ -229,7 +232,7 @@ public void testInsufficientPermissionsPriorToSecretRotation() { entityManager, metaStoreManager, authenticatedPrincipal, - new TestPolarisCallContextCatalogFactory(), + callContextCatalogFactory, CATALOG_NAME, polarisAuthorizer); @@ -260,7 +263,7 @@ public void testInsufficientPermissionsPriorToSecretRotation() { entityManager, metaStoreManager, authenticatedPrincipal1, - new TestPolarisCallContextCatalogFactory(), + callContextCatalogFactory, CATALOG_NAME, polarisAuthorizer); @@ -1675,13 +1678,13 @@ public void testSendNotificationSufficientPrivileges() { PolarisCallContextCatalogFactory factory = new PolarisCallContextCatalogFactory( - new RealmEntityManagerFactory() { + new RealmEntityManagerFactory(managerFactory) { @Override public PolarisEntityManager getOrCreateEntityManager(RealmContext realmContext) { return entityManager; } }, - metaStoreManagerFactory, + managerFactory, Mockito.mock(), new DefaultFileIOFactory()) { @Override @@ -1814,4 +1817,13 @@ public void testSendNotificationInsufficientPermissions() { newWrapper(Set.of(PRINCIPAL_ROLE1)).sendNotification(table, request); }); } + + public static class Profile implements QuarkusTestProfile { + + @Override + public Map getConfigOverrides() { + return Map.of( + "polaris.config.feature-configurations.ALLOW_EXTERNAL_METADATA_FILE_LOCATION", "true"); + } + } } diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogIntegrationTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogIntegrationTest.java index 9a01f67ad..aedebc25e 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogIntegrationTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogIntegrationTest.java @@ -18,45 +18,40 @@ */ package org.apache.polaris.service.catalog; -import static org.apache.polaris.service.context.DefaultContextResolver.REALM_PROPERTY_KEY; +import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.google.common.collect.ImmutableMap; -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; +import io.quarkus.test.junit.QuarkusTest; +import jakarta.inject.Inject; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.core.Response; -import java.io.IOException; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.UUID; -import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.BaseTable; import org.apache.iceberg.BaseTransaction; +import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; -import org.apache.iceberg.TableMetadata; -import org.apache.iceberg.TableMetadataParser; import org.apache.iceberg.Transaction; import org.apache.iceberg.UpdatePartitionSpec; import org.apache.iceberg.UpdateSchema; import org.apache.iceberg.catalog.CatalogTests; import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.SessionCatalog; import org.apache.iceberg.catalog.TableCommit; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.CommitFailedException; import org.apache.iceberg.exceptions.ForbiddenException; import org.apache.iceberg.expressions.Expressions; -import org.apache.iceberg.io.ResolvingFileIO; +import org.apache.iceberg.rest.HTTPClient; import org.apache.iceberg.rest.RESTCatalog; +import org.apache.iceberg.rest.auth.OAuth2Properties; import org.apache.iceberg.rest.responses.ErrorResponse; import org.apache.iceberg.types.Types; import org.apache.polaris.core.PolarisConfiguration; @@ -79,36 +74,27 @@ import org.apache.polaris.core.admin.model.ViewPrivilege; import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.PolarisEntityConstants; -import org.apache.polaris.service.PolarisApplication; -import org.apache.polaris.service.auth.TokenUtils; -import org.apache.polaris.service.config.PolarisApplicationConfig; -import org.apache.polaris.service.test.PolarisConnectionExtension; -import org.apache.polaris.service.test.PolarisConnectionExtension.PolarisToken; -import org.apache.polaris.service.test.PolarisRealm; -import org.apache.polaris.service.test.SnowmanCredentialsExtension; -import org.apache.polaris.service.test.SnowmanCredentialsExtension.SnowmanCredentials; -import org.apache.polaris.service.test.TestEnvironmentExtension; +import org.apache.polaris.service.auth.BasePolarisAuthenticator; +import org.apache.polaris.service.test.PolarisIntegrationTestHelper; import org.apache.polaris.service.types.NotificationRequest; import org.apache.polaris.service.types.NotificationType; import org.apache.polaris.service.types.TableUpdateNotification; import org.assertj.core.api.Assertions; import org.assertj.core.api.InstanceOfAssertFactories; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; -import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestInstance.Lifecycle; /** * Import the full core Iceberg catalog tests by hitting the REST service via the RESTCatalog * client. */ -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class, - SnowmanCredentialsExtension.class -}) +@QuarkusTest +@TestInstance(Lifecycle.PER_CLASS) public class PolarisRestCatalogIntegrationTest extends CatalogTests { private static final String TEST_ROLE_ARN = Optional.ofNullable(System.getenv("INTEGRATION_TEST_ROLE_ARN")) @@ -116,66 +102,29 @@ public class PolarisRestCatalogIntegrationTest extends CatalogTests private static final String S3_BUCKET_BASE = Optional.ofNullable(System.getenv("INTEGRATION_TEST_S3_PATH")) .orElse("file:///tmp/buckets/my-bucket"); - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config( - "server.adminConnectors[0].port", "0")); // Bind to random port to support parallelism protected static final String VIEW_QUERY = "select * from ns1.layer1_table"; private RESTCatalog restCatalog; private String currentCatalogName; - private String userToken; - private String realm; private final String catalogBaseLocation = S3_BUCKET_BASE + "/" + System.getenv("USER") + "/path/to/data"; - private static final String[] DEFAULT_CATALOG_PROPERTIES = { - "allow.unstructured.table.location", "true", - "allow.external.table.location", "true" - }; + @Inject PolarisIntegrationTestHelper testHelper; - @Retention(RetentionPolicy.RUNTIME) - private @interface CatalogConfig { - Catalog.TypeEnum value() default Catalog.TypeEnum.INTERNAL; - - String[] properties() default { - "allow.unstructured.table.location", "true", - "allow.external.table.location", "true" - }; - } - - @Retention(RetentionPolicy.RUNTIME) - private @interface RestCatalogConfig { - String[] value() default {}; + @BeforeAll + public void setUp(TestInfo testInfo) { + testHelper.setUp(testInfo); } - @BeforeAll - public static void setup(@PolarisRealm String realm) throws IOException { - // Set up test location - PolarisConnectionExtension.createTestDir(realm); + @AfterAll + public void tearDown() { + testHelper.tearDown(); } @BeforeEach - public void before( - TestInfo testInfo, - PolarisToken adminToken, - SnowmanCredentials snowmanCredentials, - @PolarisRealm String realm) { - this.realm = realm; - userToken = - TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), - snowmanCredentials.clientId(), - snowmanCredentials.clientSecret(), - realm); + void before(TestInfo testInfo) { testInfo .getTestMethod() .ifPresent( @@ -189,27 +138,21 @@ public void before( .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) .setAllowedLocations(List.of("s3://my-old-bucket/path/to/data")) .build(); - Optional catalogConfig = - testInfo - .getTestMethod() - .flatMap(m -> Optional.ofNullable(m.getAnnotation(CatalogConfig.class))); - org.apache.polaris.core.admin.model.CatalogProperties.Builder catalogPropsBuilder = - org.apache.polaris.core.admin.model.CatalogProperties.builder( - catalogBaseLocation); - String[] properties = - catalogConfig.map(CatalogConfig::properties).orElse(DEFAULT_CATALOG_PROPERTIES); - for (int i = 0; i < properties.length; i += 2) { - catalogPropsBuilder.addProperty(properties[i], properties[i + 1]); - } + org.apache.polaris.core.admin.model.CatalogProperties.builder(catalogBaseLocation) + .addProperty( + PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), + "true") + .addProperty( + PolarisConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), + "true"); if (!S3_BUCKET_BASE.startsWith("file:/")) { catalogPropsBuilder.addProperty( CatalogEntity.REPLACE_NEW_LOCATION_PREFIX_WITH_CATALOG_DEFAULT_KEY, "file:"); } Catalog catalog = PolarisCatalog.builder() - .setType( - catalogConfig.map(CatalogConfig::value).orElse(Catalog.TypeEnum.INTERNAL)) + .setType(Catalog.TypeEnum.INTERNAL) .setName(currentCatalogName) .setProperties(catalogPropsBuilder.build()) .setStorageConfigInfo( @@ -218,31 +161,128 @@ public void before( StorageConfigInfo.StorageTypeEnum.FILE, List.of("file://")) : awsConfigModel) .build(); + try (Response response = + testHelper + .client + .target( + String.format( + "http://localhost:%d/api/management/v1/catalogs", + testHelper.localPort)) + .request("application/json") + .header("Authorization", "Bearer " + testHelper.adminToken) + .header(REALM_PROPERTY_KEY, testHelper.realm) + .post(Entity.json(catalog))) { + assertThat(response) + .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); + } - Optional restCatalogConfig = - testInfo - .getTestMethod() - .flatMap( - m -> - Optional.ofNullable( - m.getAnnotation( - PolarisRestCatalogIntegrationTest.RestCatalogConfig.class))); - ImmutableMap.Builder extraPropertiesBuilder = - ImmutableMap.builder(); - restCatalogConfig.ifPresent( - config -> { - for (int i = 0; i < config.value().length; i += 2) { - extraPropertiesBuilder.put(config.value()[i], config.value()[i + 1]); - } - }); - restCatalog = - TestUtil.createSnowmanManagedCatalog( - EXT, - adminToken, - snowmanCredentials, - realm, - catalog, - extraPropertiesBuilder.build()); + // Create a new CatalogRole that has CATALOG_MANAGE_CONTENT and CATALOG_MANAGE_ACCESS + CatalogRole newRole = new CatalogRole("custom-admin"); + try (Response response = + testHelper + .client + .target( + String.format( + "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles", + testHelper.localPort, currentCatalogName)) + .request("application/json") + .header("Authorization", "Bearer " + testHelper.adminToken) + .header(REALM_PROPERTY_KEY, testHelper.realm) + .post(Entity.json(newRole))) { + assertThat(response) + .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); + } + CatalogGrant grantResource = + new CatalogGrant( + CatalogPrivilege.CATALOG_MANAGE_CONTENT, GrantResource.TypeEnum.CATALOG); + try (Response response = + testHelper + .client + .target( + String.format( + "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/custom-admin/grants", + testHelper.localPort, currentCatalogName)) + .request("application/json") + .header("Authorization", "Bearer " + testHelper.adminToken) + .header(REALM_PROPERTY_KEY, testHelper.realm) + .put(Entity.json(grantResource))) { + assertThat(response) + .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); + } + CatalogGrant grantAccessResource = + new CatalogGrant( + CatalogPrivilege.CATALOG_MANAGE_ACCESS, GrantResource.TypeEnum.CATALOG); + try (Response response = + testHelper + .client + .target( + String.format( + "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/custom-admin/grants", + testHelper.localPort, currentCatalogName)) + .request("application/json") + .header("Authorization", "Bearer " + testHelper.adminToken) + .header(REALM_PROPERTY_KEY, testHelper.realm) + .put(Entity.json(grantAccessResource))) { + assertThat(response) + .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); + } + + // Assign this new CatalogRole to the service_admin PrincipalRole + try (Response response = + testHelper + .client + .target( + String.format( + "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/custom-admin", + testHelper.localPort, currentCatalogName)) + .request("application/json") + .header("Authorization", "Bearer " + testHelper.adminToken) + .header(REALM_PROPERTY_KEY, testHelper.realm) + .get()) { + assertThat(response) + .returns(Response.Status.OK.getStatusCode(), Response::getStatus); + CatalogRole catalogRole = response.readEntity(CatalogRole.class); + try (Response assignResponse = + testHelper + .client + .target( + String.format( + "http://localhost:%d/api/management/v1/principal-roles/catalog-admin/catalog-roles/%s", + testHelper.localPort, currentCatalogName)) + .request("application/json") + .header("Authorization", "Bearer " + testHelper.adminToken) + .header(REALM_PROPERTY_KEY, testHelper.realm) + .put(Entity.json(catalogRole))) { + assertThat(assignResponse) + .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); + } + } + + SessionCatalog.SessionContext context = SessionCatalog.SessionContext.createEmpty(); + this.restCatalog = + new RESTCatalog( + context, + (config) -> + HTTPClient.builder(config) + .uri(config.get(CatalogProperties.URI)) + .build()); + this.restCatalog.initialize( + "polaris", + ImmutableMap.of( + CatalogProperties.URI, + "http://localhost:" + testHelper.localPort + "/api/catalog", + OAuth2Properties.CREDENTIAL, + testHelper.snowmanCredentials.clientId() + + ":" + + testHelper.snowmanCredentials.clientSecret(), + OAuth2Properties.SCOPE, + BasePolarisAuthenticator.PRINCIPAL_ROLE_ALL, + CatalogProperties.FILE_IO_IMPL, + "org.apache.iceberg.inmemory.InMemoryFileIO", + "warehouse", + currentCatalogName, + "header." + REALM_PROPERTY_KEY, + testHelper.realm)); }); } @@ -274,14 +314,15 @@ protected boolean overridesRequestedLocation() { private void createCatalogRole(String catalogRoleName) { CatalogRole catalogRole = new CatalogRole(catalogRoleName); try (Response response = - EXT.client() + testHelper + .client .target( String.format( "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles", - EXT.getLocalPort(), currentCatalogName)) + testHelper.localPort, currentCatalogName)) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + testHelper.userToken) + .header(REALM_PROPERTY_KEY, testHelper.realm) .post(Entity.json(catalogRole))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } @@ -289,14 +330,15 @@ private void createCatalogRole(String catalogRoleName) { private void addGrant(String catalogRoleName, GrantResource grant) { try (Response response = - EXT.client() + testHelper + .client .target( String.format( "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/%s/grants", - EXT.getLocalPort(), currentCatalogName, catalogRoleName)) + testHelper.localPort, currentCatalogName, catalogRoleName)) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + testHelper.userToken) + .header(REALM_PROPERTY_KEY, testHelper.realm) .put(Entity.json(grant))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } @@ -424,14 +466,15 @@ public void testListGrantsOnCatalogObjectsToCatalogRoles() { // List grants for catalogrole1 try (Response response = - EXT.client() + testHelper + .client .target( String.format( "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/%s/grants", - EXT.getLocalPort(), currentCatalogName, "catalogrole1")) + testHelper.localPort, currentCatalogName, "catalogrole1")) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + testHelper.userToken) + .header(REALM_PROPERTY_KEY, testHelper.realm) .get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -443,14 +486,15 @@ public void testListGrantsOnCatalogObjectsToCatalogRoles() { // List grants for catalogrole2 try (Response response = - EXT.client() + testHelper + .client .target( String.format( "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/%s/grants", - EXT.getLocalPort(), currentCatalogName, "catalogrole2")) + testHelper.localPort, currentCatalogName, "catalogrole2")) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + testHelper.userToken) + .header(REALM_PROPERTY_KEY, testHelper.realm) .get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -494,14 +538,15 @@ public void testListGrantsAfterRename() { GrantResource.TypeEnum.TABLE); try (Response response = - EXT.client() + testHelper + .client .target( String.format( "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/%s/grants", - EXT.getLocalPort(), currentCatalogName, "catalogrole1")) + testHelper.localPort, currentCatalogName, "catalogrole1")) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + testHelper.userToken) + .header(REALM_PROPERTY_KEY, testHelper.realm) .get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -513,16 +558,17 @@ public void testListGrantsAfterRename() { } @Test - public void testCreateTableWithOverriddenBaseLocation(PolarisToken adminToken) { + public void testCreateTableWithOverriddenBaseLocation() { try (Response response = - EXT.client() + testHelper + .client .target( String.format( "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), currentCatalogName)) + testHelper.localPort, currentCatalogName)) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + testHelper.adminToken) + .header(REALM_PROPERTY_KEY, testHelper.realm) .get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); Catalog catalog = response.readEntity(Catalog.class); @@ -530,14 +576,15 @@ public void testCreateTableWithOverriddenBaseLocation(PolarisToken adminToken) { catalogProps.put( PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "false"); try (Response updateResponse = - EXT.client() + testHelper + .client .target( String.format( "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), catalog.getName())) + testHelper.localPort, catalog.getName())) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + testHelper.adminToken) + .header(REALM_PROPERTY_KEY, testHelper.realm) .put( Entity.json( new UpdateCatalogRequest( @@ -569,17 +616,17 @@ public void testCreateTableWithOverriddenBaseLocation(PolarisToken adminToken) { } @Test - public void testCreateTableWithOverriddenBaseLocationCannotOverlapSibling( - PolarisToken adminToken) { + public void testCreateTableWithOverriddenBaseLocationCannotOverlapSibling() { try (Response response = - EXT.client() + testHelper + .client .target( String.format( "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), currentCatalogName)) + testHelper.localPort, currentCatalogName)) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + testHelper.adminToken) + .header(REALM_PROPERTY_KEY, testHelper.realm) .get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); Catalog catalog = response.readEntity(Catalog.class); @@ -587,14 +634,15 @@ public void testCreateTableWithOverriddenBaseLocationCannotOverlapSibling( catalogProps.put( PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "false"); try (Response updateResponse = - EXT.client() + testHelper + .client .target( String.format( "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), catalog.getName())) + testHelper.localPort, catalog.getName())) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + testHelper.adminToken) + .header(REALM_PROPERTY_KEY, testHelper.realm) .put( Entity.json( new UpdateCatalogRequest( @@ -635,17 +683,17 @@ public void testCreateTableWithOverriddenBaseLocationCannotOverlapSibling( } @Test - public void testCreateTableWithOverriddenBaseLocationMustResideInNsDirectory( - PolarisToken adminToken) { + public void testCreateTableWithOverriddenBaseLocationMustResideInNsDirectory() { try (Response response = - EXT.client() + testHelper + .client .target( String.format( "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), currentCatalogName)) + testHelper.localPort, currentCatalogName)) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + testHelper.adminToken) + .header(REALM_PROPERTY_KEY, testHelper.realm) .get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); Catalog catalog = response.readEntity(Catalog.class); @@ -653,14 +701,15 @@ public void testCreateTableWithOverriddenBaseLocationMustResideInNsDirectory( catalogProps.put( PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "false"); try (Response updateResponse = - EXT.client() + testHelper + .client .target( String.format( "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), catalog.getName())) + testHelper.localPort, catalog.getName())) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + testHelper.adminToken) + .header(REALM_PROPERTY_KEY, testHelper.realm) .put( Entity.json( new UpdateCatalogRequest( @@ -688,106 +737,6 @@ public void testCreateTableWithOverriddenBaseLocationMustResideInNsDirectory( .isInstanceOf(ForbiddenException.class); } - /** - * Create an EXTERNAL catalog. The test configuration, by default, disables access delegation for - * EXTERNAL catalogs, so register a table and try to load it with the REST client configured to - * try to fetch vended credentials. Expect a ForbiddenException. - */ - @CatalogConfig(Catalog.TypeEnum.EXTERNAL) - @RestCatalogConfig({"header.X-Iceberg-Access-Delegation", "vended-credentials"}) - @Test - public void testLoadTableWithAccessDelegationForExternalCatalogWithConfigDisabled() { - Namespace ns1 = Namespace.of("ns1"); - restCatalog.createNamespace(ns1); - TableMetadata tableMetadata = - TableMetadata.newTableMetadata( - new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))), - PartitionSpec.unpartitioned(), - "file:///tmp/ns1/my_table", - Map.of()); - try (ResolvingFileIO resolvingFileIO = new ResolvingFileIO()) { - resolvingFileIO.initialize(Map.of()); - resolvingFileIO.setConf(new Configuration()); - String fileLocation = "file:///tmp/ns1/my_table/metadata/v1.metadata.json"; - TableMetadataParser.write(tableMetadata, resolvingFileIO.newOutputFile(fileLocation)); - restCatalog.registerTable(TableIdentifier.of(ns1, "my_table"), fileLocation); - try { - Assertions.assertThatThrownBy( - () -> restCatalog.loadTable(TableIdentifier.of(ns1, "my_table"))) - .isInstanceOf(ForbiddenException.class) - .hasMessageContaining("Access Delegation is not enabled for this catalog") - .hasMessageContaining( - PolarisConfiguration.ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING.catalogConfig()); - } finally { - resolvingFileIO.deleteFile(fileLocation); - } - } - } - - /** - * Create an EXTERNAL catalog. The test configuration, by default, disables access delegation for - * EXTERNAL catalogs. Register a table and attempt to load it WITHOUT access delegation. This - * should succeed. - */ - @CatalogConfig(Catalog.TypeEnum.EXTERNAL) - @Test - public void testLoadTableWithoutAccessDelegationForExternalCatalogWithConfigDisabled() { - Namespace ns1 = Namespace.of("ns1"); - restCatalog.createNamespace(ns1); - TableMetadata tableMetadata = - TableMetadata.newTableMetadata( - new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))), - PartitionSpec.unpartitioned(), - "file:///tmp/ns1/my_table", - Map.of()); - try (ResolvingFileIO resolvingFileIO = new ResolvingFileIO()) { - resolvingFileIO.initialize(Map.of()); - resolvingFileIO.setConf(new Configuration()); - String fileLocation = "file:///tmp/ns1/my_table/metadata/v1.metadata.json"; - TableMetadataParser.write(tableMetadata, resolvingFileIO.newOutputFile(fileLocation)); - restCatalog.registerTable(TableIdentifier.of(ns1, "my_table"), fileLocation); - try { - restCatalog.loadTable(TableIdentifier.of(ns1, "my_table")); - } finally { - resolvingFileIO.deleteFile(fileLocation); - } - } - } - - /** - * Create an EXTERNAL catalog. The test configuration, by default, disables access delegation for - * EXTERNAL catalogs. However, we set enable.credential.vending to true - * for this catalog, enabling it. Register a table and attempt to load it WITH access delegation. - * This should succeed. - */ - @CatalogConfig( - value = Catalog.TypeEnum.EXTERNAL, - properties = {"enable.credential.vending", "true"}) - @RestCatalogConfig({"header.X-Iceberg-Access-Delegation", "vended-credentials"}) - @Test - public void testLoadTableWithAccessDelegationForExternalCatalogWithConfigEnabledForCatalog() { - Namespace ns1 = Namespace.of("ns1"); - restCatalog.createNamespace(ns1); - TableMetadata tableMetadata = - TableMetadata.newTableMetadata( - new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))), - PartitionSpec.unpartitioned(), - "file:///tmp/ns1/my_table", - Map.of()); - try (ResolvingFileIO resolvingFileIO = new ResolvingFileIO()) { - resolvingFileIO.initialize(Map.of()); - resolvingFileIO.setConf(new Configuration()); - String fileLocation = "file:///tmp/ns1/my_table/metadata/v1.metadata.json"; - TableMetadataParser.write(tableMetadata, resolvingFileIO.newOutputFile(fileLocation)); - restCatalog.registerTable(TableIdentifier.of(ns1, "my_table"), fileLocation); - try { - restCatalog.loadTable(TableIdentifier.of(ns1, "my_table")); - } finally { - resolvingFileIO.deleteFile(fileLocation); - } - } - } - @Test public void testSendNotificationInternalCatalog() { NotificationRequest notification = new NotificationRequest(); @@ -803,13 +752,14 @@ public void testSendNotificationInternalCatalog() { String notificationUrl = String.format( "http://localhost:%d/api/catalog/v1/%s/namespaces/ns1/tables/tbl1/notifications", - EXT.getLocalPort(), currentCatalogName); + testHelper.localPort, currentCatalogName); try (Response response = - EXT.client() + testHelper + .client .target(notificationUrl) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + testHelper.userToken) + .header(REALM_PROPERTY_KEY, testHelper.realm) .post(Entity.json(notification))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus) @@ -820,11 +770,12 @@ public void testSendNotificationInternalCatalog() { // NotificationType.VALIDATE should also surface the same error. notification.setNotificationType(NotificationType.VALIDATE); try (Response response = - EXT.client() + testHelper + .client .target(notificationUrl) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + testHelper.userToken) + .header(REALM_PROPERTY_KEY, testHelper.realm) .post(Entity.json(notification))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus) diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogViewAwsIntegrationTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogViewAwsIntegrationTest.java index 8b357c7f6..aa1eb6ae2 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogViewAwsIntegrationTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogViewAwsIntegrationTest.java @@ -18,6 +18,7 @@ */ package org.apache.polaris.service.catalog; +import io.quarkus.test.junit.QuarkusTest; import java.util.List; import java.util.Optional; import java.util.stream.Stream; @@ -26,6 +27,7 @@ import org.assertj.core.util.Strings; /** Runs PolarisRestCatalogViewIntegrationTest on AWS. */ +@QuarkusTest public class PolarisRestCatalogViewAwsIntegrationTest extends PolarisRestCatalogViewIntegrationTest { public static final String ROLE_ARN = diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogViewAzureIntegrationTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogViewAzureIntegrationTest.java index 475cb166f..e4a57faa8 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogViewAzureIntegrationTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogViewAzureIntegrationTest.java @@ -18,6 +18,7 @@ */ package org.apache.polaris.service.catalog; +import io.quarkus.test.junit.QuarkusTest; import java.util.List; import java.util.stream.Stream; import org.apache.polaris.core.admin.model.AzureStorageConfigInfo; @@ -25,6 +26,7 @@ import org.assertj.core.util.Strings; /** Runs PolarisRestCatalogViewIntegrationTest on Azure. */ +@QuarkusTest public class PolarisRestCatalogViewAzureIntegrationTest extends PolarisRestCatalogViewIntegrationTest { public static final String TENANT_ID = System.getenv("INTEGRATION_TEST_AZURE_TENANT_ID"); diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogViewFileIntegrationTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogViewFileIntegrationTest.java index fbf1714f0..85cfa8500 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogViewFileIntegrationTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogViewFileIntegrationTest.java @@ -18,11 +18,13 @@ */ package org.apache.polaris.service.catalog; +import io.quarkus.test.junit.QuarkusTest; import java.util.List; import org.apache.polaris.core.admin.model.FileStorageConfigInfo; import org.apache.polaris.core.admin.model.StorageConfigInfo; /** Runs PolarisRestCatalogViewIntegrationTest on the local filesystem. */ +@QuarkusTest public class PolarisRestCatalogViewFileIntegrationTest extends PolarisRestCatalogViewIntegrationTest { public static final String BASE_LOCATION = "file:///tmp/buckets/my-bucket"; diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogViewGcpIntegrationTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogViewGcpIntegrationTest.java index f59be9c09..57a031e52 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogViewGcpIntegrationTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogViewGcpIntegrationTest.java @@ -18,6 +18,7 @@ */ package org.apache.polaris.service.catalog; +import io.quarkus.test.junit.QuarkusTest; import java.util.List; import java.util.stream.Stream; import org.apache.polaris.core.admin.model.GcpStorageConfigInfo; @@ -25,6 +26,7 @@ import org.assertj.core.util.Strings; /** Runs PolarisRestCatalogViewIntegrationTest on GCP. */ +@QuarkusTest public class PolarisRestCatalogViewGcpIntegrationTest extends PolarisRestCatalogViewIntegrationTest { public static final String SERVICE_ACCOUNT = diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogViewIntegrationTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogViewIntegrationTest.java index e9897a1f7..c79274ae2 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogViewIntegrationTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogViewIntegrationTest.java @@ -18,14 +18,12 @@ */ package org.apache.polaris.service.catalog; -import static org.apache.polaris.service.context.DefaultContextResolver.REALM_PROPERTY_KEY; +import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; +import jakarta.inject.Inject; import jakarta.ws.rs.core.Response; -import java.io.IOException; +import java.lang.reflect.Field; +import java.nio.file.Path; import java.util.Map; import org.apache.iceberg.rest.RESTCatalog; import org.apache.iceberg.view.ViewCatalogTests; @@ -34,75 +32,60 @@ import org.apache.polaris.core.admin.model.PolarisCatalog; import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.entity.CatalogEntity; -import org.apache.polaris.service.PolarisApplication; -import org.apache.polaris.service.config.PolarisApplicationConfig; -import org.apache.polaris.service.test.PolarisConnectionExtension; -import org.apache.polaris.service.test.PolarisConnectionExtension.PolarisToken; -import org.apache.polaris.service.test.PolarisRealm; -import org.apache.polaris.service.test.SnowmanCredentialsExtension; -import org.apache.polaris.service.test.SnowmanCredentialsExtension.SnowmanCredentials; -import org.apache.polaris.service.test.TestEnvironment; -import org.apache.polaris.service.test.TestEnvironmentExtension; -import org.junit.jupiter.api.Assumptions; +import org.apache.polaris.service.test.PolarisIntegrationTestHelper; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.TestInfo; -import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.io.TempDir; /** * Import the full core Iceberg catalog tests by hitting the REST service via the RESTCatalog * client. */ -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class, - SnowmanCredentialsExtension.class -}) +@TestInstance(TestInstance.Lifecycle.PER_CLASS) public abstract class PolarisRestCatalogViewIntegrationTest extends ViewCatalogTests { - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config( - "server.adminConnectors[0].port", "0")); // Bind to random port to support parallelism private RESTCatalog restCatalog; + @Inject PolarisIntegrationTestHelper testHelper; + @BeforeAll - public static void setup(@PolarisRealm String realm) throws IOException { - // Set up test location - PolarisConnectionExtension.createTestDir(realm); + public void setUp(TestInfo testInfo) { + testHelper.setUp(testInfo); } - @BeforeEach - public void before( - TestInfo testInfo, - PolarisToken adminToken, - SnowmanCredentials snowmanCredentials, - @PolarisRealm String realm, - TestEnvironment testEnv) { + @AfterAll + public void tearDown() { + testHelper.tearDown(); + } - Assumptions.assumeFalse(shouldSkip()); + @BeforeEach + public void setUpTempDir(@TempDir Path tempDir) throws Exception { + // see https://github.com/quarkusio/quarkus/issues/13261 + Field field = ViewCatalogTests.class.getDeclaredField("tempDir"); + field.setAccessible(true); + field.set(this, tempDir); + } - String userToken = adminToken.token(); + @BeforeEach + void before(TestInfo testInfo) { testInfo .getTestMethod() .ifPresent( method -> { - String catalogName = method.getName() + testEnv.testId(); + String catalogName = method.getName(); try (Response response = - testEnv - .apiClient() + testHelper + .client .target( String.format( - "%s/api/management/v1/catalogs/%s", testEnv.baseUri(), catalogName)) + "http://localhost:%d/api/management/v1/catalogs/%s", + testHelper.localPort, catalogName)) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + testHelper.adminToken) + .header(REALM_PROPERTY_KEY, testHelper.realm) .get()) { if (response.getStatus() == Response.Status.OK.getStatusCode()) { // Already exists! Must be in a parameterized test. @@ -138,15 +121,7 @@ public void before( .setProperties(props) .setStorageConfigInfo(storageConfig) .build(); - restCatalog = - TestUtil.createSnowmanManagedCatalog( - testEnv.apiClient(), - testEnv.baseUri().toString(), - adminToken, - snowmanCredentials, - realm, - catalog, - Map.of()); + restCatalog = TestUtil.createSnowmanManagedCatalog(testHelper, catalog, Map.of()); }); } diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisSparkIntegrationTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisSparkIntegrationTest.java index 35bb96096..ffd032006 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisSparkIntegrationTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisSparkIntegrationTest.java @@ -18,18 +18,15 @@ */ package org.apache.polaris.service.catalog; -import static org.apache.polaris.service.context.DefaultContextResolver.REALM_PROPERTY_KEY; +import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.adobe.testing.s3mock.testcontainers.S3MockContainer; -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; +import io.quarkus.test.junit.QuarkusTest; +import jakarta.inject.Inject; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.core.Response; -import java.io.IOException; import java.time.Instant; import java.util.List; import java.util.Map; @@ -41,11 +38,7 @@ import org.apache.polaris.core.admin.model.ExternalCatalog; import org.apache.polaris.core.admin.model.PolarisCatalog; import org.apache.polaris.core.admin.model.StorageConfigInfo; -import org.apache.polaris.service.PolarisApplication; -import org.apache.polaris.service.config.PolarisApplicationConfig; -import org.apache.polaris.service.test.PolarisConnectionExtension; -import org.apache.polaris.service.test.PolarisRealm; -import org.apache.polaris.service.test.TestEnvironmentExtension; +import org.apache.polaris.service.test.PolarisIntegrationTestHelper; import org.apache.polaris.service.types.NotificationRequest; import org.apache.polaris.service.types.NotificationType; import org.apache.polaris.service.types.TableUpdateNotification; @@ -58,52 +51,42 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestInstance; import org.slf4j.LoggerFactory; -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class -}) +@QuarkusTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) public class PolarisSparkIntegrationTest { - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config( - "server.adminConnectors[0].port", "0")); // Bind to random port to support parallelism public static final String CATALOG_NAME = "mycatalog"; public static final String EXTERNAL_CATALOG_NAME = "external_catalog"; - private static final S3MockContainer s3Container = + + private final S3MockContainer s3Container = new S3MockContainer("3.11.0").withInitialBuckets("my-bucket,my-old-bucket"); - private static PolarisConnectionExtension.PolarisToken polarisToken; - private static SparkSession spark; - private String realm; + + private SparkSession spark; + + @Inject PolarisIntegrationTestHelper testHelper; @BeforeAll - public static void setup( - PolarisConnectionExtension.PolarisToken polarisToken, @PolarisRealm String realm) - throws IOException { + public void setUp(TestInfo testInfo) { s3Container.start(); - PolarisSparkIntegrationTest.polarisToken = polarisToken; + testHelper.setUp(testInfo); + } - // Set up test location - PolarisConnectionExtension.createTestDir(realm); + @AfterAll + public void tearDown() { + testHelper.tearDown(); } @AfterAll - public static void cleanup() { + public void cleanup() { s3Container.stop(); } @BeforeEach - public void before(@PolarisRealm String realm) { - this.realm = realm; + public void before() { AwsStorageConfigInfo awsConfigModel = AwsStorageConfigInfo.builder() .setRoleArn("arn:aws:iam::123456789012:role/my-role") @@ -140,12 +123,14 @@ public void before(@PolarisRealm String realm) { .build(); try (Response response = - EXT.client() + testHelper + .client .target( - String.format("http://localhost:%d/api/management/v1/catalogs", EXT.getLocalPort())) + String.format( + "http://localhost:%d/api/management/v1/catalogs", testHelper.localPort)) .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "BEARER " + testHelper.adminToken) + .header(REALM_PROPERTY_KEY, testHelper.realm) .post(Entity.json(catalog))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } @@ -178,12 +163,14 @@ public void before(@PolarisRealm String realm) { .setRemoteUrl("http://dummy_url") .build(); try (Response response = - EXT.client() + testHelper + .client .target( - String.format("http://localhost:%d/api/management/v1/catalogs", EXT.getLocalPort())) + String.format( + "http://localhost:%d/api/management/v1/catalogs", testHelper.localPort)) .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "BEARER " + testHelper.adminToken) + .header(REALM_PROPERTY_KEY, testHelper.realm) .post(Entity.json(externalCatalog))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } @@ -215,11 +202,11 @@ private SparkSession.Builder withCatalog(SparkSession.Builder builder, String ca .config(String.format("spark.sql.catalog.%s.type", catalogName), "rest") .config( String.format("spark.sql.catalog.%s.uri", catalogName), - "http://localhost:" + EXT.getLocalPort() + "/api/catalog") + "http://localhost:" + testHelper.localPort + "/api/catalog") .config(String.format("spark.sql.catalog.%s.warehouse", catalogName), catalogName) .config(String.format("spark.sql.catalog.%s.scope", catalogName), "PRINCIPAL_ROLE:ALL") - .config(String.format("spark.sql.catalog.%s.header.realm", catalogName), realm) - .config(String.format("spark.sql.catalog.%s.token", catalogName), polarisToken.token()) + .config(String.format("spark.sql.catalog.%s.header.realm", catalogName), testHelper.realm) + .config(String.format("spark.sql.catalog.%s.token", catalogName), testHelper.adminToken) .config(String.format("spark.sql.catalog.%s.s3.access-key-id", catalogName), "fakekey") .config( String.format("spark.sql.catalog.%s.s3.secret-access-key", catalogName), "fakesecret") @@ -254,14 +241,15 @@ private void cleanupCatalog(String catalogName) { onSpark("DROP NAMESPACE " + namespace.getString(0)); } try (Response response = - EXT.client() + testHelper + .client .target( String.format( "http://localhost:%d/api/management/v1/catalogs/" + catalogName, - EXT.getLocalPort())) + testHelper.localPort)) .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "BEARER " + testHelper.adminToken) + .header(REALM_PROPERTY_KEY, testHelper.realm) .delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -303,16 +291,17 @@ public void testCreateAndUpdateExternalTable() { LoadTableResponse tableResponse = loadTable(CATALOG_NAME, "ns1", "tb1"); try (Response registerResponse = - EXT.client() + testHelper + .client .target( String.format( "http://localhost:%d/api/catalog/v1/" + EXTERNAL_CATALOG_NAME + "/namespaces/externalns1/register", - EXT.getLocalPort())) + testHelper.localPort)) .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "BEARER " + testHelper.adminToken) + .header(REALM_PROPERTY_KEY, testHelper.realm) .post( Entity.json( ImmutableRegisterTableRequest.builder() @@ -344,14 +333,15 @@ public void testCreateAndUpdateExternalTable() { notificationRequest.setPayload(updateNotification); notificationRequest.setNotificationType(NotificationType.UPDATE); try (Response notifyResponse = - EXT.client() + testHelper + .client .target( String.format( "http://localhost:%d/api/catalog/v1/%s/namespaces/externalns1/tables/mytb1/notifications", - EXT.getLocalPort(), EXTERNAL_CATALOG_NAME)) + testHelper.localPort, EXTERNAL_CATALOG_NAME)) .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "BEARER " + testHelper.adminToken) + .header(REALM_PROPERTY_KEY, testHelper.realm) .post(Entity.json(notificationRequest))) { assertThat(notifyResponse) .returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); @@ -378,21 +368,22 @@ public void testCreateView() { private LoadTableResponse loadTable(String catalog, String namespace, String table) { try (Response response = - EXT.client() + testHelper + .client .target( String.format( "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/tables/%s", - EXT.getLocalPort(), catalog, namespace, table)) + testHelper.localPort, catalog, namespace, table)) .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "BEARER " + testHelper.adminToken) + .header(REALM_PROPERTY_KEY, testHelper.realm) .get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); return response.readEntity(LoadTableResponse.class); } } - private static Dataset onSpark(@Language("SQL") String sql) { + private Dataset onSpark(@Language("SQL") String sql) { return spark.sql(sql); } } diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/TestUtil.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/TestUtil.java index 37c13f4a1..900b87ac5 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/TestUtil.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/TestUtil.java @@ -18,12 +18,10 @@ */ package org.apache.polaris.service.catalog; -import static org.apache.polaris.service.context.DefaultContextResolver.REALM_PROPERTY_KEY; +import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; import static org.assertj.core.api.Assertions.assertThat; import com.google.common.collect.ImmutableMap; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import jakarta.ws.rs.client.Client; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.core.Response; import java.util.Map; @@ -38,26 +36,17 @@ import org.apache.polaris.core.admin.model.CatalogRole; import org.apache.polaris.core.admin.model.GrantResource; import org.apache.polaris.service.auth.BasePolarisAuthenticator; -import org.apache.polaris.service.config.PolarisApplicationConfig; -import org.apache.polaris.service.test.PolarisConnectionExtension; -import org.apache.polaris.service.test.SnowmanCredentialsExtension; +import org.apache.polaris.service.test.PolarisIntegrationTestHelper; /** Test utilities for catalog tests */ public class TestUtil { /** Performs createSnowmanManagedCatalog() on a Dropwizard instance of Polaris */ public static RESTCatalog createSnowmanManagedCatalog( - DropwizardAppExtension EXT, - PolarisConnectionExtension.PolarisToken adminToken, - SnowmanCredentialsExtension.SnowmanCredentials snowmanCredentials, - String realm, - Catalog catalog, - Map extraProperties) { + PolarisIntegrationTestHelper helper, Catalog catalog, Map extraProperties) { return createSnowmanManagedCatalog( - EXT.client(), - String.format("http://localhost:%d", EXT.getLocalPort()), - adminToken, - snowmanCredentials, - realm, + helper, + String.format("http://localhost:%d", helper.getLocalPort()), + helper.realm, catalog, extraProperties); } @@ -69,19 +58,18 @@ public static RESTCatalog createSnowmanManagedCatalog( * @return A client to interact with the catalog. */ public static RESTCatalog createSnowmanManagedCatalog( - Client client, + PolarisIntegrationTestHelper helper, String baseUrl, - PolarisConnectionExtension.PolarisToken adminToken, - SnowmanCredentialsExtension.SnowmanCredentials snowmanCredentials, String realm, Catalog catalog, Map extraProperties) { String currentCatalogName = catalog.getName(); try (Response response = - client + helper + .client .target(String.format("%s/api/management/v1/catalogs", baseUrl)) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) + .header("Authorization", "Bearer " + helper.adminToken) .header(REALM_PROPERTY_KEY, realm) .post(Entity.json(catalog))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -90,12 +78,13 @@ public static RESTCatalog createSnowmanManagedCatalog( // Create a new CatalogRole that has CATALOG_MANAGE_CONTENT and CATALOG_MANAGE_ACCESS CatalogRole newRole = new CatalogRole("custom-admin"); try (Response response = - client + helper + .client .target( String.format( "%s/api/management/v1/catalogs/%s/catalog-roles", baseUrl, currentCatalogName)) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) + .header("Authorization", "Bearer " + helper.adminToken) .header(REALM_PROPERTY_KEY, realm) .post(Entity.json(newRole))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -103,13 +92,14 @@ public static RESTCatalog createSnowmanManagedCatalog( CatalogGrant grantResource = new CatalogGrant(CatalogPrivilege.CATALOG_MANAGE_CONTENT, GrantResource.TypeEnum.CATALOG); try (Response response = - client + helper + .client .target( String.format( "%s/api/management/v1/catalogs/%s/catalog-roles/custom-admin/grants", baseUrl, currentCatalogName)) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) + .header("Authorization", "Bearer " + helper.adminToken) .header(REALM_PROPERTY_KEY, realm) .put(Entity.json(grantResource))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -117,13 +107,14 @@ public static RESTCatalog createSnowmanManagedCatalog( CatalogGrant grantAccessResource = new CatalogGrant(CatalogPrivilege.CATALOG_MANAGE_ACCESS, GrantResource.TypeEnum.CATALOG); try (Response response = - client + helper + .client .target( String.format( "%s/api/management/v1/catalogs/%s/catalog-roles/custom-admin/grants", baseUrl, currentCatalogName)) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) + .header("Authorization", "Bearer " + helper.adminToken) .header(REALM_PROPERTY_KEY, realm) .put(Entity.json(grantAccessResource))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -131,27 +122,29 @@ public static RESTCatalog createSnowmanManagedCatalog( // Assign this new CatalogRole to the service_admin PrincipalRole try (Response response = - client + helper + .client .target( String.format( "%s/api/management/v1/catalogs/%s/catalog-roles/custom-admin", baseUrl, currentCatalogName)) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) + .header("Authorization", "Bearer " + helper.adminToken) .header(REALM_PROPERTY_KEY, realm) .get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); CatalogRole catalogRole = response.readEntity(CatalogRole.class); try (Response assignResponse = - client + helper + .client .target( String.format( "%s/api/management/v1/principal-roles/%s/catalog-roles/%s", baseUrl, - snowmanCredentials.identifier().principalRoleName(), + helper.snowmanCredentials.identifier().principalRoleName(), currentCatalogName)) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) + .header("Authorization", "Bearer " + helper.adminToken) .header(REALM_PROPERTY_KEY, realm) .put(Entity.json(catalogRole))) { assertThat(assignResponse) @@ -170,7 +163,9 @@ public static RESTCatalog createSnowmanManagedCatalog( .put(CatalogProperties.URI, baseUrl + "/api/catalog") .put( OAuth2Properties.CREDENTIAL, - snowmanCredentials.clientId() + ":" + snowmanCredentials.clientSecret()) + helper.snowmanCredentials.clientId() + + ":" + + helper.snowmanCredentials.clientSecret()) .put(OAuth2Properties.SCOPE, BasePolarisAuthenticator.PRINCIPAL_ROLE_ALL) .put(CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO") .put("warehouse", currentCatalogName) diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/io/FileIOIntegrationTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/io/FileIOIntegrationTest.java index 77317c000..97951aba4 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/io/FileIOIntegrationTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/io/FileIOIntegrationTest.java @@ -25,10 +25,9 @@ import com.azure.core.exception.AzureException; import com.google.cloud.storage.StorageException; import com.google.common.collect.Iterators; -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; +import io.quarkus.test.junit.QuarkusMock; +import io.quarkus.test.junit.QuarkusTest; +import jakarta.inject.Inject; import java.util.Collection; import java.util.Iterator; import java.util.List; @@ -51,52 +50,37 @@ import org.apache.polaris.core.admin.model.FileStorageConfigInfo; import org.apache.polaris.core.admin.model.PolarisCatalog; import org.apache.polaris.core.admin.model.StorageConfigInfo; -import org.apache.polaris.service.PolarisApplication; import org.apache.polaris.service.catalog.TestUtil; -import org.apache.polaris.service.config.PolarisApplicationConfig; import org.apache.polaris.service.exception.IcebergExceptionMapper; -import org.apache.polaris.service.test.PolarisConnectionExtension; -import org.apache.polaris.service.test.PolarisRealm; -import org.apache.polaris.service.test.SnowmanCredentialsExtension; -import org.apache.polaris.service.test.TestEnvironmentExtension; +import org.apache.polaris.service.test.PolarisIntegrationTestHelper; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestInstance.Lifecycle; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import software.amazon.awssdk.services.s3.model.S3Exception; /** Collection of File IO integration tests */ -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class, - SnowmanCredentialsExtension.class -}) +@QuarkusTest +@TestInstance(Lifecycle.PER_CLASS) public class FileIOIntegrationTest { - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config("server.adminConnectors[0].port", "0"), - ConfigOverride.config("featureConfiguration.MAX_FILE_IO_READ_BYTES", "10"), - ConfigOverride.config("io.factoryType", "test")); private static final String catalogBaseLocation = "file:/tmp/buckets/my-bucket/path/to/data"; - private static TestFileIOFactory ioFactory; + private static RESTCatalog restCatalog; private static Table table; + @Inject PolarisIntegrationTestHelper testHelper; + @Inject FileIOFactory ioFactory; + @BeforeAll - public static void beforeAll( - PolarisConnectionExtension.PolarisToken adminToken, - SnowmanCredentialsExtension.SnowmanCredentials snowmanCredentials, - @PolarisRealm String realm) { - ioFactory = (TestFileIOFactory) EXT.getConfiguration().getFileIOFactory(); + public void beforeAll(TestInfo testInfo) { + testHelper.setUp(testInfo); + QuarkusMock.installMockForType(new TestFileIOFactory(), FileIOFactory.class); FileStorageConfigInfo storageConfigInfo = FileStorageConfigInfo.builder() @@ -113,9 +97,7 @@ public static void beforeAll( .setStorageConfigInfo(storageConfigInfo) .build(); - restCatalog = - TestUtil.createSnowmanManagedCatalog( - EXT, adminToken, snowmanCredentials, realm, catalog, Map.of()); + restCatalog = TestUtil.createSnowmanManagedCatalog(testHelper, catalog, Map.of()); Namespace namespace = Namespace.of("myns"); restCatalog.createNamespace(namespace); @@ -129,6 +111,11 @@ public static void beforeAll( .create(); } + @AfterAll + public void tearDown() { + testHelper.tearDown(); + } + @Test void testGetLengthExceptionSupplier() { InMemoryFileIO inMemoryFileIO = new InMemoryFileIO(); @@ -154,11 +141,12 @@ void testGetLengthExceptionSupplier() { @ParameterizedTest @MethodSource("getIOExceptionTypeTestConfigs") void testIOExceptionExceptionTypes(int uniqueId, IOExceptionTypeTestConfig config) { - ioFactory.loadFileIOExceptionSupplier = config.loadFileIOExceptionSupplier; - ioFactory.newInputFileExceptionSupplier = config.newInputFileExceptionSupplier; - ioFactory.newOutputFileExceptionSupplier = config.newOutputFileExceptionSupplier; + TestFileIOFactory testIOFactory = (TestFileIOFactory) ioFactory; + testIOFactory.loadFileIOExceptionSupplier = config.loadFileIOExceptionSupplier; + testIOFactory.newInputFileExceptionSupplier = config.newInputFileExceptionSupplier; + testIOFactory.newOutputFileExceptionSupplier = config.newOutputFileExceptionSupplier; - assertThrows(config.expectedException, () -> config.workload.run(uniqueId)); + assertThrows(ForbiddenException.class, () -> config.workload.run(uniqueId)); } private static Stream getIOExceptionTypeTestConfigs() { @@ -167,7 +155,6 @@ private static Stream getIOExceptionTypeTestConfigs() { List> configs = Stream.of( IOExceptionTypeTestConfig.allVariants( - ForbiddenException.class, () -> S3Exception.builder() .statusCode(403) @@ -175,15 +162,12 @@ private static Stream getIOExceptionTypeTestConfigs() { .build(), FileIOIntegrationTest::workloadCreateTable), IOExceptionTypeTestConfig.allVariants( - ForbiddenException.class, () -> new AzureException(accessDeniedHint.next()), FileIOIntegrationTest::workloadCreateTable), IOExceptionTypeTestConfig.allVariants( - ForbiddenException.class, () -> new StorageException(403, accessDeniedHint.next()), FileIOIntegrationTest::workloadCreateTable), IOExceptionTypeTestConfig.allVariants( - ForbiddenException.class, () -> S3Exception.builder() .statusCode(403) @@ -191,11 +175,9 @@ private static Stream getIOExceptionTypeTestConfigs() { .build(), FileIOIntegrationTest::workloadUpdateTableProperties), IOExceptionTypeTestConfig.allVariants( - ForbiddenException.class, () -> new AzureException(accessDeniedHint.next()), FileIOIntegrationTest::workloadUpdateTableProperties), IOExceptionTypeTestConfig.allVariants( - ForbiddenException.class, () -> new StorageException(403, accessDeniedHint.next()), FileIOIntegrationTest::workloadUpdateTableProperties)) .flatMap(Collection::stream) @@ -223,7 +205,6 @@ private static void workloadCreateTable(int uniqueId) { * particular IO operation fails with the given exception specified by the Suppliers. */ record IOExceptionTypeTestConfig( - Class expectedException, Optional> loadFileIOExceptionSupplier, Optional> newInputFileExceptionSupplier, Optional> newOutputFileExceptionSupplier, @@ -238,26 +219,14 @@ interface Workload { * possible step of the IO */ static Collection> allVariants( - Class exceptionType, Supplier exceptionSupplier, Workload workload) { + Supplier exceptionSupplier, Workload workload) { return List.of( new IOExceptionTypeTestConfig<>( - exceptionType, - Optional.of(exceptionSupplier), - Optional.empty(), - Optional.empty(), - workload), + Optional.of(exceptionSupplier), Optional.empty(), Optional.empty(), workload), new IOExceptionTypeTestConfig<>( - exceptionType, - Optional.empty(), - Optional.of(exceptionSupplier), - Optional.empty(), - workload), + Optional.empty(), Optional.of(exceptionSupplier), Optional.empty(), workload), new IOExceptionTypeTestConfig<>( - exceptionType, - Optional.empty(), - Optional.empty(), - Optional.of(exceptionSupplier), - workload)); + Optional.empty(), Optional.empty(), Optional.of(exceptionSupplier), workload)); } } } diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/io/TestFileIOFactory.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/io/TestFileIOFactory.java index 9afeeb155..cb0db8e68 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/io/TestFileIOFactory.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/io/TestFileIOFactory.java @@ -18,7 +18,7 @@ */ package org.apache.polaris.service.catalog.io; -import com.fasterxml.jackson.annotation.JsonTypeName; +import jakarta.enterprise.inject.Vetoed; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -32,8 +32,8 @@ * A FileIOFactory that measures the number of bytes read, files written, and files deleted. It can * inject exceptions at various parts of the IO construction. */ -@JsonTypeName("test") -public class TestFileIOFactory implements FileIOFactory { +@Vetoed +public class TestFileIOFactory extends DefaultFileIOFactory { private final List ios = new ArrayList<>(); // When present, the following will be used to throw exceptions at various parts of the IO diff --git a/polaris-service/src/test/java/org/apache/polaris/service/config/DefaultConfigurationStoreTest.java b/polaris-service/src/test/java/org/apache/polaris/service/config/DefaultConfigurationStoreTest.java index 7c981ab94..bdeeb6023 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/config/DefaultConfigurationStoreTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/config/DefaultConfigurationStoreTest.java @@ -33,7 +33,7 @@ public void testGetConfiguration() { DefaultConfigurationStore defaultConfigurationStore = new DefaultConfigurationStore(Map.of("key1", 1, "key2", "value")); InMemoryPolarisMetaStoreManagerFactory metastoreFactory = - new InMemoryPolarisMetaStoreManagerFactory(); + new InMemoryPolarisMetaStoreManagerFactory(null, null); PolarisCallContext callCtx = new PolarisCallContext( metastoreFactory.getOrCreateSessionSupplier(() -> "realm1").get(), @@ -60,14 +60,17 @@ public void testGetRealmConfiguration() { String realm2KeyTwoValue = "value3"; DefaultConfigurationStore defaultConfigurationStore = new DefaultConfigurationStore( - Map.of("key1", defaultKeyOneValue, "key2", defaultKeyTwoValue), - Map.of( - "realm1", - Map.of("key1", realm1KeyOneValue), - "realm2", - Map.of("key1", realm2KeyOneValue, "key2", realm2KeyTwoValue))); + Map.of("key1", defaultKeyOneValue, "key2", defaultKeyTwoValue) + // FIXME implement realm overrides + // , + // Map.of( + // "realm1", + // Map.of("key1", realm1KeyOneValue), + // "realm2", + // Map.of("key1", realm2KeyOneValue, "key2", realm2KeyTwoValue)) + ); InMemoryPolarisMetaStoreManagerFactory metastoreFactory = - new InMemoryPolarisMetaStoreManagerFactory(); + new InMemoryPolarisMetaStoreManagerFactory(null, null); // check realm1 values PolarisCallContext realm1Ctx = diff --git a/polaris-service/src/test/java/org/apache/polaris/service/ratelimiter/MockRealmTokenBucketRateLimiter.java b/polaris-service/src/test/java/org/apache/polaris/service/ratelimiter/MockRealmTokenBucketRateLimiter.java index 95570091b..6e5c5da51 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/ratelimiter/MockRealmTokenBucketRateLimiter.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/ratelimiter/MockRealmTokenBucketRateLimiter.java @@ -18,28 +18,16 @@ */ package org.apache.polaris.service.ratelimiter; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import java.time.Clock; +import java.time.Duration; import java.time.Instant; import java.time.ZoneOffset; import org.threeten.extra.MutableClock; /** RealmTokenBucketRateLimiter with a mock clock */ -@JsonTypeName("mock-realm-token-bucket") public class MockRealmTokenBucketRateLimiter extends RealmTokenBucketRateLimiter { public static MutableClock CLOCK = MutableClock.of(Instant.now(), ZoneOffset.UTC); - @JsonCreator - public MockRealmTokenBucketRateLimiter( - @JsonProperty("requestsPerSecond") final long requestsPerSecond, - @JsonProperty("windowSeconds") final long windowSeconds) { - super(requestsPerSecond, windowSeconds); - } - - @Override - protected Clock getClock() { - return CLOCK; + public MockRealmTokenBucketRateLimiter(long requestsPerSecond, Duration window) { + super(requestsPerSecond, window, CLOCK); } } diff --git a/polaris-service/src/test/java/org/apache/polaris/service/ratelimiter/RateLimiterFilterTest.java b/polaris-service/src/test/java/org/apache/polaris/service/ratelimiter/RateLimiterFilterTest.java index 772fc6aac..b0284f9a1 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/ratelimiter/RateLimiterFilterTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/ratelimiter/RateLimiterFilterTest.java @@ -18,104 +18,101 @@ */ package org.apache.polaris.service.ratelimiter; -import static org.apache.polaris.core.monitor.PolarisMetricRegistry.*; -import static org.apache.polaris.service.TimedApplicationEventListener.SINGLETON_METRIC_NAME; -import static org.apache.polaris.service.TimedApplicationEventListener.TAG_API_NAME; +import static org.apache.polaris.core.monitor.PolarisMetricRegistry.SUFFIX_ERROR; +import static org.apache.polaris.core.monitor.PolarisMetricRegistry.TAG_RESP_CODE; import static org.junit.jupiter.api.Assertions.assertTrue; -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; import io.micrometer.core.instrument.Tag; +import io.quarkus.test.junit.QuarkusMock; +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; +import jakarta.inject.Inject; import jakarta.ws.rs.core.Response; import java.time.Duration; import java.util.List; +import java.util.Map; import java.util.function.Consumer; -import org.apache.polaris.service.PolarisApplication; -import org.apache.polaris.service.config.PolarisApplicationConfig; -import org.apache.polaris.service.test.PolarisConnectionExtension; -import org.apache.polaris.service.test.PolarisRealm; -import org.apache.polaris.service.test.SnowmanCredentialsExtension; -import org.apache.polaris.service.test.TestEnvironmentExtension; +import org.apache.polaris.service.test.PolarisIntegrationTestHelper; import org.apache.polaris.service.test.TestMetricsUtil; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestInstance; import org.threeten.extra.MutableClock; /** Main integration tests for rate limiting */ -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class, - SnowmanCredentialsExtension.class -}) +@QuarkusTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@TestProfile(RateLimiterFilterTest.Profile.class) public class RateLimiterFilterTest { + private static final long REQUESTS_PER_SECOND = 5; - private static final long WINDOW_SECONDS = 10; - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config("server.adminConnectors[0].port", "0"), - ConfigOverride.config("rateLimiter.type", "mock-realm-token-bucket"), - ConfigOverride.config( - "rateLimiter.requestsPerSecond", String.valueOf(REQUESTS_PER_SECOND)), - ConfigOverride.config("rateLimiter.windowSeconds", String.valueOf(WINDOW_SECONDS))); - - private static String userToken; - private static String realm; + private static final Duration WINDOW = Duration.ofSeconds(10); + + // FIXME these constants come from TimedApplicationEventListener + /** + * Each API will increment a common counter (SINGLETON_METRIC_NAME) but have its API name tagged + * (TAG_API_NAME). + */ + public static final String SINGLETON_METRIC_NAME = "polaris.api"; + + public static final String TAG_API_NAME = "api_name"; + + @Inject PolarisIntegrationTestHelper testHelper; + private static MutableClock clock = MockRealmTokenBucketRateLimiter.CLOCK; @BeforeAll - public static void setup( - PolarisConnectionExtension.PolarisToken userToken, @PolarisRealm String polarisRealm) { - realm = polarisRealm; - RateLimiterFilterTest.userToken = userToken.token(); + public void setUp(TestInfo testInfo) { + QuarkusMock.installMockForType( + new MockRealmTokenBucketRateLimiter(REQUESTS_PER_SECOND, WINDOW), RateLimiter.class); + testHelper.setUp(testInfo); + } + + @AfterAll + public void tearDown() { + testHelper.tearDown(); } @BeforeEach @AfterEach public void resetRateLimiter() { - clock.add( - Duration.ofSeconds(2 * WINDOW_SECONDS)); // Clear any counters from before/after this test + clock.add(WINDOW.multipliedBy(2)); // Clear any counters from before/after this test } @Test public void testRateLimiter() { Consumer requestAsserter = - TestUtil.constructRequestAsserter(EXT, userToken, realm); + TestUtil.constructRequestAsserter(testHelper, testHelper.realm); - for (int i = 0; i < REQUESTS_PER_SECOND * WINDOW_SECONDS; i++) { + for (int i = 0; i < REQUESTS_PER_SECOND * WINDOW.toSeconds(); i++) { requestAsserter.accept(Response.Status.OK); } requestAsserter.accept(Response.Status.TOO_MANY_REQUESTS); // Ensure that a different realm identifier gets a separate limit Consumer requestAsserter2 = - TestUtil.constructRequestAsserter(EXT, userToken, realm + "2"); + TestUtil.constructRequestAsserter(testHelper, testHelper + "2"); requestAsserter2.accept(Response.Status.OK); } @Test public void testMetricsAreEmittedWhenRateLimiting() { Consumer requestAsserter = - TestUtil.constructRequestAsserter(EXT, userToken, realm); + TestUtil.constructRequestAsserter(testHelper, testHelper.realm); - for (int i = 0; i < REQUESTS_PER_SECOND * WINDOW_SECONDS; i++) { + for (int i = 0; i < REQUESTS_PER_SECOND * WINDOW.toSeconds(); i++) { requestAsserter.accept(Response.Status.OK); } requestAsserter.accept(Response.Status.TOO_MANY_REQUESTS); assertTrue( TestMetricsUtil.getTotalCounter( - EXT, + testHelper, SINGLETON_METRIC_NAME + SUFFIX_ERROR, List.of( Tag.of(TAG_API_NAME, "polaris.principal-roles.listPrincipalRoles"), @@ -124,4 +121,16 @@ public void testMetricsAreEmittedWhenRateLimiting() { String.valueOf(Response.Status.TOO_MANY_REQUESTS.getStatusCode())))) > 0); } + + public static class Profile implements QuarkusTestProfile { + + @Override + public Map getConfigOverrides() { + return Map.of( + "polaris.rate-limiter.type", "realm-token-bucket", + "polaris.rate-limiter.realm-token-bucket.requests-per-second", + String.valueOf(REQUESTS_PER_SECOND), + "polaris.rate-limiter.realm-token-bucket.window", WINDOW.toString()); + } + } } diff --git a/polaris-service/src/test/java/org/apache/polaris/service/ratelimiter/RealmTokenBucketRateLimiterTest.java b/polaris-service/src/test/java/org/apache/polaris/service/ratelimiter/RealmTokenBucketRateLimiterTest.java index 10cd71c54..048f13c12 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/ratelimiter/RealmTokenBucketRateLimiterTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/ratelimiter/RealmTokenBucketRateLimiterTest.java @@ -27,7 +27,7 @@ public class RealmTokenBucketRateLimiterTest { @Test void testDifferentBucketsDontTouch() { - RateLimiter rateLimiter = new MockRealmTokenBucketRateLimiter(10, 10); + RateLimiter rateLimiter = new MockRealmTokenBucketRateLimiter(10, Duration.ofSeconds(10)); RateLimitResultAsserter asserter = new RateLimitResultAsserter(rateLimiter); MutableClock clock = MockRealmTokenBucketRateLimiter.CLOCK; diff --git a/polaris-service/src/test/java/org/apache/polaris/service/ratelimiter/TestUtil.java b/polaris-service/src/test/java/org/apache/polaris/service/ratelimiter/TestUtil.java index 43404be2e..83ddc4c70 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/ratelimiter/TestUtil.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/ratelimiter/TestUtil.java @@ -18,35 +18,28 @@ */ package org.apache.polaris.service.ratelimiter; -import static org.apache.polaris.service.context.DefaultContextResolver.REALM_PROPERTY_KEY; +import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; import static org.assertj.core.api.Assertions.assertThat; -import io.dropwizard.testing.junit5.DropwizardAppExtension; import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.Response.Status; import java.util.function.Consumer; -import org.apache.polaris.service.config.PolarisApplicationConfig; +import org.apache.polaris.service.test.PolarisIntegrationTestHelper; /** Common test utils for testing rate limiting */ public class TestUtil { - /** - * Constructs a function that makes a request to list all principal roles and asserts the status - * of the response. This is a relatively simple type of request that can be used for validating - * whether the rate limiter intervenes. - */ - public static Consumer constructRequestAsserter( - DropwizardAppExtension dropwizardAppExtension, - String userToken, - String realm) { + public static Consumer constructRequestAsserter( + PolarisIntegrationTestHelper testHelper, String realm) { return (Response.Status status) -> { try (Response response = - dropwizardAppExtension - .client() + testHelper + .client .target( String.format( "http://localhost:%d/api/management/v1/principal-roles", - dropwizardAppExtension.getLocalPort())) + testHelper.localPort)) .request("application/json") - .header("Authorization", "Bearer " + userToken) + .header("Authorization", "Bearer " + testHelper.adminToken) .header(REALM_PROPERTY_KEY, realm) .get()) { assertThat(response).returns(status.getStatusCode(), Response::getStatus); diff --git a/polaris-service/src/test/java/org/apache/polaris/service/ratelimiter/TokenBucketRateLimiterTest.java b/polaris-service/src/test/java/org/apache/polaris/service/ratelimiter/TokenBucketRateLimiterTest.java index 57a62e265..92b874bd1 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/ratelimiter/TokenBucketRateLimiterTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/ratelimiter/TokenBucketRateLimiterTest.java @@ -59,7 +59,7 @@ void testBasic() { @Test void testConcurrent() throws InterruptedException { int maxTokens = 100; - int numTasks = 50000; + int numTasks = 5000; // FIXME 50000 yields OOME int tokensPerSecond = 10; // Can be anything above 0 TokenBucketRateLimiter rl = @@ -71,7 +71,6 @@ void testConcurrent() throws InterruptedException { try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { for (int i = 0; i < numTasks; i++) { - int i_ = i; executor.submit( () -> { try { diff --git a/polaris-service/src/test/java/org/apache/polaris/service/task/ManifestFileCleanupTaskHandlerTest.java b/polaris-service/src/test/java/org/apache/polaris/service/task/ManifestFileCleanupTaskHandlerTest.java index 711e661f2..8f654e5c5 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/task/ManifestFileCleanupTaskHandlerTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/task/ManifestFileCleanupTaskHandlerTest.java @@ -21,6 +21,8 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.assertj.core.api.Assertions.assertThatPredicate; +import io.quarkus.test.junit.QuarkusTest; +import jakarta.inject.Inject; import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -42,19 +44,14 @@ import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.AsyncTaskType; import org.apache.polaris.core.entity.TaskEntity; -import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; -import org.junit.jupiter.api.BeforeEach; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.junit.jupiter.api.Test; +@QuarkusTest class ManifestFileCleanupTaskHandlerTest { - private InMemoryPolarisMetaStoreManagerFactory metaStoreManagerFactory; - private RealmContext realmContext; + @Inject MetaStoreManagerFactory metaStoreManagerFactory; - @BeforeEach - void setUp() { - metaStoreManagerFactory = new InMemoryPolarisMetaStoreManagerFactory(); - realmContext = () -> "realmName"; - } + private final RealmContext realmContext = () -> "realmName"; @Test public void testCleanupFileNotExists() throws IOException { diff --git a/polaris-service/src/test/java/org/apache/polaris/service/task/TableCleanupTaskHandlerTest.java b/polaris-service/src/test/java/org/apache/polaris/service/task/TableCleanupTaskHandlerTest.java index ab9f9324c..cb771b1bf 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/task/TableCleanupTaskHandlerTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/task/TableCleanupTaskHandlerTest.java @@ -20,6 +20,8 @@ import static org.assertj.core.api.Assertions.assertThat; +import io.quarkus.test.junit.QuarkusTest; +import jakarta.inject.Inject; import java.io.IOException; import java.util.List; import org.apache.commons.codec.binary.Base64; @@ -39,22 +41,17 @@ import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.TableLikeEntity; import org.apache.polaris.core.entity.TaskEntity; -import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mockito; import org.slf4j.LoggerFactory; +@QuarkusTest class TableCleanupTaskHandlerTest { - private InMemoryPolarisMetaStoreManagerFactory metaStoreManagerFactory; - private RealmContext realmContext; + @Inject MetaStoreManagerFactory metaStoreManagerFactory; - @BeforeEach - void setUp() { - metaStoreManagerFactory = new InMemoryPolarisMetaStoreManagerFactory(); - realmContext = () -> "realmName"; - } + private final RealmContext realmContext = () -> "realmName"; @Test public void testTableCleanup() throws IOException { diff --git a/polaris-service/src/test/java/org/apache/polaris/service/task/TaskTestUtils.java b/polaris-service/src/test/java/org/apache/polaris/service/task/TaskTestUtils.java index 1e5612e25..818f87654 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/task/TaskTestUtils.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/task/TaskTestUtils.java @@ -18,6 +18,7 @@ */ package org.apache.polaris.service.task; +import jakarta.annotation.Nonnull; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Arrays; @@ -40,7 +41,6 @@ import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.PositionOutputStream; import org.apache.iceberg.types.Types; -import org.jetbrains.annotations.NotNull; public class TaskTestUtils { static ManifestFile manifestFile( @@ -83,7 +83,7 @@ static void writeTableMetadata(FileIO fileIO, String metadataFile, Snapshot... s out.close(); } - static @NotNull TestSnapshot newSnapshot( + static @Nonnull TestSnapshot newSnapshot( FileIO fileIO, String manifestListLocation, long sequenceNumber, diff --git a/polaris-service/src/test/java/org/apache/polaris/service/test/DropwizardTestEnvironmentResolver.java b/polaris-service/src/test/java/org/apache/polaris/service/test/DropwizardTestEnvironmentResolver.java deleted file mode 100644 index 4aa659749..000000000 --- a/polaris-service/src/test/java/org/apache/polaris/service/test/DropwizardTestEnvironmentResolver.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.test; - -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import java.lang.reflect.Field; -import java.lang.reflect.Modifier; -import java.util.Arrays; -import java.util.Optional; -import org.apache.polaris.core.entity.PolarisGrantRecord; -import org.jetbrains.annotations.Nullable; -import org.junit.jupiter.api.extension.ExtensionContext; -import org.junit.jupiter.api.extension.ParameterResolutionException; -import org.junit.platform.commons.util.ReflectionUtils; -import org.slf4j.LoggerFactory; - -public class DropwizardTestEnvironmentResolver implements TestEnvironmentResolver { - /** - * Resolves the TestEnvironment to point to the local Dropwizard instance. - * - * @param extensionContext - * @return - */ - @Override - public TestEnvironment resolveTestEnvironment(ExtensionContext extensionContext) { - try { - DropwizardAppExtension dropwizardAppExtension = findDropwizardExtension(extensionContext); - if (dropwizardAppExtension == null) { - throw new ParameterResolutionException("Could not find DropwizardAppExtension."); - } - return new TestEnvironment( - dropwizardAppExtension.client(), - String.format("http://localhost:%d", dropwizardAppExtension.getLocalPort())); - } catch (IllegalAccessException e) { - throw new RuntimeException(e); - } - } - - public static @Nullable DropwizardAppExtension findDropwizardExtension( - ExtensionContext extensionContext) throws IllegalAccessException { - Field dropwizardExtensionField = - findAnnotatedFields(extensionContext.getRequiredTestClass(), true); - if (dropwizardExtensionField == null) { - LoggerFactory.getLogger(PolarisGrantRecord.class) - .warn( - "Unable to find dropwizard extension field in test class {}", - extensionContext.getRequiredTestClass()); - return null; - } - DropwizardAppExtension appExtension = - (DropwizardAppExtension) ReflectionUtils.makeAccessible(dropwizardExtensionField).get(null); - return appExtension; - } - - private static Field findAnnotatedFields(Class testClass, boolean isStaticMember) { - final Optional set = - Arrays.stream(testClass.getDeclaredFields()) - .filter(m -> isStaticMember == Modifier.isStatic(m.getModifiers())) - .filter(m -> DropwizardAppExtension.class.isAssignableFrom(m.getType())) - .findFirst(); - if (set.isPresent()) { - return set.get(); - } - if (!testClass.getSuperclass().equals(Object.class)) { - return findAnnotatedFields(testClass.getSuperclass(), isStaticMember); - } - return null; - } -} diff --git a/polaris-service/src/test/java/org/apache/polaris/service/test/PolarisConnectionExtension.java b/polaris-service/src/test/java/org/apache/polaris/service/test/PolarisConnectionExtension.java deleted file mode 100644 index 1dda24967..000000000 --- a/polaris-service/src/test/java/org/apache/polaris/service/test/PolarisConnectionExtension.java +++ /dev/null @@ -1,211 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.test; - -import static org.apache.polaris.service.context.DefaultContextResolver.REALM_PROPERTY_KEY; -import static org.apache.polaris.service.test.DropwizardTestEnvironmentResolver.findDropwizardExtension; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import java.io.IOException; -import java.net.URI; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.core.entity.PolarisEntityConstants; -import org.apache.polaris.core.entity.PolarisEntitySubType; -import org.apache.polaris.core.entity.PolarisEntityType; -import org.apache.polaris.core.entity.PolarisPrincipalSecrets; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.service.auth.TokenUtils; -import org.apache.polaris.service.config.PolarisApplicationConfig; -import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; -import org.junit.jupiter.api.extension.AfterAllCallback; -import org.junit.jupiter.api.extension.BeforeAllCallback; -import org.junit.jupiter.api.extension.ExtensionContext; -import org.junit.jupiter.api.extension.ExtensionContext.Namespace; -import org.junit.jupiter.api.extension.ParameterContext; -import org.junit.jupiter.api.extension.ParameterResolutionException; -import org.junit.jupiter.api.extension.ParameterResolver; - -public class PolarisConnectionExtension - implements BeforeAllCallback, AfterAllCallback, ParameterResolver { - - public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - private MetaStoreManagerFactory metaStoreManagerFactory; - private DropwizardAppExtension dropwizardAppExtension; - - public record PolarisToken(String token) {} - - private static PolarisPrincipalSecrets adminSecrets; - private static String realm; - - @Override - public void beforeAll(ExtensionContext extensionContext) throws Exception { - dropwizardAppExtension = findDropwizardExtension(extensionContext); - if (dropwizardAppExtension == null) { - return; - } - - // Generate unique realm using test name for each test since the tests can run in parallel - realm = extensionContext.getRequiredTestClass().getName().replace('.', '_'); - extensionContext - .getStore(Namespace.create(extensionContext.getRequiredTestClass())) - .put(REALM_PROPERTY_KEY, realm); - - try { - PolarisApplicationConfig config = - (PolarisApplicationConfig) dropwizardAppExtension.getConfiguration(); - metaStoreManagerFactory = config.getMetaStoreManagerFactory(); - if (!(metaStoreManagerFactory instanceof InMemoryPolarisMetaStoreManagerFactory)) { - metaStoreManagerFactory.bootstrapRealms(List.of(realm)); - } - - URI testEnvUri = TestEnvironmentExtension.getEnv(extensionContext).baseUri(); - String path = testEnvUri.getPath(); - if (path.isEmpty()) { - path = "/"; - } - - RealmContext realmContext = - config - .getRealmContextResolver() - .resolveRealmContext( - String.format("%s://%s", testEnvUri.getScheme(), testEnvUri.getHost()), - "GET", - path, - Map.of(), - Map.of(REALM_PROPERTY_KEY, realm)); - CallContext ctx = - config - .getCallContextResolver() - .resolveCallContext(realmContext, "GET", path, Map.of(), Map.of()); - CallContext.setCurrentContext(ctx); - PolarisMetaStoreManager metaStoreManager = - metaStoreManagerFactory.getOrCreateMetaStoreManager(ctx.getRealmContext()); - PolarisMetaStoreManager.EntityResult principal = - metaStoreManager.readEntityByName( - ctx.getPolarisCallContext(), - null, - PolarisEntityType.PRINCIPAL, - PolarisEntitySubType.NULL_SUBTYPE, - PolarisEntityConstants.getRootPrincipalName()); - - Map propertiesMap = readInternalProperties(principal); - adminSecrets = - metaStoreManager - .loadPrincipalSecrets(ctx.getPolarisCallContext(), propertiesMap.get("client_id")) - .getPrincipalSecrets(); - } finally { - CallContext.unsetCurrentContext(); - } - } - - @Override - public void afterAll(ExtensionContext context) { - if (!(metaStoreManagerFactory instanceof InMemoryPolarisMetaStoreManagerFactory)) { - metaStoreManagerFactory.purgeRealms(List.of(realm)); - } - } - - public static void createTestDir(String realm) throws IOException { - // Set up the database location - Path testDir = Path.of("build/test_data/polaris/" + realm); - if (Files.exists(testDir)) { - if (Files.isDirectory(testDir)) { - Files.walk(testDir) - .sorted(Comparator.reverseOrder()) - .forEach( - path -> { - try { - Files.delete(path); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - - } else { - Files.delete(testDir); - } - } - Files.createDirectories(testDir); - } - - static PolarisPrincipalSecrets getAdminSecrets() { - return adminSecrets; - } - - @Override - public boolean supportsParameter( - ParameterContext parameterContext, ExtensionContext extensionContext) - throws ParameterResolutionException { - return parameterContext - .getParameter() - .getType() - .equals(PolarisConnectionExtension.PolarisToken.class) - || parameterContext.getParameter().getType().equals(MetaStoreManagerFactory.class) - || parameterContext.getParameter().getType().equals(PolarisPrincipalSecrets.class) - || (parameterContext.getParameter().getType().equals(String.class) - && parameterContext.getParameter().isAnnotationPresent(PolarisRealm.class)); - } - - @Override - public Object resolveParameter( - ParameterContext parameterContext, ExtensionContext extensionContext) - throws ParameterResolutionException { - if (parameterContext.getParameter().getType().equals(PolarisToken.class)) { - try { - TestEnvironment testEnv = TestEnvironmentExtension.getEnv(extensionContext); - String token = - TokenUtils.getTokenFromSecrets( - testEnv.apiClient(), - testEnv.baseUri().toString(), - adminSecrets.getPrincipalClientId(), - adminSecrets.getMainSecret(), - realm); - return new PolarisToken(token); - } catch (IllegalAccessException e) { - throw new ParameterResolutionException(e.getMessage()); - } - } else if (parameterContext.getParameter().getType().equals(String.class) - && parameterContext.getParameter().isAnnotationPresent(PolarisRealm.class)) { - return realm; - } else { - return metaStoreManagerFactory; - } - } - - private static Map readInternalProperties( - PolarisMetaStoreManager.EntityResult principal) { - try { - return OBJECT_MAPPER.readValue( - principal.getEntity().getInternalProperties(), - new TypeReference>() {}); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } -} diff --git a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/test/PolarisIntegrationTestHelper.java b/polaris-service/src/test/java/org/apache/polaris/service/test/PolarisIntegrationTestHelper.java similarity index 90% rename from polaris-service-quarkus/src/test/java/org/apache/polaris/service/test/PolarisIntegrationTestHelper.java rename to polaris-service/src/test/java/org/apache/polaris/service/test/PolarisIntegrationTestHelper.java index 4ff723678..608d90a30 100644 --- a/polaris-service-quarkus/src/test/java/org/apache/polaris/service/test/PolarisIntegrationTestHelper.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/test/PolarisIntegrationTestHelper.java @@ -67,11 +67,15 @@ public class PolarisIntegrationTestHelper { @Inject public CallContextResolver callContextResolver; @Inject public ObjectMapper objectMapper; - public record SnowmanCredentials(String clientId, String clientSecret) {} + public record SnowmanIdentifier(String principalName, String principalRoleName) {} + + public record SnowmanCredentials( + String clientId, String clientSecret, SnowmanIdentifier identifier) {} public String realm; public Client client; public int localPort; + public int localManagementPort; public PolarisPrincipalSecrets adminSecrets; public SnowmanCredentials snowmanCredentials; public String adminToken; @@ -83,6 +87,7 @@ public void setUp(TestInfo testInfo) { realm = testInfo.getTestClass().orElseThrow().getName().replace('.', '_'); client = ClientBuilder.newClient(); localPort = Integer.getInteger("quarkus.http.port"); + localManagementPort = Integer.getInteger("quarkus.management.port"); fetchAdminSecrets(); adminToken = TokenUtils.getTokenFromSecrets( @@ -101,6 +106,14 @@ public void setUp(TestInfo testInfo) { realm); } + public int getLocalPort() { + return localPort; + } + + public int getLocalManagementPort() { + return localManagementPort; + } + private void fetchAdminSecrets() { try { if (!(metaStoreManagerFactory instanceof InMemoryPolarisMetaStoreManagerFactory)) { @@ -136,7 +149,8 @@ private void fetchAdminSecrets() { private void createSnowmanCredentials() { - PrincipalRole principalRole = new PrincipalRole("catalog-admin"); + SnowmanIdentifier snowmanIdentifier = getSnowmanIdentifier(); + PrincipalRole principalRole = new PrincipalRole(snowmanIdentifier.principalRoleName()); try (Response createPrResponse = client @@ -150,7 +164,7 @@ private void createSnowmanCredentials() { .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } - Principal principal = new Principal("snowman"); + Principal principal = new Principal(snowmanIdentifier.principalName()); try (Response createPResponse = client @@ -161,6 +175,7 @@ private void createSnowmanCredentials() { .post(Entity.json(principal))) { assertThat(createPResponse) .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); + PrincipalWithCredentials snowmanWithCredentials = createPResponse.readEntity(PrincipalWithCredentials.class); try (Response rotateResp = @@ -168,7 +183,7 @@ private void createSnowmanCredentials() { .target( String.format( "http://localhost:%d/api/management/v1/principals/%s/rotate", - localPort, "snowman")) + localPort, principal.getName())) .request(MediaType.APPLICATION_JSON) .header( "Authorization", @@ -190,14 +205,15 @@ private void createSnowmanCredentials() { snowmanCredentials = new SnowmanCredentials( snowmanWithCredentials.getCredentials().getClientId(), - snowmanWithCredentials.getCredentials().getClientSecret()); + snowmanWithCredentials.getCredentials().getClientSecret(), + snowmanIdentifier); } try (Response assignPrResponse = client .target( String.format( - "http://localhost:%d/api/management/v1/principals/snowman/principal-roles", - localPort)) + "http://localhost:%d/api/management/v1/principals/%s/principal-roles", + localPort, principal.getName())) .request("application/json") .header("Authorization", "Bearer " + adminToken) // how is token getting used? .header(REALM_PROPERTY_KEY, realm) @@ -229,6 +245,10 @@ private Map readInternalProperties( } } + private static SnowmanIdentifier getSnowmanIdentifier() { + return new SnowmanIdentifier("snowman", "catalog-admin"); + } + /** Workaround for class loading issues with Quarkus tests. */ @Vetoed private static class MockFileIOFactory extends DefaultFileIOFactory { diff --git a/polaris-service/src/test/java/org/apache/polaris/service/test/PolarisRealm.java b/polaris-service/src/test/java/org/apache/polaris/service/test/PolarisRealm.java deleted file mode 100644 index b7d29dfc8..000000000 --- a/polaris-service/src/test/java/org/apache/polaris/service/test/PolarisRealm.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.test; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -/** - * Annotation used to specify where to inject the Polaris test realm identifier. This is provided by - * PolarisConnectionExtension. - */ -@Target({ElementType.PARAMETER}) -@Retention(RetentionPolicy.RUNTIME) -public @interface PolarisRealm {} diff --git a/polaris-service/src/test/java/org/apache/polaris/service/test/SnowmanCredentialsExtension.java b/polaris-service/src/test/java/org/apache/polaris/service/test/SnowmanCredentialsExtension.java deleted file mode 100644 index fb2caafd0..000000000 --- a/polaris-service/src/test/java/org/apache/polaris/service/test/SnowmanCredentialsExtension.java +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.test; - -import static org.apache.polaris.service.context.DefaultContextResolver.REALM_PROPERTY_KEY; -import static org.assertj.core.api.Assertions.assertThat; - -import jakarta.ws.rs.client.Entity; -import jakarta.ws.rs.core.MediaType; -import jakarta.ws.rs.core.Response; -import org.apache.polaris.core.admin.model.GrantPrincipalRoleRequest; -import org.apache.polaris.core.admin.model.Principal; -import org.apache.polaris.core.admin.model.PrincipalRole; -import org.apache.polaris.core.admin.model.PrincipalWithCredentials; -import org.apache.polaris.core.entity.PolarisPrincipalSecrets; -import org.apache.polaris.service.auth.TokenUtils; -import org.junit.jupiter.api.extension.AfterAllCallback; -import org.junit.jupiter.api.extension.BeforeAllCallback; -import org.junit.jupiter.api.extension.ExtensionContext; -import org.junit.jupiter.api.extension.ExtensionContext.Namespace; -import org.junit.jupiter.api.extension.ParameterContext; -import org.junit.jupiter.api.extension.ParameterResolutionException; -import org.junit.jupiter.api.extension.ParameterResolver; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class SnowmanCredentialsExtension - implements BeforeAllCallback, AfterAllCallback, ParameterResolver { - - private static final Logger LOGGER = LoggerFactory.getLogger(SnowmanCredentialsExtension.class); - private SnowmanCredentials snowmanCredentials; - - public record SnowmanIdentifier(String principalName, String principalRoleName) {} - - public record SnowmanCredentials( - String clientId, String clientSecret, SnowmanIdentifier identifier) {} - - @Override - public void beforeAll(ExtensionContext extensionContext) throws Exception { - PolarisPrincipalSecrets adminSecrets = PolarisConnectionExtension.getAdminSecrets(); - String realm = - extensionContext - .getStore(Namespace.create(extensionContext.getRequiredTestClass())) - .get(REALM_PROPERTY_KEY, String.class); - - if (adminSecrets == null) { - LOGGER - .atError() - .log( - "No admin secrets configured - you must also configure your test with PolarisConnectionExtension"); - return; - } - - TestEnvironment testEnv = TestEnvironmentExtension.getEnv(extensionContext); - String userToken = - TokenUtils.getTokenFromSecrets( - testEnv.apiClient(), - testEnv.baseUri().toString(), - adminSecrets.getPrincipalClientId(), - adminSecrets.getMainSecret(), - realm); - - SnowmanIdentifier snowmanIdentifier = getSnowmanIdentifier(testEnv); - PrincipalRole principalRole = new PrincipalRole(snowmanIdentifier.principalRoleName()); - try (Response createPrResponse = - testEnv - .apiClient() - .target(String.format("%s/api/management/v1/principal-roles", testEnv.baseUri())) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .post(Entity.json(principalRole))) { - assertThat(createPrResponse) - .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - - Principal principal = new Principal(snowmanIdentifier.principalName()); - - try (Response createPResponse = - testEnv - .apiClient() - .target(String.format("%s/api/management/v1/principals", testEnv.baseUri())) - .request("application/json") - .header("Authorization", "Bearer " + userToken) // how is token getting used? - .header(REALM_PROPERTY_KEY, realm) - .post(Entity.json(principal))) { - assertThat(createPResponse) - .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - PrincipalWithCredentials snowmanWithCredentials = - createPResponse.readEntity(PrincipalWithCredentials.class); - try (Response rotateResp = - testEnv - .apiClient() - .target( - String.format( - "%s/api/management/v1/principals/%s/rotate", - testEnv.baseUri(), principal.getName())) - .request(MediaType.APPLICATION_JSON) - .header( - "Authorization", - "Bearer " - + TokenUtils.getTokenFromSecrets( - testEnv.apiClient(), - testEnv.baseUri().toString(), - snowmanWithCredentials.getCredentials().getClientId(), - snowmanWithCredentials.getCredentials().getClientSecret(), - realm)) - .header(REALM_PROPERTY_KEY, realm) - .post(Entity.json(snowmanWithCredentials))) { - - assertThat(rotateResp).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - - // Use the rotated credentials. - snowmanWithCredentials = rotateResp.readEntity(PrincipalWithCredentials.class); - } - snowmanCredentials = - new SnowmanCredentials( - snowmanWithCredentials.getCredentials().getClientId(), - snowmanWithCredentials.getCredentials().getClientSecret(), - snowmanIdentifier); - } - try (Response assignPrResponse = - testEnv - .apiClient() - .target( - String.format( - "%s/api/management/v1/principals/%s/principal-roles", - testEnv.baseUri(), principal.getName())) - .request("application/json") - .header("Authorization", "Bearer " + userToken) // how is token getting used? - .header(REALM_PROPERTY_KEY, realm) - .put(Entity.json(new GrantPrincipalRoleRequest(principalRole)))) { - assertThat(assignPrResponse) - .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - } - - @Override - public void afterAll(ExtensionContext extensionContext) throws Exception { - PolarisPrincipalSecrets adminSecrets = PolarisConnectionExtension.getAdminSecrets(); - String realm = - extensionContext - .getStore(Namespace.create(extensionContext.getRequiredTestClass())) - .get(REALM_PROPERTY_KEY, String.class); - - if (adminSecrets == null) { - LOGGER - .atError() - .log( - "No admin secrets configured - you must also configure your test with PolarisConnectionExtension"); - return; - } - - TestEnvironment testEnv = TestEnvironmentExtension.getEnv(extensionContext); - String userToken = - TokenUtils.getTokenFromSecrets( - testEnv.apiClient(), - testEnv.baseUri().toString(), - adminSecrets.getPrincipalClientId(), - adminSecrets.getMainSecret(), - realm); - - SnowmanIdentifier snowmanIdentifier = getSnowmanIdentifier(testEnv); - testEnv - .apiClient() - .target( - String.format( - "%s/api/management/v1/principal-roles/%s", - testEnv.baseUri(), snowmanIdentifier.principalRoleName())) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .delete() - .close(); - - testEnv - .apiClient() - .target( - String.format( - "%s/api/management/v1/principals/%s", - testEnv.baseUri(), snowmanIdentifier.principalName())) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .delete() - .close(); - } - - // FIXME - this would be better done with a Credentials-specific annotation processor so - // tests could declare which credentials they want (e.g., @TestCredentials("root") ) - // For now, snowman comes from here and root comes from PolarisConnectionExtension - - @Override - public boolean supportsParameter( - ParameterContext parameterContext, ExtensionContext extensionContext) - throws ParameterResolutionException { - - return parameterContext.getParameter().getType() == SnowmanCredentials.class; - } - - @Override - public Object resolveParameter( - ParameterContext parameterContext, ExtensionContext extensionContext) - throws ParameterResolutionException { - return snowmanCredentials; - } - - private static SnowmanIdentifier getSnowmanIdentifier(TestEnvironment testEnv) { - return new SnowmanIdentifier("snowman" + testEnv.testId(), "catalog-admin" + testEnv.testId()); - } -} diff --git a/polaris-service/src/test/java/org/apache/polaris/service/test/TestEnvironment.java b/polaris-service/src/test/java/org/apache/polaris/service/test/TestEnvironment.java deleted file mode 100644 index c65751536..000000000 --- a/polaris-service/src/test/java/org/apache/polaris/service/test/TestEnvironment.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.test; - -import jakarta.ws.rs.client.Client; -import java.net.URI; -import java.util.UUID; - -/** - * Defines the test environment that a test should run in. - * - * @param apiClient The HTTP client to use when making requests - * @param baseUri The base URL that requests should target, for example http://localhost:1234 - * @param testId An ID unique to this test. This can be used to prefix resource names, such as - * catalog names, to prevent collision. - */ -public record TestEnvironment(Client apiClient, URI baseUri, String testId) { - public TestEnvironment(Client apiClient, String baseUri) { - this(apiClient, URI.create(baseUri), UUID.randomUUID().toString().replace("-", "")); - } -} diff --git a/polaris-service/src/test/java/org/apache/polaris/service/test/TestEnvironmentExtension.java b/polaris-service/src/test/java/org/apache/polaris/service/test/TestEnvironmentExtension.java deleted file mode 100644 index aff617d5c..000000000 --- a/polaris-service/src/test/java/org/apache/polaris/service/test/TestEnvironmentExtension.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.test; - -import java.lang.reflect.InvocationTargetException; -import java.util.Optional; -import org.junit.jupiter.api.extension.ExtensionContext; -import org.junit.jupiter.api.extension.ParameterContext; -import org.junit.jupiter.api.extension.ParameterResolutionException; -import org.junit.jupiter.api.extension.ParameterResolver; - -/** - * JUnit test extension that determines the TestEnvironment. Falls back to targetting the local - * Dropwizard instance. - */ -public class TestEnvironmentExtension implements ParameterResolver { - /** - * Environment variable that specifies the test environment resolver. This should be a class name. - * If this is not set, falls back to DropwizardTestEnvironmentResolver. - */ - public static final String ENV_TEST_ENVIRONMENT_RESOLVER_IMPL = - "INTEGRATION_TEST_ENVIRONMENT_RESOLVER_IMPL"; - - private static final String ENV_PROPERTY_KEY = "testenvironment"; - - public static TestEnvironment getEnv(ExtensionContext extensionContext) - throws IllegalAccessException { - // This must be cached because the TestEnvironment has a randomly generated ID - return extensionContext - .getStore(ExtensionContext.Namespace.create(extensionContext.getRequiredTestClass())) - .getOrComputeIfAbsent( - ENV_PROPERTY_KEY, - (k) -> getTestEnvironmentResolver().resolveTestEnvironment(extensionContext), - TestEnvironment.class); - } - - @Override - public boolean supportsParameter( - ParameterContext parameterContext, ExtensionContext extensionContext) - throws ParameterResolutionException { - return parameterContext.getParameter().getType().equals(TestEnvironment.class); - } - - @Override - public Object resolveParameter( - ParameterContext parameterContext, ExtensionContext extensionContext) - throws ParameterResolutionException { - try { - return getEnv(extensionContext); - } catch (IllegalAccessException e) { - throw new ParameterResolutionException(e.getMessage()); - } - } - - private static TestEnvironmentResolver getTestEnvironmentResolver() { - String impl = - Optional.ofNullable(System.getenv(ENV_TEST_ENVIRONMENT_RESOLVER_IMPL)) - .orElse(DropwizardTestEnvironmentResolver.class.getName()); - - try { - return (TestEnvironmentResolver) (Class.forName(impl).getDeclaredConstructor().newInstance()); - } catch (InstantiationException - | IllegalAccessException - | IllegalArgumentException - | InvocationTargetException - | ClassNotFoundException - | NoSuchMethodException e) { - throw new IllegalArgumentException( - String.format( - "Failed to initialize TestEnvironmentResolver using implementation %s.", impl), - e); - } - } -} diff --git a/polaris-service/src/test/java/org/apache/polaris/service/test/TestEnvironmentResolver.java b/polaris-service/src/test/java/org/apache/polaris/service/test/TestEnvironmentResolver.java deleted file mode 100644 index 7064fbabd..000000000 --- a/polaris-service/src/test/java/org/apache/polaris/service/test/TestEnvironmentResolver.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.test; - -import org.junit.jupiter.api.extension.ExtensionContext; - -/** Interface for determining the test environment that tests should run in */ -public interface TestEnvironmentResolver { - TestEnvironment resolveTestEnvironment(ExtensionContext extensionContext); -} diff --git a/polaris-service/src/test/java/org/apache/polaris/service/test/TestMetricsUtil.java b/polaris-service/src/test/java/org/apache/polaris/service/test/TestMetricsUtil.java index f76de83fa..e1304def0 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/test/TestMetricsUtil.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/test/TestMetricsUtil.java @@ -20,13 +20,11 @@ import static org.assertj.core.api.AssertionsForClassTypes.assertThat; -import io.dropwizard.testing.junit5.DropwizardAppExtension; import io.micrometer.core.instrument.Tag; import jakarta.ws.rs.core.Response; import java.util.Collection; import java.util.List; import org.apache.commons.lang3.StringUtils; -import org.apache.polaris.service.config.PolarisApplicationConfig; /** Utils for working with metrics in tests */ public class TestMetricsUtil { @@ -34,9 +32,7 @@ public class TestMetricsUtil { /** Gets a total counter by calling the Prometheus metrics endpoint */ public static double getTotalCounter( - DropwizardAppExtension dropwizardAppExtension, - String metricName, - Collection tags) { + PolarisIntegrationTestHelper helper, String metricName, Collection tags) { metricName += SUFFIX_TOTAL; metricName = metricName.replace('.', '_').replace('-', '_'); @@ -48,10 +44,9 @@ public static double getTotalCounter( tags.stream().map(tag -> String.format("%s=\"%s\"", tag.getKey(), tag.getValue())).toList(); Response response = - dropwizardAppExtension - .client() - .target( - String.format("http://localhost:%d/metrics", dropwizardAppExtension.getAdminPort())) + helper + .client + .target(String.format("http://localhost:%d/metrics", helper.localManagementPort)) .request() .get(); assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); diff --git a/polaris-service/src/test/resources/META-INF/services/org.apache.polaris.service.auth.DiscoverableAuthenticator b/polaris-service/src/test/resources/META-INF/services/org.apache.polaris.service.auth.DiscoverableAuthenticator deleted file mode 100644 index c8652a626..000000000 --- a/polaris-service/src/test/resources/META-INF/services/org.apache.polaris.service.auth.DiscoverableAuthenticator +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -org.apache.polaris.service.auth.TestInlineBearerTokenPolarisAuthenticator \ No newline at end of file diff --git a/polaris-service/src/test/resources/META-INF/services/org.apache.polaris.service.catalog.io.FileIOFactory b/polaris-service/src/test/resources/META-INF/services/org.apache.polaris.service.catalog.io.FileIOFactory deleted file mode 100644 index 21db576ff..000000000 --- a/polaris-service/src/test/resources/META-INF/services/org.apache.polaris.service.catalog.io.FileIOFactory +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -org.apache.polaris.service.catalog.io.TestFileIOFactory \ No newline at end of file diff --git a/polaris-service/src/test/resources/META-INF/services/org.apache.polaris.service.ratelimiter.RateLimiter b/polaris-service/src/test/resources/META-INF/services/org.apache.polaris.service.ratelimiter.RateLimiter deleted file mode 100644 index c43c88a2a..000000000 --- a/polaris-service/src/test/resources/META-INF/services/org.apache.polaris.service.ratelimiter.RateLimiter +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -org.apache.polaris.service.ratelimiter.MockRealmTokenBucketRateLimiter diff --git a/polaris-service/src/test/resources/polaris-server-integrationtest.yml b/polaris-service/src/test/resources/polaris-server-integrationtest.yml deleted file mode 100644 index 10fd38d86..000000000 --- a/polaris-service/src/test/resources/polaris-server-integrationtest.yml +++ /dev/null @@ -1,161 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -server: - # Maximum number of threads. - maxThreads: 200 - - # Minimum number of thread to keep alive. - minThreads: 10 - applicationConnectors: - # HTTP-specific options. - - type: http - - # The port on which the HTTP server listens for service requests. - port: 8181 - - adminConnectors: - - type: http - port: 8182 - - # The hostname of the interface to which the HTTP server socket wil be found. If omitted, the - # socket will listen on all interfaces. - #bindHost: localhost - - # ssl: - # keyStore: ./example.keystore - # keyStorePassword: example - # - # keyStoreType: JKS # (optional, JKS is default) - - # HTTP request log settings - requestLog: - appenders: - # Settings for logging to stdout. - - type: console - - # Settings for logging to a file. - - type: file - - # The file to which statements will be logged. - currentLogFilename: ./logs/request.log - - # When the log file rolls over, the file will be archived to requests-2012-03-15.log.gz, - # requests.log will be truncated, and new statements written to it. - archivedLogFilenamePattern: ./logs/requests-%d.log.gz - - # The maximum number of log files to archive. - archivedFileCount: 14 - - # Enable archiving if the request log entries go to the their own file - archive: true - -featureConfiguration: - ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING: true - ALLOW_WILDCARD_LOCATION: true - ALLOW_SPECIFYING_FILE_IO_IMPL: true - SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION: true - ALLOW_OVERLAPPING_CATALOG_URLS: true - ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING: false - SUPPORTED_CATALOG_STORAGE_TYPES: - - FILE - - S3 - - GCS - - AZURE - -metaStoreManager: - type: in-memory - -io: - factoryType: default - -oauth2: - type: default - tokenBroker: - type: symmetric-key - secret: polaris - -authenticator: - class: org.apache.polaris.service.auth.DefaultPolarisAuthenticator - tokenBroker: - type: symmetric-key - secret: polaris - - -callContextResolver: - type: default - -realmContextResolver: - type: default - -defaultRealm: POLARIS - -cors: - allowed-origins: - - localhost - - # Logging settings. -logging: - - # The default level of all loggers. Can be OFF, ERROR, WARN, INFO, DEBUG, TRACE, or ALL. - level: INFO - - # Logger-specific levels. - loggers: - org.apache.polaris: DEBUG - - appenders: - - - type: console - # If true, write log statements to stdout. - # enabled: true - # Do not display log statements below this threshold to stdout. - threshold: ALL - # Custom Logback PatternLayout with threadname. - logFormat: "%-5p [%d{ISO8601} - %-6r] [%t] [%X{aid}%X{sid}%X{tid}%X{wid}%X{oid}%X{srv}%X{job}%X{rid}] %c{30}: %m %kvp%n%ex" - - # Settings for logging to a file. - - type: file - # If true, write log statements to a file. - # enabled: true - # Do not write log statements below this threshold to the file. - threshold: ALL - # Custom Logback PatternLayout with threadname. - logFormat: "%-5p [%d{ISO8601} - %-6r] [%t] [%X{aid}%X{sid}%X{tid}%X{wid}%X{oid}%X{srv}%X{job}%X{rid}] %c: %m %kvp%n%ex" - - # when using json logging, you must use a format like this, else the - # mdc section of the json log will be incorrect - # logFormat: "%-5p [%d{ISO8601} - %-6r] [%t] [%X] %c: %m%n%ex" - - # The file to which statements will be logged. - currentLogFilename: ./logs/iceberg-rest.log - # When the log file rolls over, the file will be archived to polaris-2012-03-15.log.gz, - # polaris.log will be truncated, and new statements written to it. - archivedLogFilenamePattern: ./logs/iceberg-rest-%d.log.gz - # The maximum number of log files to archive. - archivedFileCount: 14 - -# Limits the size of request bodies sent to Polaris. -1 means no limit. -maxRequestBodyBytes: 1000000 - -# Limits the request rate per realm -rateLimiter: - type: realm-token-bucket - requestsPerSecond: 9999 - windowSeconds: 10