Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

implement rest endpoint #23214

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 40 additions & 4 deletions presto-function-namespace-managers/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,29 @@
<scope>provided</scope>
</dependency>

<dependency>
<groupId>com.facebook.airlift</groupId>
<artifactId>http-client</artifactId>
</dependency>

<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>

<dependency>
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-jdk8</artifactId>
</dependency>

<<<<<<< HEAD
=======
<dependency>
<groupId>com.facebook.drift</groupId>
<artifactId>drift-transport-netty</artifactId>
</dependency>

>>>>>>> 39fc0ce0ef (implement rest endpoint)
<!-- for testing -->
<dependency>
<groupId>com.facebook.presto</groupId>
Expand All @@ -185,13 +208,26 @@
</dependency>

<dependency>
<groupId>com.facebook.drift</groupId>
<artifactId>drift-transport-netty</artifactId>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<groupId>com.facebook.airlift</groupId>
<artifactId>jaxrs</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>com.facebook.airlift</groupId>
<artifactId>jaxrs-testing</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>javax.ws.rs</groupId>
<artifactId>javax.ws.rs-api</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -325,6 +325,7 @@ protected ScalarFunctionImplementation sqlInvokedFunctionToImplementation(SqlInv
return new SqlInvokedScalarFunctionImplementation(function.getBody());
case THRIFT:
case GRPC:
case REST:
checkArgument(function.getFunctionHandle().isPresent(), "Need functionHandle to get function implementation");
return new RemoteScalarFunctionImplementation(function.getFunctionHandle().get(), function.getRoutineCharacteristics().getLanguage(), implementationType);
case JAVA:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.facebook.presto.functionNamespace;

import com.google.inject.BindingAnnotation;

import java.lang.annotation.Retention;

import static java.lang.annotation.RetentionPolicy.RUNTIME;

@Retention(RUNTIME)
@BindingAnnotation
public @interface ForRestServer
{
}
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

import com.facebook.presto.functionNamespace.json.JsonFileBasedFunctionNamespaceManagerFactory;
import com.facebook.presto.functionNamespace.mysql.MySqlFunctionNamespaceManagerFactory;
import com.facebook.presto.functionNamespace.rest.RestBasedFunctionNamespaceManagerFactory;
import com.facebook.presto.spi.Plugin;
import com.facebook.presto.spi.function.FunctionNamespaceManagerFactory;
import com.google.common.collect.ImmutableList;
Expand All @@ -25,6 +26,8 @@ public class FunctionNamespaceManagerPlugin
@Override
public Iterable<FunctionNamespaceManagerFactory> getFunctionNamespaceManagerFactories()
{
return ImmutableList.of(new MySqlFunctionNamespaceManagerFactory(), new JsonFileBasedFunctionNamespaceManagerFactory());
return ImmutableList.of(new MySqlFunctionNamespaceManagerFactory(),
new JsonFileBasedFunctionNamespaceManagerFactory(),
new RestBasedFunctionNamespaceManagerFactory());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.facebook.presto.functionNamespace.json;
package com.facebook.presto.functionNamespace;

import com.facebook.presto.common.type.TypeSignature;
import com.facebook.presto.spi.function.AggregationFunctionMetadata;
import com.facebook.presto.spi.function.FunctionKind;
import com.facebook.presto.spi.function.RoutineCharacteristics;
import com.facebook.presto.spi.function.SqlFunctionId;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.ImmutableList;

Expand All @@ -30,9 +32,6 @@
import static com.google.common.collect.ImmutableList.toImmutableList;
import static java.util.Objects.requireNonNull;

/**
* The function metadata provided by the Json file to the {@link JsonFileBasedFunctionNamespaceManager}.
*/
public class JsonBasedUdfFunctionMetadata
{
/**
Expand Down Expand Up @@ -64,6 +63,8 @@ public class JsonBasedUdfFunctionMetadata
* Optional Aggregate-specific metadata (required for aggregation functions)
*/
private final Optional<AggregationFunctionMetadata> aggregateMetadata;
private final Optional<SqlFunctionId> functionId;
private final Optional<String> version;

@JsonCreator
public JsonBasedUdfFunctionMetadata(
Expand All @@ -73,7 +74,9 @@ public JsonBasedUdfFunctionMetadata(
@JsonProperty("paramTypes") List<TypeSignature> paramTypes,
@JsonProperty("schema") String schema,
@JsonProperty("routineCharacteristics") RoutineCharacteristics routineCharacteristics,
@JsonProperty("aggregateMetadata") Optional<AggregationFunctionMetadata> aggregateMetadata)
@JsonProperty("aggregateMetadata") Optional<AggregationFunctionMetadata> aggregateMetadata,
@JsonProperty("functionId") Optional<SqlFunctionId> functionId,
@JsonProperty("version") Optional<String> version)
{
this.docString = requireNonNull(docString, "docString is null");
this.functionKind = requireNonNull(functionKind, "functionKind is null");
Expand All @@ -85,6 +88,8 @@ public JsonBasedUdfFunctionMetadata(
checkArgument(
(functionKind == AGGREGATE && aggregateMetadata.isPresent()) || (functionKind != AGGREGATE && !aggregateMetadata.isPresent()),
"aggregateMetadata must be present for aggregation functions and absent otherwise");
this.functionId = requireNonNull(functionId, "functionId is null");
this.version = requireNonNull(version, "version is null");
}

public String getDocString()
Expand All @@ -102,6 +107,7 @@ public TypeSignature getOutputType()
return outputType;
}

@JsonIgnore
public List<String> getParamNames()
{
return IntStream.range(0, paramTypes.size()).boxed().map(idx -> "input" + idx).collect(toImmutableList());
Expand All @@ -126,4 +132,14 @@ public Optional<AggregationFunctionMetadata> getAggregateMetadata()
{
return aggregateMetadata;
}

public Optional<SqlFunctionId> getFunctionId()
{
return functionId;
}

public Optional<String> getVersion()
{
return version;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.facebook.presto.functionNamespace.json;
package com.facebook.presto.functionNamespace;

import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
*/
package com.facebook.presto.functionNamespace.json;

import com.facebook.presto.functionNamespace.UdfFunctionSignatureMap;

public interface FunctionDefinitionProvider
{
UdfFunctionSignatureMap getUdfDefinition(String filePath);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
package com.facebook.presto.functionNamespace.json;

import com.facebook.airlift.log.Logger;
import com.facebook.presto.functionNamespace.JsonBasedUdfFunctionMetadata;
import com.facebook.presto.functionNamespace.UdfFunctionSignatureMap;

import java.io.IOException;
import java.nio.file.Files;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,10 @@
import com.facebook.presto.common.type.TypeSignature;
import com.facebook.presto.common.type.UserDefinedType;
import com.facebook.presto.functionNamespace.AbstractSqlInvokedFunctionNamespaceManager;
import com.facebook.presto.functionNamespace.JsonBasedUdfFunctionMetadata;
import com.facebook.presto.functionNamespace.ServingCatalog;
import com.facebook.presto.functionNamespace.SqlInvokedFunctionNamespaceManagerConfig;
import com.facebook.presto.functionNamespace.UdfFunctionSignatureMap;
import com.facebook.presto.functionNamespace.execution.SqlFunctionExecutors;
import com.facebook.presto.spi.PrestoException;
import com.facebook.presto.spi.function.AggregationFunctionImplementation;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.facebook.presto.functionNamespace.rest;

import com.facebook.presto.functionNamespace.ForRestServer;
import com.google.inject.Binder;
import com.google.inject.Module;

import static com.facebook.airlift.http.client.HttpClientBinder.httpClientBinder;

public class RestBasedCommunicationModule
implements Module
{
@Override
public void configure(Binder binder)
{
httpClientBinder(binder).bindHttpClient("restServer", ForRestServer.class);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.facebook.presto.functionNamespace.rest;

import com.facebook.airlift.http.client.HttpClient;
import com.facebook.airlift.http.client.Request;
import com.facebook.airlift.http.client.StatusResponseHandler;
import com.facebook.airlift.json.JsonCodec;
import com.facebook.presto.functionNamespace.ForRestServer;
import com.facebook.presto.functionNamespace.JsonBasedUdfFunctionMetadata;
import com.facebook.presto.functionNamespace.UdfFunctionSignatureMap;
import com.facebook.presto.spi.PrestoException;
import com.google.common.collect.ImmutableMap;
import com.google.inject.Inject;

import java.net.URI;
import java.util.List;
import java.util.Map;

import static com.facebook.airlift.http.client.HttpUriBuilder.uriBuilderFrom;
import static com.facebook.airlift.http.client.JsonResponseHandler.createJsonResponseHandler;
import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED;

public class RestBasedFunctionApis
{
public static final String ALL_FUNCTIONS_ENDPOINT = "/v1/functions";

private final HttpClient httpClient;

private final JsonCodec<Map<String, List<JsonBasedUdfFunctionMetadata>>> functionSignatureMapJsonCodec;

private final RestBasedFunctionNamespaceManagerConfig managerConfig;

@Inject
public RestBasedFunctionApis(
JsonCodec<Map<String, List<JsonBasedUdfFunctionMetadata>>> nativeFunctionSignatureMapJsonCodec,
@ForRestServer HttpClient httpClient,
RestBasedFunctionNamespaceManagerConfig managerConfig)
{
this.functionSignatureMapJsonCodec = nativeFunctionSignatureMapJsonCodec;
this.httpClient = httpClient;
this.managerConfig = managerConfig;
}

public String getFunctionsETag()
{
try {
URI uri = uriBuilderFrom(URI.create(managerConfig.getRestUrl()))
.appendPath(ALL_FUNCTIONS_ENDPOINT)
.build();
Request request = Request.builder()
.prepareHead()
.setUri(uri)
.build();

StatusResponseHandler.StatusResponse response = httpClient.execute(request, StatusResponseHandler.createStatusResponseHandler());
String version = response.getHeader("ETag");
if (version == null) {
throw new IllegalStateException("Failed to retrieve API version: 'ETag' header is missing");
}
return version;
}
catch (Exception e) {
throw new IllegalStateException("Failed to get functions ETag from REST server, " + e.getMessage());
}
}

public UdfFunctionSignatureMap getAllFunctions()
{
return getFunctionsAt(ALL_FUNCTIONS_ENDPOINT);
}

public UdfFunctionSignatureMap getFunctions(String schema)
{
return getFunctionsAt(ALL_FUNCTIONS_ENDPOINT + "/" + schema);
}

public UdfFunctionSignatureMap getFunctions(String schema, String functionName)
{
return getFunctionsAt(ALL_FUNCTIONS_ENDPOINT + "/" + schema + "/" + functionName);
}

public String addFunction(String schema, String functionName, JsonBasedUdfFunctionMetadata metadata)
{
throw new PrestoException(NOT_SUPPORTED, "Add Function is yet to be added");
}

public String updateFunction(String schema, String functionName, String functionId, JsonBasedUdfFunctionMetadata metadata)
{
throw new PrestoException(NOT_SUPPORTED, "Update Function is yet to be added");
}

public String deleteFunction(String schema, String functionName, String functionId)
{
throw new PrestoException(NOT_SUPPORTED, "Delete Function is yet to be added");
}

private UdfFunctionSignatureMap getFunctionsAt(String endpoint)
throws IllegalStateException
{
try {
URI uri = uriBuilderFrom(URI.create(managerConfig.getRestUrl()))
.appendPath(endpoint)
.build();
Request request = Request.builder()
.prepareGet()
.setUri(uri)
.build();

Map<String, List<JsonBasedUdfFunctionMetadata>> nativeFunctionSignatureMap = httpClient.execute(request, createJsonResponseHandler(functionSignatureMapJsonCodec));
return new UdfFunctionSignatureMap(ImmutableMap.copyOf(nativeFunctionSignatureMap));
}
catch (Exception e) {
throw new IllegalStateException("Failed to get function definitions from REST server, " + e.getMessage());
}
}
}
Loading
Loading