Skip to content

Commit

Permalink
[feature](merge-cloud) Add SecurityChecker for cloud
Browse files Browse the repository at this point in the history
  • Loading branch information
SWJTU-ZhangLei committed Mar 28, 2024
1 parent 2882115 commit 9ff480b
Show file tree
Hide file tree
Showing 13 changed files with 278 additions and 33 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

package org.apache.doris.jdbc;

import org.apache.doris.cloud.security.SecurityChecker;
import org.apache.doris.common.exception.InternalException;
import org.apache.doris.common.exception.UdfRuntimeException;
import org.apache.doris.common.jni.utils.UdfUtils;
Expand Down Expand Up @@ -316,7 +317,7 @@ private void init(JdbcDataSourceConfig config, String sql) throws UdfRuntimeExce
DruidDataSource ds = new DruidDataSource();
ds.setDriverClassLoader(classLoader);
ds.setDriverClassName(config.getJdbcDriverClass());
ds.setUrl(config.getJdbcUrl());
ds.setUrl(SecurityChecker.getInstance().getSafeJdbcUrl(config.getJdbcUrl()));
ds.setUsername(config.getJdbcUser());
ds.setPassword(config.getJdbcPassword());
ds.setMinIdle(config.getConnectionPoolMinSize()); // default 1
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

package org.apache.doris.jdbc;

import org.apache.doris.cloud.security.SecurityChecker;
import org.apache.doris.common.exception.InternalException;
import org.apache.doris.common.exception.UdfRuntimeException;
import org.apache.doris.common.jni.utils.UdfUtils;
Expand Down Expand Up @@ -362,7 +363,7 @@ private void init(JdbcDataSourceConfig config, String sql) throws UdfRuntimeExce
DruidDataSource ds = new DruidDataSource();
ds.setDriverClassLoader(classLoader);
ds.setDriverClassName(config.getJdbcDriverClass());
ds.setUrl(config.getJdbcUrl());
ds.setUrl(SecurityChecker.getInstance().getSafeJdbcUrl(config.getJdbcUrl()));
ds.setUsername(config.getJdbcUser());
ds.setPassword(config.getJdbcPassword());
ds.setMinIdle(config.getConnectionPoolMinSize()); // default 1
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.doris.cloud.security;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.lang.reflect.Method;

public class AliSecurityChecker extends SecurityChecker {
private static final Logger LOG = LogManager.getLogger(AliSecurityChecker.class);

private static Method jdbcUrlCheckMethod = null;
private static Method urlSecurityCheckMethod = null;
private static Method urlSecurityStopCheckMethod = null;

static {
try {
Class clazz = Class.forName("com.aliyun.securitysdk.SecurityUtil");
jdbcUrlCheckMethod = clazz.getMethod("filterJdbcConnectionSource", String.class);
urlSecurityCheckMethod = clazz.getMethod("startSSRFNetHookChecking", String.class);
urlSecurityStopCheckMethod = clazz.getMethod("stopSSRFNetHookChecking", String.class);
} catch (Exception e) {
LOG.warn("exception:", e);
e.printStackTrace();
System.out.println("Failed to find com.aliyun.securitysdk.SecurityUtil's method");
}
}

protected AliSecurityChecker() {}

@Override
public String getSafeJdbcUrl(String originJdbcUrl) throws Exception {
if (jdbcUrlCheckMethod != null) {
return (String) (jdbcUrlCheckMethod.invoke(null, originJdbcUrl));
}
throw new Exception("SecurityUtil.filterJdbcConnectionSource not found");
}

@Override
public void startSSRFChecking(String originUri) throws Exception {
if (urlSecurityCheckMethod != null) {
urlSecurityCheckMethod.invoke(null, originUri);
return;
}
throw new Exception("SecurityUtil.startSSRFNetHookChecking not found");
}

@Override
public void stopSSRFChecking() {
if (urlSecurityStopCheckMethod != null) {
try {
urlSecurityStopCheckMethod.invoke(null);
} catch (Exception e) {
LOG.warn("failed to stop SSRF checking, log and ignore.", e);
}
return;
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.doris.cloud.security;

public class DummySecurityChecker extends SecurityChecker {
protected DummySecurityChecker() {}

@Override
public String getSafeJdbcUrl(String originJdbcUrl) throws Exception {
return originJdbcUrl;
}

@Override
public void startSSRFChecking(String originUri) throws Exception {
return;
}

@Override
public void stopSSRFChecking() {
return;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.doris.cloud.security;

import org.apache.doris.common.Config;

/**
* SecurityChecker is used to check if the url is safe
*/
public abstract class SecurityChecker {
private static class SingletonHolder {
private static final SecurityChecker INSTANCE = Config.apsaradb_env_enabled
? new AliSecurityChecker() : new DummySecurityChecker();
}

protected SecurityChecker() {}

public static SecurityChecker getInstance() {
return SingletonHolder.INSTANCE;
}

/**
* Check and return safe jdbc url, avoid sql injection or other security issues
* @param originJdbcUrl
* @return
* @throws Exception
*/
public abstract String getSafeJdbcUrl(String originJdbcUrl) throws Exception;

/**
* Check if the uri is safe, avoid SSRF attack
* Only handle http:// https://
* @param originUri
* @throws Exception
*/
public abstract void startSSRFChecking(String originUri) throws Exception;

public abstract void stopSSRFChecking();
}
Original file line number Diff line number Diff line change
Expand Up @@ -2633,6 +2633,9 @@ public static boolean isNotCloudMode() {
@ConfField
public static int drop_user_notify_ms_max_times = 86400;

@ConfField(mutable = true)
public static boolean apsaradb_env_enabled = false;

//==========================================================================
// end of cloud config
//==========================================================================
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@

package org.apache.doris.common.util;


import org.apache.doris.catalog.Env;
import org.apache.doris.cloud.security.SecurityChecker;
import org.apache.doris.system.SystemInfoService.HostInfo;

import com.google.common.collect.Maps;
Expand All @@ -30,16 +30,22 @@

public class HttpURLUtil {


public static HttpURLConnection getConnectionWithNodeIdent(String request) throws IOException {
URL url = new URL(request);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
// Must use Env.getServingEnv() instead of getCurrentEnv(),
// because here we need to obtain selfNode through the official service catalog.
HostInfo selfNode = Env.getServingEnv().getSelfNode();
conn.setRequestProperty(Env.CLIENT_NODE_HOST_KEY, selfNode.getHost());
conn.setRequestProperty(Env.CLIENT_NODE_PORT_KEY, selfNode.getPort() + "");
return conn;
try {
SecurityChecker.getInstance().startSSRFChecking(request);
URL url = new URL(request);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
// Must use Env.getServingEnv() instead of getCurrentEnv(),
// because here we need to obtain selfNode through the official service catalog.
HostInfo selfNode = Env.getServingEnv().getSelfNode();
conn.setRequestProperty(Env.CLIENT_NODE_HOST_KEY, selfNode.getHost());
conn.setRequestProperty(Env.CLIENT_NODE_PORT_KEY, selfNode.getPort() + "");
return conn;
} catch (Exception e) {
throw new IOException(e);
} finally {
SecurityChecker.getInstance().stopSSRFChecking();
}
}

public static Map<String, String> getNodeIdentHeaders() throws IOException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.apache.doris.analysis.DropFileStmt;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Env;
import org.apache.doris.cloud.security.SecurityChecker;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.io.Text;
Expand Down Expand Up @@ -49,7 +50,6 @@
import java.net.URLConnection;
import java.nio.file.Paths;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Base64;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -289,6 +289,7 @@ public SmallFile getSmallFile(long fileId) {
private SmallFile downloadAndCheck(long dbId, String catalog, String fileName,
String downloadUrl, String md5sum, boolean saveContent) throws DdlException {
try {
SecurityChecker.getInstance().startSSRFChecking(downloadUrl);
URL url = new URL(downloadUrl);
// get file length
URLConnection urlConnection = url.openConnection();
Expand Down Expand Up @@ -366,13 +367,15 @@ private SmallFile downloadAndCheck(long dbId, String catalog, String fileName,
checksum, false /* not content */);
}
return smallFile;
} catch (IOException | NoSuchAlgorithmException e) {
} catch (Exception e) {
LOG.warn("failed to get file from url: {}", downloadUrl, e);
String errorMsg = e.getMessage();
if (e instanceof FileNotFoundException) {
errorMsg = "File not found";
}
throw new DdlException("Failed to get file from url: " + downloadUrl + ". Error: " + errorMsg);
} finally {
SecurityChecker.getInstance().stopSSRFChecking();
}
}

Expand Down
31 changes: 23 additions & 8 deletions fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.cloud.security.SecurityChecker;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.FeNameFormat;
import org.apache.doris.datasource.InternalCatalog;
Expand Down Expand Up @@ -323,6 +324,7 @@ public static String getResultForUrl(String urlStr, String encodedAuthInfo, int
StringBuilder sb = new StringBuilder();
InputStream stream = null;
try {
SecurityChecker.getInstance().startSSRFChecking(urlStr);
URL url = new URL(urlStr);
URLConnection conn = url.openConnection();
if (encodedAuthInfo != null) {
Expand Down Expand Up @@ -350,6 +352,7 @@ public static String getResultForUrl(String urlStr, String encodedAuthInfo, int
return null;
}
}
SecurityChecker.getInstance().stopSSRFChecking();
}
if (LOG.isDebugEnabled()) {
LOG.debug("get result from url {}: {}", urlStr, sb.toString());
Expand Down Expand Up @@ -471,14 +474,26 @@ public static String ordinal(int i) {
// If no auth info, pass a null.
public static InputStream getInputStreamFromUrl(String urlStr, String encodedAuthInfo, int connectTimeoutMs,
int readTimeoutMs) throws IOException {
URL url = new URL(urlStr);
URLConnection conn = url.openConnection();
if (encodedAuthInfo != null) {
conn.setRequestProperty("Authorization", "Basic " + encodedAuthInfo);
}
conn.setConnectTimeout(connectTimeoutMs);
conn.setReadTimeout(readTimeoutMs);
return conn.getInputStream();
boolean needSecurityCheck = !(urlStr.startsWith("/") || urlStr.startsWith("file://"));
try {
if (needSecurityCheck) {
SecurityChecker.getInstance().startSSRFChecking(urlStr);
}
URL url = new URL(urlStr);
URLConnection conn = url.openConnection();
if (encodedAuthInfo != null) {
conn.setRequestProperty("Authorization", "Basic " + encodedAuthInfo);
}
conn.setConnectTimeout(connectTimeoutMs);
conn.setReadTimeout(readTimeoutMs);
return conn.getInputStream();
} catch (Exception e) {
throw new IOException(e);
} finally {
if (needSecurityCheck) {
SecurityChecker.getInstance().stopSSRFChecking();
}
}
}

public static boolean showHiddenColumns() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

package org.apache.doris.datasource.es;

import org.apache.doris.cloud.security.SecurityChecker;
import org.apache.doris.common.util.JsonUtil;

import com.fasterxml.jackson.databind.JsonNode;
Expand Down Expand Up @@ -235,11 +236,19 @@ private Response executeResponse(OkHttpClient httpClient, String path) throws IO
if (!currentNode.endsWith("/")) {
currentNode = currentNode + "/";
}
Request request = builder.get().url(currentNode + path).build();
if (LOG.isInfoEnabled()) {
LOG.info("es rest client request URL: {}", request.url().toString());
String url = currentNode + path;
try {
SecurityChecker.getInstance().startSSRFChecking(url);
Request request = builder.get().url(currentNode + path).build();
if (LOG.isInfoEnabled()) {
LOG.info("es rest client request URL: {}", request.url().toString());
}
return httpClient.newCall(request).execute();
} catch (Exception e) {
throw new IOException(e);
} finally {
SecurityChecker.getInstance().stopSSRFChecking();
}
return httpClient.newCall(request).execute();
}

/**
Expand Down
Loading

0 comments on commit 9ff480b

Please sign in to comment.