diff --git a/build-module.sh b/build-module.sh index b9e0e0a97..42d5fa8d9 100755 --- a/build-module.sh +++ b/build-module.sh @@ -29,6 +29,21 @@ MODULE_NAME=$1 cd $SRC_DIR mvn package -B --quiet -pl :$MODULE_NAME -am -Dmaven.test.skip=true || exit 1 + +if [ "$MODULE_NAME" == "addax-core" ]; then + cp -a core/target/${MODULE_NAME}-${version}.jar ${ADDAX_HOME}/lib + exit 0 +fi + +if [ "$MODULE_NAME" == "addax-common" ];then + cp -a common/target/${MODULE_NAME}-${version}.jar ${ADDAX_HOME}/lib + exit 0 +fi + +if [ "$MODULE_NAME" == "addax-rdbms" -o "$MODULE_NAME" == "addax-storage" -o "$MODULE_NAME" == "addax-transformer" ]; then + cp -a lib/${MODULE_NAME}/target/${MODULE_NAME}-${version}.jar ${ADDAX_HOME}/lib + exit 0 +fi # if the module nam ends with reader, then the module base directory is plugin/reader, # else the directory is plugin/writer if [[ $MODULE_NAME =~ .*"reader" ]]; then diff --git a/common/src/main/java/com/wgzhao/addax/common/element/BoolColumn.java b/common/src/main/java/com/wgzhao/addax/common/element/BoolColumn.java index 4c7486ec2..10860abca 100644 --- a/common/src/main/java/com/wgzhao/addax/common/element/BoolColumn.java +++ b/common/src/main/java/com/wgzhao/addax/common/element/BoolColumn.java @@ -19,7 +19,7 @@ package com.wgzhao.addax.common.element; -import com.wgzhao.addax.common.exception.CommonErrorCode; +import com.wgzhao.addax.common.spi.ErrorCode; import com.wgzhao.addax.common.exception.AddaxException; import java.math.BigDecimal; @@ -122,21 +122,21 @@ public BigDecimal asBigDecimal() public Date asDate() { throw AddaxException.asAddaxException( - CommonErrorCode.CONVERT_NOT_SUPPORT, "Bool type cannot be converted to Date."); + ErrorCode.CONVERT_NOT_SUPPORT, "Bool type cannot be converted to Date."); } @Override public byte[] asBytes() { throw AddaxException.asAddaxException( - CommonErrorCode.CONVERT_NOT_SUPPORT, "Bool type cannot be converted to Bytes."); + ErrorCode.CONVERT_NOT_SUPPORT, "Bool type cannot be converted to Bytes."); } @Override public Timestamp asTimestamp() { throw AddaxException.asAddaxException( - CommonErrorCode.CONVERT_NOT_SUPPORT, "Bool type cannot be converted to Timestamp."); + ErrorCode.CONVERT_NOT_SUPPORT, "Bool type cannot be converted to Timestamp."); } private void validate(final String data) @@ -150,7 +150,7 @@ private void validate(final String data) } throw AddaxException.asAddaxException( - CommonErrorCode.CONVERT_NOT_SUPPORT, + ErrorCode.CONVERT_NOT_SUPPORT, String.format("String [%s] cannot be converted to Bool .", data)); } } diff --git a/common/src/main/java/com/wgzhao/addax/common/element/BytesColumn.java b/common/src/main/java/com/wgzhao/addax/common/element/BytesColumn.java index 34b0cc440..f5edf87ce 100644 --- a/common/src/main/java/com/wgzhao/addax/common/element/BytesColumn.java +++ b/common/src/main/java/com/wgzhao/addax/common/element/BytesColumn.java @@ -19,7 +19,7 @@ package com.wgzhao.addax.common.element; -import com.wgzhao.addax.common.exception.CommonErrorCode; +import com.wgzhao.addax.common.spi.ErrorCode; import com.wgzhao.addax.common.exception.AddaxException; import org.apache.commons.lang3.ArrayUtils; @@ -68,7 +68,7 @@ public String asString() } catch (Exception e) { throw AddaxException.asAddaxException( - CommonErrorCode.CONVERT_NOT_SUPPORT, + ErrorCode.CONVERT_NOT_SUPPORT, String.format("Bytes[%s] cannot be converted to String .", this.toString())); } } @@ -89,41 +89,41 @@ public Long asLong() public BigDecimal asBigDecimal() { throw AddaxException.asAddaxException( - CommonErrorCode.CONVERT_NOT_SUPPORT, "Bytes type cannot converted to BigDecimal."); + ErrorCode.CONVERT_NOT_SUPPORT, "Bytes type cannot converted to BigDecimal."); } @Override public BigInteger asBigInteger() { throw AddaxException.asAddaxException( - CommonErrorCode.CONVERT_NOT_SUPPORT, "Bytes type cannot converted to BigInteger."); + ErrorCode.CONVERT_NOT_SUPPORT, "Bytes type cannot converted to BigInteger."); } @Override public Timestamp asTimestamp() { throw AddaxException.asAddaxException( - CommonErrorCode.CONVERT_NOT_SUPPORT, "Bytes type cannot converted to Timestamp."); + ErrorCode.CONVERT_NOT_SUPPORT, "Bytes type cannot converted to Timestamp."); } @Override public Double asDouble() { throw AddaxException.asAddaxException( - CommonErrorCode.CONVERT_NOT_SUPPORT, "Bytes type cannot converted to Long."); + ErrorCode.CONVERT_NOT_SUPPORT, "Bytes type cannot converted to Long."); } @Override public Date asDate() { throw AddaxException.asAddaxException( - CommonErrorCode.CONVERT_NOT_SUPPORT, "Bytes type cannot converted to Date."); + ErrorCode.CONVERT_NOT_SUPPORT, "Bytes type cannot converted to Date."); } @Override public Boolean asBoolean() { throw AddaxException.asAddaxException( - CommonErrorCode.CONVERT_NOT_SUPPORT, "Bytes type cannot converted to Boolean."); + ErrorCode.CONVERT_NOT_SUPPORT, "Bytes type cannot converted to Boolean."); } } diff --git a/common/src/main/java/com/wgzhao/addax/common/element/ColumnCast.java b/common/src/main/java/com/wgzhao/addax/common/element/ColumnCast.java index 52de00b3b..fc6c63f29 100644 --- a/common/src/main/java/com/wgzhao/addax/common/element/ColumnCast.java +++ b/common/src/main/java/com/wgzhao/addax/common/element/ColumnCast.java @@ -19,7 +19,7 @@ package com.wgzhao.addax.common.element; -import com.wgzhao.addax.common.exception.CommonErrorCode; +import com.wgzhao.addax.common.spi.ErrorCode; import com.wgzhao.addax.common.exception.AddaxException; import com.wgzhao.addax.common.util.Configuration; import org.apache.commons.lang3.time.DateFormatUtils; @@ -207,7 +207,7 @@ static String asString(final DateColumn column) DateCast.datetimeFormat, DateCast.timeZoner); default: throw AddaxException - .asAddaxException(CommonErrorCode.CONVERT_NOT_SUPPORT, + .asAddaxException(ErrorCode.CONVERT_NOT_SUPPORT, "An unsupported type occurred for the date type. Currently, only DATE/TIME/DATETIME are supported."); } } diff --git a/common/src/main/java/com/wgzhao/addax/common/element/DateColumn.java b/common/src/main/java/com/wgzhao/addax/common/element/DateColumn.java index 14469758f..8b14e7c85 100644 --- a/common/src/main/java/com/wgzhao/addax/common/element/DateColumn.java +++ b/common/src/main/java/com/wgzhao/addax/common/element/DateColumn.java @@ -19,7 +19,7 @@ package com.wgzhao.addax.common.element; -import com.wgzhao.addax.common.exception.CommonErrorCode; +import com.wgzhao.addax.common.spi.ErrorCode; import com.wgzhao.addax.common.exception.AddaxException; import java.math.BigDecimal; @@ -141,7 +141,7 @@ public String asString() } catch (Exception e) { throw AddaxException.asAddaxException( - CommonErrorCode.CONVERT_NOT_SUPPORT, + ErrorCode.CONVERT_NOT_SUPPORT, String.format("Date[%s] type cannot be converted to String .", this)); } } @@ -160,28 +160,28 @@ public Date asDate() public byte[] asBytes() { throw AddaxException.asAddaxException( - CommonErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "Bytes")); + ErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "Bytes")); } @Override public Boolean asBoolean() { throw AddaxException.asAddaxException( - CommonErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "Boolean")); + ErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "Boolean")); } @Override public Double asDouble() { throw AddaxException.asAddaxException( - CommonErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "Double")); + ErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "Double")); } @Override public BigInteger asBigInteger() { throw AddaxException.asAddaxException( - CommonErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "BigInteger")); + ErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "BigInteger")); } @Override @@ -197,7 +197,7 @@ public Timestamp asTimestamp() public BigDecimal asBigDecimal() { throw AddaxException.asAddaxException( - CommonErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "BigDecimal")); + ErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "BigDecimal")); } public DateType getSubType() diff --git a/common/src/main/java/com/wgzhao/addax/common/element/DoubleColumn.java b/common/src/main/java/com/wgzhao/addax/common/element/DoubleColumn.java index 4f11df554..ed6a31406 100644 --- a/common/src/main/java/com/wgzhao/addax/common/element/DoubleColumn.java +++ b/common/src/main/java/com/wgzhao/addax/common/element/DoubleColumn.java @@ -19,7 +19,7 @@ package com.wgzhao.addax.common.element; -import com.wgzhao.addax.common.exception.CommonErrorCode; +import com.wgzhao.addax.common.spi.ErrorCode; import com.wgzhao.addax.common.exception.AddaxException; import java.math.BigDecimal; @@ -103,7 +103,7 @@ public BigDecimal asBigDecimal() } catch (NumberFormatException e) { throw AddaxException.asAddaxException( - CommonErrorCode.CONVERT_NOT_SUPPORT, + ErrorCode.CONVERT_NOT_SUPPORT, String.format("String[%s] cannot be converted to Double.", this.getRawData())); } @@ -166,28 +166,28 @@ public String asString() public Boolean asBoolean() { throw AddaxException.asAddaxException( - CommonErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "Boolean")); + ErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "Boolean")); } @Override public Date asDate() { throw AddaxException.asAddaxException( - CommonErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "Date")); + ErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "Date")); } @Override public byte[] asBytes() { throw AddaxException.asAddaxException( - CommonErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "Bytes")); + ErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "Bytes")); } @Override public Timestamp asTimestamp() { throw AddaxException.asAddaxException( - CommonErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "Timestamp")); + ErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "Timestamp")); } private void validate( String data) @@ -206,7 +206,7 @@ private void validate( String data) } catch (Exception e) { throw AddaxException.asAddaxException( - CommonErrorCode.CONVERT_NOT_SUPPORT, + ErrorCode.CONVERT_NOT_SUPPORT, String.format("String[%s] cannot be converted to Double.", data)); } } diff --git a/common/src/main/java/com/wgzhao/addax/common/element/LongColumn.java b/common/src/main/java/com/wgzhao/addax/common/element/LongColumn.java index 555fb8adb..12a1faefa 100644 --- a/common/src/main/java/com/wgzhao/addax/common/element/LongColumn.java +++ b/common/src/main/java/com/wgzhao/addax/common/element/LongColumn.java @@ -20,7 +20,7 @@ package com.wgzhao.addax.common.element; import com.wgzhao.addax.common.exception.AddaxException; -import com.wgzhao.addax.common.exception.CommonErrorCode; +import com.wgzhao.addax.common.spi.ErrorCode; import org.apache.commons.lang3.math.NumberUtils; import java.math.BigDecimal; @@ -56,7 +56,7 @@ public LongColumn(String data) } catch (Exception e) { throw AddaxException.asAddaxException( - CommonErrorCode.CONVERT_NOT_SUPPORT, + ErrorCode.CONVERT_NOT_SUPPORT, String.format("Cannot convert the string [%s] to Long.", data)); } } @@ -177,6 +177,6 @@ public Date asDate() public byte[] asBytes() { throw AddaxException.asAddaxException( - CommonErrorCode.CONVERT_NOT_SUPPORT, "Long type cannot be converted to Bytes."); + ErrorCode.CONVERT_NOT_SUPPORT, "Long type cannot be converted to Bytes."); } } diff --git a/common/src/main/java/com/wgzhao/addax/common/element/OverFlowUtil.java b/common/src/main/java/com/wgzhao/addax/common/element/OverFlowUtil.java index fb9b91d47..94b14a4b1 100644 --- a/common/src/main/java/com/wgzhao/addax/common/element/OverFlowUtil.java +++ b/common/src/main/java/com/wgzhao/addax/common/element/OverFlowUtil.java @@ -19,7 +19,7 @@ package com.wgzhao.addax.common.element; -import com.wgzhao.addax.common.exception.CommonErrorCode; +import com.wgzhao.addax.common.spi.ErrorCode; import com.wgzhao.addax.common.exception.AddaxException; import java.math.BigDecimal; @@ -50,7 +50,7 @@ public static void validateLongNotOverFlow(final BigInteger integer) if (isOverFlow) { throw AddaxException.asAddaxException( - CommonErrorCode.CONVERT_OVER_FLOW, + ErrorCode.CONVERT_OVER_FLOW, String.format("An overflow occurred when converting [%s] to Long type.", integer)); } } @@ -76,7 +76,7 @@ public static void validateDoubleNotOverFlow(final BigDecimal decimal) boolean isOverFlow = OverFlowUtil.isDoubleOverFlow(decimal); if (isOverFlow) { throw AddaxException.asAddaxException( - CommonErrorCode.CONVERT_OVER_FLOW, + ErrorCode.CONVERT_OVER_FLOW, String.format("An overflow occurred when converting [%s] to Double type.", decimal.toPlainString())); } diff --git a/common/src/main/java/com/wgzhao/addax/common/element/StringColumn.java b/common/src/main/java/com/wgzhao/addax/common/element/StringColumn.java index 64d357f30..62813bd41 100644 --- a/common/src/main/java/com/wgzhao/addax/common/element/StringColumn.java +++ b/common/src/main/java/com/wgzhao/addax/common/element/StringColumn.java @@ -19,7 +19,7 @@ package com.wgzhao.addax.common.element; -import com.wgzhao.addax.common.exception.CommonErrorCode; +import com.wgzhao.addax.common.spi.ErrorCode; import com.wgzhao.addax.common.exception.AddaxException; import java.math.BigDecimal; @@ -63,7 +63,7 @@ private void validateDoubleSpecific(final String data) if ("NaN".equals(data) || "Infinity".equals(data) || "-Infinity".equals(data)) { throw AddaxException.asAddaxException( - CommonErrorCode.CONVERT_NOT_SUPPORT, + ErrorCode.CONVERT_NOT_SUPPORT, String.format("['%s'] belongs to the special Double type and cannot be converted to other type.", data)); } } @@ -82,7 +82,7 @@ public BigInteger asBigInteger() } catch (Exception e) { throw AddaxException.asAddaxException( - CommonErrorCode.CONVERT_NOT_SUPPORT, String.format( + ErrorCode.CONVERT_NOT_SUPPORT, String.format( "['%s'] cannot be converted to BigInteger.", this.asString())); } } @@ -112,7 +112,7 @@ public Long asLong() } catch (Exception e) { throw AddaxException.asAddaxException( - CommonErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "Long")); + ErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "Long")); } } @@ -130,7 +130,7 @@ public BigDecimal asBigDecimal() } catch (Exception e) { throw AddaxException.asAddaxException( - CommonErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "BigDecimal")); + ErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "BigDecimal")); } } @@ -176,7 +176,7 @@ public Boolean asBoolean() } throw AddaxException.asAddaxException( - CommonErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "Boolean")); + ErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "Boolean")); } @Override @@ -187,7 +187,7 @@ public Date asDate() } catch (Exception e) { throw AddaxException.asAddaxException( - CommonErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "Date")); + ErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "Date")); } } @@ -199,7 +199,7 @@ public byte[] asBytes() } catch (Exception e) { throw AddaxException.asAddaxException( - CommonErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "Bytes")); + ErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "Bytes")); } } } diff --git a/common/src/main/java/com/wgzhao/addax/common/element/TimestampColumn.java b/common/src/main/java/com/wgzhao/addax/common/element/TimestampColumn.java index 3dec514e0..5416f9128 100644 --- a/common/src/main/java/com/wgzhao/addax/common/element/TimestampColumn.java +++ b/common/src/main/java/com/wgzhao/addax/common/element/TimestampColumn.java @@ -1,7 +1,7 @@ package com.wgzhao.addax.common.element; import com.wgzhao.addax.common.exception.AddaxException; -import com.wgzhao.addax.common.exception.CommonErrorCode; +import com.wgzhao.addax.common.spi.ErrorCode; import java.math.BigDecimal; import java.math.BigInteger; @@ -91,24 +91,24 @@ public Date asDate() @Override public byte[] asBytes() { - throw AddaxException.asAddaxException(CommonErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "Bytes")); + throw AddaxException.asAddaxException(ErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "Bytes")); } @Override public Boolean asBoolean() { - throw AddaxException.asAddaxException(CommonErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "Boolean")); + throw AddaxException.asAddaxException(ErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "Boolean")); } @Override public BigDecimal asBigDecimal() { - throw AddaxException.asAddaxException(CommonErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "BigDecimal")); + throw AddaxException.asAddaxException(ErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "BigDecimal")); } @Override public BigInteger asBigInteger() { - throw AddaxException.asAddaxException(CommonErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "BigInteger")); + throw AddaxException.asAddaxException(ErrorCode.CONVERT_NOT_SUPPORT, String.format(errorTemplate, "BigInteger")); } } diff --git a/common/src/main/java/com/wgzhao/addax/common/exception/AddaxException.java b/common/src/main/java/com/wgzhao/addax/common/exception/AddaxException.java index e63a92877..137f089e7 100644 --- a/common/src/main/java/com/wgzhao/addax/common/exception/AddaxException.java +++ b/common/src/main/java/com/wgzhao/addax/common/exception/AddaxException.java @@ -20,6 +20,9 @@ package com.wgzhao.addax.common.exception; import com.wgzhao.addax.common.spi.ErrorCode; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.PrintWriter; import java.io.StringWriter; @@ -27,7 +30,7 @@ public class AddaxException extends RuntimeException { - + private static final Logger logger = LoggerFactory.getLogger(AddaxException.class); private static final long serialVersionUID = 1L; private final transient ErrorCode errorCode; @@ -35,6 +38,7 @@ public class AddaxException public AddaxException(ErrorCode errorCode, String errorMessage) { super(errorMessage); + logger.error("Error Code: {}: {}", errorCode.getCode(), errorMessage); this.errorCode = errorCode; } @@ -47,7 +51,10 @@ private AddaxException(ErrorCode errorCode, String errorMessage, Throwable cause public static AddaxException asAddaxException(ErrorCode errorCode, String message) { - return new AddaxException(errorCode, message); + if (StringUtils.isBlank(message)) { + message = errorCode.getDescription(); + } + throw new RuntimeException(message); } public static AddaxException asAddaxException(ErrorCode errorCode, String message, Throwable cause) diff --git a/common/src/main/java/com/wgzhao/addax/common/exception/ColumnErrorCode.java b/common/src/main/java/com/wgzhao/addax/common/exception/ColumnErrorCode.java deleted file mode 100644 index ac06924fb..000000000 --- a/common/src/main/java/com/wgzhao/addax/common/exception/ColumnErrorCode.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.common.exception; - -import com.wgzhao.addax.common.spi.ErrorCode; - -/** - * general Column Error code - */ -public enum ColumnErrorCode - implements ErrorCode -{ - - ILLEGAL_VALUE("ColumnError-01", "您填写的参数值不合法."), - MIXED_INDEX_VALUE("ColumnError-02", "您的列信息配置同时包含了index,value."), - NO_INDEX_VALUE("ColumnError-03", "您明确的配置列信息,但未填写相应的index,value."), - NOT_SUPPORT_TYPE("ColumnError-04", "您配置的列类型暂不支持."); - - private final String code; - - private final String describe; - - ColumnErrorCode(String code, String describe) - { - this.code = code; - this.describe = describe; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.describe; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Describe:[%s]", this.code, this.describe); - } -} diff --git a/common/src/main/java/com/wgzhao/addax/common/exception/CommonErrorCode.java b/common/src/main/java/com/wgzhao/addax/common/exception/CommonErrorCode.java deleted file mode 100644 index 8dbfecd8f..000000000 --- a/common/src/main/java/com/wgzhao/addax/common/exception/CommonErrorCode.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.common.exception; - -import com.wgzhao.addax.common.spi.ErrorCode; - -/** - * - */ -public enum CommonErrorCode - implements ErrorCode -{ - - CONFIG_ERROR("Common-00", "您提供的配置文件存在错误信息，请检查您的作业配置 ."), - CONVERT_NOT_SUPPORT("Common-01", "同步数据出现业务脏数据情况，数据类型转换错误 ."), - CONVERT_OVER_FLOW("Common-02", "同步数据出现业务脏数据情况，数据类型转换溢出 ."), - RETRY_FAIL("Common-10", "方法调用多次仍旧失败 ."), - RUNTIME_ERROR("Common-11", "运行时内部调用错误 ."), - HOOK_INTERNAL_ERROR("Common-12", "Hook运行错误 ."), - SHUT_DOWN_TASK("Common-20", "Task收到了shutdown指令，为failover做准备"), - WAIT_TIME_EXCEED("Common-21", "等待时间超出范围"), - TASK_HUNG_EXPIRED("Common-22", "任务hung住，Expired"); - - private final String code; - - private final String describe; - - CommonErrorCode(String code, String describe) - { - this.code = code; - this.describe = describe; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.describe; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Describe:[%s]", this.code, - this.describe); - } -} diff --git a/common/src/main/java/com/wgzhao/addax/common/spi/ErrorCode.java b/common/src/main/java/com/wgzhao/addax/common/spi/ErrorCode.java index f79041718..dace9590e 100644 --- a/common/src/main/java/com/wgzhao/addax/common/spi/ErrorCode.java +++ b/common/src/main/java/com/wgzhao/addax/common/spi/ErrorCode.java @@ -6,9 +6,7 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * * http://www.apache.org/licenses/LICENSE-2.0 - * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -19,35 +17,73 @@ package com.wgzhao.addax.common.spi; + /** - * 尤其注意：最好提供toString()实现。例如： - * - * 
- *
- * @Override
- * public String toString() {
- * 	return String.format("Code:[%s], Description:[%s]. ", this.code, this.describe);
- * }
- *
+ * Enum representing common error codes and their descriptions. */ -public interface ErrorCode +public enum ErrorCode { - // 错误码编号 - String getCode(); - - // 错误码描述 - String getDescription(); - - /** - * 必须提供toString的实现 - * - * 
-     * @Override
-     * public String toString() {
-     * 	return String.format("Code:[%s], Description:[%s]. ", this.code, this.describe);
-     * }
-     *
- */ - @Override - String toString(); + // Required and Illegal Values + REQUIRED_VALUE("1001", "Missing required parameters"), + ILLEGAL_VALUE("1002", "Illegal parameter value"), + CONFIG_ERROR("1003", "Configuration error"), + + // Permission and Connection Errors + PERMISSION_ERROR("2001", "Permission denied"), + CONNECT_ERROR("2002", "Connection error"), + LOGIN_ERROR("2003", "Login error"), + + // Data Conversion Errors + CONVERT_NOT_SUPPORT("3001", "Unsupported data type conversion"), + NOT_SUPPORT_TYPE("3002", "Unsupported data type"), + CONVERT_OVER_FLOW("3003", "Data type conversion overflow"), + ENCODING_ERROR("3004", "Encoding error"), + + // Execution Errors + RETRY_FAIL("4001", "Retry failed"), + EXECUTE_FAIL("4002", "Execution failed"), + IO_ERROR("4003", "IO error"), + + // Runtime Errors + RUNTIME_ERROR("5001", "Runtime error"), + HOOK_INTERNAL_ERROR("5002", "Hook internal error"), + SHUT_DOWN_TASK("5003", "Task shutdown"), + WAIT_TIME_EXCEED("5004", "Wait time exceeded"), + TASK_HUNG_EXPIRED("5005", "Task hung expired"), + + // Plugin Errors + PLUGIN_INSTALL_ERROR("6001", "Plugins installation error"), + PLUGIN_INIT_ERROR("6002", "Plugins initialization error"), + OVER_LIMIT_ERROR("6003", "Over limit error"); + + private final String code; + + private final String description; + + ErrorCode(String code, String description) + { + if (code == null || code.isEmpty()) { + throw new IllegalArgumentException("Error code cannot be null or empty"); + } + if (description == null || description.isEmpty()) { + throw new IllegalArgumentException("Error description cannot be null or empty"); + } + this.code = code; + this.description = description; + } + + public String getCode() + { + return this.code; + } + + public String getDescription() + { + return this.description; + } + + public String toString() + { + return String.format("Error code: %s, Description: %s", this.code, this.description); + } } diff --git a/common/src/main/java/com/wgzhao/addax/common/util/Configuration.java b/common/src/main/java/com/wgzhao/addax/common/util/Configuration.java index 305c0a8ef..77794854f 100644 --- a/common/src/main/java/com/wgzhao/addax/common/util/Configuration.java +++ b/common/src/main/java/com/wgzhao/addax/common/util/Configuration.java @@ -22,7 +22,6 @@ import com.alibaba.fastjson2.JSON; import com.alibaba.fastjson2.JSONWriter; import com.wgzhao.addax.common.exception.AddaxException; -import com.wgzhao.addax.common.exception.CommonErrorCode; import com.wgzhao.addax.common.spi.ErrorCode; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.CharUtils; @@ -43,6 +42,8 @@ import java.util.Map; import java.util.Set; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; + /** * Configuration 提供多级JSON配置信息无损存储
* 实例代码:
@@ -82,7 +83,7 @@ private Configuration(String json) this.root = JSON.parse(json); } catch (Exception e) { - throw AddaxException.asAddaxException(CommonErrorCode.CONFIG_ERROR, + throw AddaxException.asAddaxException(ErrorCode.CONFIG_ERROR, String.format("The configuration is incorrect. The configuration you provided is " + "not in valid JSON format: %s.", e.getMessage())); } @@ -108,7 +109,7 @@ public static Configuration from(String json) return new Configuration(json); } catch (Exception e) { - throw AddaxException.asAddaxException(CommonErrorCode.CONFIG_ERROR, + throw AddaxException.asAddaxException(ErrorCode.CONFIG_ERROR, e); } } @@ -123,12 +124,12 @@ public static Configuration from(File file) .toString(new FileInputStream(file), StandardCharsets.UTF_8)); } catch (FileNotFoundException e) { - throw AddaxException.asAddaxException(CommonErrorCode.CONFIG_ERROR, + throw AddaxException.asAddaxException(ErrorCode.CONFIG_ERROR, String.format("No such file: %s", file.getAbsolutePath())); } catch (IOException e) { throw AddaxException.asAddaxException( - CommonErrorCode.CONFIG_ERROR, + ErrorCode.CONFIG_ERROR, String.format("Failed to read the configuration [%s], reason: %s.", file.getAbsolutePath(), e)); } @@ -143,7 +144,7 @@ public static Configuration from(InputStream is) return Configuration.from(IOUtils.toString(is, StandardCharsets.UTF_8)); } catch (IOException e) { - throw AddaxException.asAddaxException(CommonErrorCode.CONFIG_ERROR, + throw AddaxException.asAddaxException(ErrorCode.CONFIG_ERROR, String.format("Failed to read the configuration reason: %s.", e)); } } @@ -167,7 +168,7 @@ public static Configuration from(final List object) private static void checkJSON(String json) { if (StringUtils.isBlank(json)) { - throw AddaxException.asAddaxException(CommonErrorCode.CONFIG_ERROR, + throw AddaxException.asAddaxException(ErrorCode.CONFIG_ERROR, "The configure file is empty."); } } @@ -182,12 +183,31 @@ public String getNecessaryValue(String key, ErrorCode errorCode) String value = this.getString(key, null); if (StringUtils.isBlank(value)) { throw AddaxException.asAddaxException(errorCode, - String.format("Illegal configuration, the item [%s] is required.", key)); + String.format("The required item '%s' is not found", key)); } + return value; + } + public String getNecessaryValue(String key) + { + String value = this.getString(key, null); + if (StringUtils.isBlank(value)) { + throw AddaxException.asAddaxException(REQUIRED_VALUE, + String.format("The required item %s is not found", key)); + } return value; } + public void getNecessaryValues(String... keys) + { + for (String key : keys) { + if (StringUtils.isBlank(this.getString(key, null))) { + throw AddaxException.asAddaxException(REQUIRED_VALUE, + String.format("The required item %s is not found", key)); + } + } + } + public String getUnnecessaryValue(String key, String defaultValue) { String value = this.getString(key, defaultValue); @@ -294,7 +314,7 @@ public Character getChar(String path) } catch (Exception e) { throw AddaxException.asAddaxException( - CommonErrorCode.CONFIG_ERROR, + ErrorCode.CONFIG_ERROR, String.format("Illegal configuration, cannot be converted to string for [%s], reason: %s.", path, e.getMessage())); } @@ -338,7 +358,7 @@ else if ("false".equalsIgnoreCase(result)) { return Boolean.FALSE; } else { - throw AddaxException.asAddaxException(CommonErrorCode.CONFIG_ERROR, + throw AddaxException.asAddaxException(ErrorCode.CONFIG_ERROR, String.format("Illegal configuration, the value [%s] of [%s] cannot be converted to bool type.", path, result)); } @@ -381,7 +401,7 @@ public Integer getInt(String path) } catch (Exception e) { throw AddaxException.asAddaxException( - CommonErrorCode.CONFIG_ERROR, + ErrorCode.CONFIG_ERROR, String.format("Illegal configuration, the value [%s] of [%s] is expected integer type.", path, e.getMessage())); } @@ -423,7 +443,7 @@ public Long getLong(String path) } catch (Exception e) { throw AddaxException.asAddaxException( - CommonErrorCode.CONFIG_ERROR, + ErrorCode.CONFIG_ERROR, String.format("Illegal configuration, the value [%s] of [%s] is expected integer/long type.", path, e.getMessage())); } @@ -465,7 +485,7 @@ public Double getDouble(String path) } catch (Exception e) { throw AddaxException.asAddaxException( - CommonErrorCode.CONFIG_ERROR, + ErrorCode.CONFIG_ERROR, String.format("Illegal configuration, the value [%s] of [%s] is expected float type.", path, e.getMessage())); } @@ -714,7 +734,7 @@ public Object remove(String path) Object result = this.get(path); if (null == result) { throw AddaxException.asAddaxException( - CommonErrorCode.RUNTIME_ERROR, + ErrorCode.RUNTIME_ERROR, String.format("Illegal configuration, the key [%s] does not exists.", path)); } @@ -848,7 +868,7 @@ private void setObject(String path, Object object) return; } - throw AddaxException.asAddaxException(CommonErrorCode.RUNTIME_ERROR, + throw AddaxException.asAddaxException(ErrorCode.RUNTIME_ERROR, String.format("Illegal value, cannot set value [%s] for key [%s]", ToStringBuilder.reflectionToString(object), path)); } @@ -894,7 +914,7 @@ Object buildObject(List paths, Object object) { if (null == paths) { throw AddaxException.asAddaxException( - CommonErrorCode.RUNTIME_ERROR, "The Path cannot be null"); + ErrorCode.RUNTIME_ERROR, "The Path cannot be null"); } if (1 == paths.size() && StringUtils.isBlank(paths.get(0))) { @@ -922,7 +942,7 @@ Object buildObject(List paths, Object object) } throw AddaxException.asAddaxException( - CommonErrorCode.RUNTIME_ERROR, String.format("the value [%s] of [%s] is illegal.", + ErrorCode.RUNTIME_ERROR, String.format("the value [%s] of [%s] is illegal.", StringUtils.join(paths, "."), path)); } @@ -1009,7 +1029,7 @@ Object setObjectRecursive(Object current, List paths, return lists; } - throw AddaxException.asAddaxException(CommonErrorCode.RUNTIME_ERROR, "System internal error."); + throw AddaxException.asAddaxException(ErrorCode.RUNTIME_ERROR, "System internal error."); } private Object findObject(String path) diff --git a/common/src/main/java/com/wgzhao/addax/common/util/ListUtil.java b/common/src/main/java/com/wgzhao/addax/common/util/ListUtil.java index 9d95d101b..4c31840f3 100644 --- a/common/src/main/java/com/wgzhao/addax/common/util/ListUtil.java +++ b/common/src/main/java/com/wgzhao/addax/common/util/ListUtil.java @@ -19,7 +19,7 @@ package com.wgzhao.addax.common.util; -import com.wgzhao.addax.common.exception.CommonErrorCode; +import com.wgzhao.addax.common.spi.ErrorCode; import com.wgzhao.addax.common.exception.AddaxException; import org.apache.commons.lang3.StringUtils; @@ -60,7 +60,7 @@ public static void makeSureNoValueDuplicate(List aList, if (list.get(i).equals(list.get(i + 1))) { throw AddaxException .asAddaxException( - CommonErrorCode.CONFIG_ERROR, + ErrorCode.CONFIG_ERROR, String.format( "您提供的作业配置信息有误, String:[%s] 不允许重复出现在列表中: [%s].", list.get(i), @@ -111,7 +111,7 @@ public static void makeSureBInA(List aList, List bList, if (!all.contains(oneValue)) { throw AddaxException .asAddaxException( - CommonErrorCode.CONFIG_ERROR, + ErrorCode.CONFIG_ERROR, String.format( "您提供的作业配置信息有误, String:[%s] 不存在于列表中:[%s].", oneValue, StringUtils.join(aList, ","))); diff --git a/core/src/main/java/com/wgzhao/addax/core/Engine.java b/core/src/main/java/com/wgzhao/addax/core/Engine.java index 5cf6730ba..f522a2e24 100644 --- a/core/src/main/java/com/wgzhao/addax/core/Engine.java +++ b/core/src/main/java/com/wgzhao/addax/core/Engine.java @@ -124,13 +124,13 @@ public static String getVersion() { } public static void main(String[] args) { - System.out.println("\n ___ _ _ \n" + + LOG.info("\n ___ _ _ \n" + " / _ \\ | | | | \n" + "/ /_\\ \\ __| | __| | __ ___ __\n" + "| _ |/ _` |/ _` |/ _` \\ \\/ /\n" + "| | | | (_| | (_| | (_| |> < \n" + - "\\_| |_/\\__,_|\\__,_|\\__,_/_/\\_\\\n"); - System.out.println(":: Addax version :: (v" + Engine.getVersion() + ")\n"); + "\\_| |_/\\__,_|\\__,_|\\__,_/_/\\_\\\n"+ + ":: Addax version :: (v{})", Engine.getVersion()); if (args.length < 2) { LOG.error("need a job file"); System.exit(1); diff --git a/core/src/main/java/com/wgzhao/addax/core/job/JobContainer.java b/core/src/main/java/com/wgzhao/addax/core/job/JobContainer.java index 674c7b901..5a9f4883a 100644 --- a/core/src/main/java/com/wgzhao/addax/core/job/JobContainer.java +++ b/core/src/main/java/com/wgzhao/addax/core/job/JobContainer.java @@ -41,7 +41,6 @@ import com.wgzhao.addax.core.statistics.container.communicator.job.StandAloneJobContainerCommunicator; import com.wgzhao.addax.core.statistics.plugin.DefaultJobPluginCollector; import com.wgzhao.addax.core.util.ErrorRecordChecker; -import com.wgzhao.addax.core.util.FrameworkErrorCode; import com.wgzhao.addax.core.util.container.ClassLoaderSwapper; import com.wgzhao.addax.core.util.container.CoreConstant; import com.wgzhao.addax.core.util.container.LoadUtil; @@ -63,6 +62,10 @@ import java.util.List; import java.util.Map; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.EXECUTE_FAIL; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; + /* * Created by jingxing on 14-8-24. *

@@ -163,7 +166,7 @@ public void start() { super.getContainerCommunicator().report(reportCommunication); throw AddaxException.asAddaxException( - FrameworkErrorCode.RUNTIME_ERROR, e); + RUNTIME_ERROR, e); } finally { if (!isDryRun) { @@ -282,7 +285,7 @@ private void preHandle() { handlerPluginType = PluginType.valueOf(handlerPluginTypeStr.toUpperCase()); } catch (IllegalArgumentException e) { throw AddaxException.asAddaxException( - FrameworkErrorCode.CONFIG_ERROR, + CONFIG_ERROR, String.format("The plugin type (%s) set for the pre-handler of job failed, reason: %s", handlerPluginTypeStr.toUpperCase(), e.getMessage())); } @@ -313,7 +316,7 @@ private void postHandle() { handlerPluginType = PluginType.valueOf(handlerPluginTypeStr.toUpperCase()); } catch (IllegalArgumentException e) { throw AddaxException.asAddaxException( - FrameworkErrorCode.CONFIG_ERROR, + CONFIG_ERROR, String.format("The plugin type (%s) set for the post-handler of job failed, reason: %s", handlerPluginTypeStr.toUpperCase(), e.getMessage())); } @@ -373,7 +376,7 @@ private void adjustChannelNumber() { Long channelLimitedByteSpeed = this.configuration.getLong(CoreConstant.CORE_TRANSPORT_CHANNEL_SPEED_BYTE, -1); if (channelLimitedByteSpeed == null || channelLimitedByteSpeed <= 0) { throw AddaxException.asAddaxException( - FrameworkErrorCode.CONFIG_ERROR, + CONFIG_ERROR, "Under the condition of total bps limit, the bps value of a single channel cannot be empty or non-positive"); } @@ -387,7 +390,7 @@ private void adjustChannelNumber() { long globalLimitedRecordSpeed = this.configuration.getInt(CoreConstant.JOB_SETTING_SPEED_RECORD, 100000); Long channelLimitedRecordSpeed = this.configuration.getLong(CoreConstant.CORE_TRANSPORT_CHANNEL_SPEED_RECORD, -1); if (channelLimitedRecordSpeed == null || channelLimitedRecordSpeed <= 0) { - throw AddaxException.asAddaxException(FrameworkErrorCode.CONFIG_ERROR, + throw AddaxException.asAddaxException(CONFIG_ERROR, "Under the condition of total tps limit, the tps value of a single channel cannot be empty or non-positive"); } @@ -443,7 +446,7 @@ private void schedule() { LOG.error("The scheduler failed to run."); this.endTransferTimeStamp = System.currentTimeMillis(); throw AddaxException.asAddaxException( - FrameworkErrorCode.RUNTIME_ERROR, e); + RUNTIME_ERROR, e); } /* @@ -631,7 +634,7 @@ private List doReaderSplit(int adviceNumber) { classLoaderSwapper.setCurrentThreadClassLoader(LoadUtil.getJarLoader(PluginType.READER, this.readerPluginName)); List readerSlicesConfigs = this.jobReader.split(adviceNumber); if (readerSlicesConfigs == null || readerSlicesConfigs.isEmpty()) { - throw AddaxException.asAddaxException(FrameworkErrorCode.PLUGIN_SPLIT_ERROR, + throw AddaxException.asAddaxException(EXECUTE_FAIL, "The number of tasks divided by the reader's job cannot be less than or equal to zero"); } LOG.info("The Reader.Job [{}] is divided into [{}] task(s).", this.readerPluginName, readerSlicesConfigs.size()); @@ -644,7 +647,7 @@ private List doWriterSplit(int readerTaskNumber) { List writerSlicesConfigs = this.jobWriter.split(readerTaskNumber); if (writerSlicesConfigs == null || writerSlicesConfigs.isEmpty()) { - throw AddaxException.asAddaxException(FrameworkErrorCode.PLUGIN_SPLIT_ERROR, + throw AddaxException.asAddaxException(EXECUTE_FAIL, "The number of tasks divided by the writer's job cannot be less than or equal to zero"); } LOG.info("The Writer.Job [{}] is divided into [{}] task(s).", this.writerPluginName, writerSlicesConfigs.size()); @@ -662,7 +665,7 @@ private List mergeReaderAndWriterTaskConfigs( List transformerConfigs) { if (readerTasksConfigs.size() != writerTasksConfigs.size()) { throw AddaxException.asAddaxException( - FrameworkErrorCode.PLUGIN_SPLIT_ERROR, + CONFIG_ERROR, String.format("The number of tasks [%d] divided by the reader's job does not equal " + "the number of tasks [%d] divided by the writer's job", readerTasksConfigs.size(), writerTasksConfigs.size()) diff --git a/core/src/main/java/com/wgzhao/addax/core/job/scheduler/AbstractScheduler.java b/core/src/main/java/com/wgzhao/addax/core/job/scheduler/AbstractScheduler.java index 3e238da1e..abf501b0f 100644 --- a/core/src/main/java/com/wgzhao/addax/core/job/scheduler/AbstractScheduler.java +++ b/core/src/main/java/com/wgzhao/addax/core/job/scheduler/AbstractScheduler.java @@ -26,7 +26,6 @@ import com.wgzhao.addax.core.statistics.communication.CommunicationTool; import com.wgzhao.addax.core.statistics.container.communicator.AbstractContainerCommunicator; import com.wgzhao.addax.core.util.ErrorRecordChecker; -import com.wgzhao.addax.core.util.FrameworkErrorCode; import com.wgzhao.addax.core.util.container.CoreConstant; import org.apache.commons.lang3.Validate; import org.slf4j.Logger; @@ -34,6 +33,8 @@ import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; + public abstract class AbstractScheduler { private static final Logger LOG = LoggerFactory.getLogger(AbstractScheduler.class); @@ -114,8 +115,7 @@ public void schedule(List configurations) // 以 failed 状态退出 LOG.error("An InterruptedException was caught!", e); - throw AddaxException.asAddaxException( - FrameworkErrorCode.RUNTIME_ERROR, e); + throw AddaxException.asAddaxException(RUNTIME_ERROR, e); } } diff --git a/core/src/main/java/com/wgzhao/addax/core/job/scheduler/processinner/ProcessInnerScheduler.java b/core/src/main/java/com/wgzhao/addax/core/job/scheduler/processinner/ProcessInnerScheduler.java index 6a183a05a..b648e5a1d 100644 --- a/core/src/main/java/com/wgzhao/addax/core/job/scheduler/processinner/ProcessInnerScheduler.java +++ b/core/src/main/java/com/wgzhao/addax/core/job/scheduler/processinner/ProcessInnerScheduler.java @@ -25,12 +25,13 @@ import com.wgzhao.addax.core.statistics.container.communicator.AbstractContainerCommunicator; import com.wgzhao.addax.core.taskgroup.TaskGroupContainer; import com.wgzhao.addax.core.taskgroup.runner.TaskGroupContainerRunner; -import com.wgzhao.addax.core.util.FrameworkErrorCode; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; + public abstract class ProcessInnerScheduler extends AbstractScheduler { @@ -61,7 +62,7 @@ public void dealFailedStat(AbstractContainerCommunicator frameworkCollector, Thr { this.taskGroupContainerExecutorService.shutdownNow(); throw AddaxException.asAddaxException( - FrameworkErrorCode.PLUGIN_RUNTIME_ERROR, throwable); + RUNTIME_ERROR, throwable); } @Override @@ -69,7 +70,7 @@ public void dealKillingStat(AbstractContainerCommunicator frameworkCollector, in { //通过进程退出返回码标示状态 this.taskGroupContainerExecutorService.shutdownNow(); - throw AddaxException.asAddaxException(FrameworkErrorCode.KILLED_EXIT_VALUE, "The job was terminated"); + throw AddaxException.asAddaxException(RUNTIME_ERROR, "The job was terminated"); } private TaskGroupContainerRunner newTaskGroupContainerRunner( diff --git a/core/src/main/java/com/wgzhao/addax/core/statistics/plugin/task/AbstractTaskPluginCollector.java b/core/src/main/java/com/wgzhao/addax/core/statistics/plugin/task/AbstractTaskPluginCollector.java index e22a2f37f..e2e754b48 100644 --- a/core/src/main/java/com/wgzhao/addax/core/statistics/plugin/task/AbstractTaskPluginCollector.java +++ b/core/src/main/java/com/wgzhao/addax/core/statistics/plugin/task/AbstractTaskPluginCollector.java @@ -26,10 +26,11 @@ import com.wgzhao.addax.common.util.Configuration; import com.wgzhao.addax.core.statistics.communication.Communication; import com.wgzhao.addax.core.statistics.communication.CommunicationTool; -import com.wgzhao.addax.core.util.FrameworkErrorCode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; + /** * Created by jingxing on 14-9-11. */ @@ -90,7 +91,7 @@ else if (this.pluginType.equals(PluginType.WRITER)) { this.communication.increaseCounter(CommunicationTool.WRITE_FAILED_BYTES, dirtyRecord.getByteSize()); } else { - throw AddaxException.asAddaxException(FrameworkErrorCode.RUNTIME_ERROR, String.format("不知道的插件类型[%s].", this.pluginType)); + throw AddaxException.asAddaxException(RUNTIME_ERROR, String.format("不知道的插件类型[%s].", this.pluginType)); } } } diff --git a/core/src/main/java/com/wgzhao/addax/core/statistics/plugin/task/util/DirtyRecord.java b/core/src/main/java/com/wgzhao/addax/core/statistics/plugin/task/util/DirtyRecord.java index 507d6ffb8..44a242334 100644 --- a/core/src/main/java/com/wgzhao/addax/core/statistics/plugin/task/util/DirtyRecord.java +++ b/core/src/main/java/com/wgzhao/addax/core/statistics/plugin/task/util/DirtyRecord.java @@ -23,7 +23,6 @@ import com.wgzhao.addax.common.element.Column; import com.wgzhao.addax.common.element.Record; import com.wgzhao.addax.common.exception.AddaxException; -import com.wgzhao.addax.core.util.FrameworkErrorCode; import java.math.BigDecimal; import java.math.BigInteger; @@ -33,6 +32,8 @@ import java.util.List; import java.util.Map; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; + public class DirtyRecord implements Record { @@ -68,31 +69,31 @@ public String toString() @Override public void setColumn(int i, Column column) { - throw AddaxException.asAddaxException(FrameworkErrorCode.RUNTIME_ERROR, NOT_SUPPORT_METHOD); + throw AddaxException.asAddaxException(RUNTIME_ERROR, NOT_SUPPORT_METHOD); } @Override public Column getColumn(int i) { - throw AddaxException.asAddaxException(FrameworkErrorCode.RUNTIME_ERROR, NOT_SUPPORT_METHOD); + throw AddaxException.asAddaxException(RUNTIME_ERROR, NOT_SUPPORT_METHOD); } @Override public int getColumnNumber() { - throw AddaxException.asAddaxException(FrameworkErrorCode.RUNTIME_ERROR, NOT_SUPPORT_METHOD); + throw AddaxException.asAddaxException(RUNTIME_ERROR, NOT_SUPPORT_METHOD); } @Override public int getByteSize() { - throw AddaxException.asAddaxException(FrameworkErrorCode.RUNTIME_ERROR, NOT_SUPPORT_METHOD); + throw AddaxException.asAddaxException(RUNTIME_ERROR, NOT_SUPPORT_METHOD); } @Override public int getMemorySize() { - throw AddaxException.asAddaxException(FrameworkErrorCode.RUNTIME_ERROR, NOT_SUPPORT_METHOD); + throw AddaxException.asAddaxException(RUNTIME_ERROR, NOT_SUPPORT_METHOD); } @Override @@ -145,54 +146,54 @@ public static Column asDirtyColumn(final Column column, int index) @Override public Long asLong() { - throw AddaxException.asAddaxException(FrameworkErrorCode.RUNTIME_ERROR, NOT_SUPPORT_METHOD); + throw AddaxException.asAddaxException(RUNTIME_ERROR, NOT_SUPPORT_METHOD); } @Override public Double asDouble() { - throw AddaxException.asAddaxException(FrameworkErrorCode.RUNTIME_ERROR, NOT_SUPPORT_METHOD); + throw AddaxException.asAddaxException(RUNTIME_ERROR, NOT_SUPPORT_METHOD); } @Override public String asString() { - throw AddaxException.asAddaxException(FrameworkErrorCode.RUNTIME_ERROR, NOT_SUPPORT_METHOD); + throw AddaxException.asAddaxException(RUNTIME_ERROR, NOT_SUPPORT_METHOD); } @Override public Date asDate() { - throw AddaxException.asAddaxException(FrameworkErrorCode.RUNTIME_ERROR, NOT_SUPPORT_METHOD); + throw AddaxException.asAddaxException(RUNTIME_ERROR, NOT_SUPPORT_METHOD); } @Override public byte[] asBytes() { - throw AddaxException.asAddaxException(FrameworkErrorCode.RUNTIME_ERROR, NOT_SUPPORT_METHOD); + throw AddaxException.asAddaxException(RUNTIME_ERROR, NOT_SUPPORT_METHOD); } @Override public Boolean asBoolean() { - throw AddaxException.asAddaxException(FrameworkErrorCode.RUNTIME_ERROR, NOT_SUPPORT_METHOD); + throw AddaxException.asAddaxException(RUNTIME_ERROR, NOT_SUPPORT_METHOD); } @Override public BigDecimal asBigDecimal() { - throw AddaxException.asAddaxException(FrameworkErrorCode.RUNTIME_ERROR, NOT_SUPPORT_METHOD); + throw AddaxException.asAddaxException(RUNTIME_ERROR, NOT_SUPPORT_METHOD); } @Override public BigInteger asBigInteger() { - throw AddaxException.asAddaxException(FrameworkErrorCode.RUNTIME_ERROR, NOT_SUPPORT_METHOD); + throw AddaxException.asAddaxException(RUNTIME_ERROR, NOT_SUPPORT_METHOD); } @Override public Timestamp asTimestamp() { - throw AddaxException.asAddaxException(FrameworkErrorCode.RUNTIME_ERROR, NOT_SUPPORT_METHOD); + throw AddaxException.asAddaxException(RUNTIME_ERROR, NOT_SUPPORT_METHOD); } } diff --git a/core/src/main/java/com/wgzhao/addax/core/taskgroup/TaskGroupContainer.java b/core/src/main/java/com/wgzhao/addax/core/taskgroup/TaskGroupContainer.java index ee52c0a49..5b6cf6ad7 100644 --- a/core/src/main/java/com/wgzhao/addax/core/taskgroup/TaskGroupContainer.java +++ b/core/src/main/java/com/wgzhao/addax/core/taskgroup/TaskGroupContainer.java @@ -22,7 +22,7 @@ import com.alibaba.fastjson2.JSON; import com.wgzhao.addax.common.constant.PluginType; import com.wgzhao.addax.common.exception.AddaxException; -import com.wgzhao.addax.common.exception.CommonErrorCode; +import com.wgzhao.addax.common.spi.ErrorCode; import com.wgzhao.addax.common.plugin.RecordSender; import com.wgzhao.addax.common.plugin.TaskPluginCollector; import com.wgzhao.addax.common.util.Configuration; @@ -42,7 +42,6 @@ import com.wgzhao.addax.core.transport.exchanger.BufferedRecordTransformerExchanger; import com.wgzhao.addax.core.transport.transformer.TransformerExecution; import com.wgzhao.addax.core.util.ClassUtil; -import com.wgzhao.addax.core.util.FrameworkErrorCode; import com.wgzhao.addax.core.util.TransformerUtil; import com.wgzhao.addax.core.util.container.CoreConstant; import com.wgzhao.addax.core.util.container.LoadUtil; @@ -56,6 +55,9 @@ import java.util.List; import java.util.Map; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; + public class TaskGroupContainer extends AbstractContainer { @@ -185,7 +187,7 @@ else if (taskCommunication.getState() == State.SUCCEEDED) { if (failedOrKilled) { lastTaskGroupContainerCommunication = reportTaskGroupCommunication(lastTaskGroupContainerCommunication, taskCountInThisTaskGroup); - throw AddaxException.asAddaxException(FrameworkErrorCode.PLUGIN_RUNTIME_ERROR, lastTaskGroupContainerCommunication.getThrowable()); + throw AddaxException.asAddaxException(RUNTIME_ERROR, lastTaskGroupContainerCommunication.getThrowable()); } //3.有任务未执行，且正在运行的任务数小于最大通道限制 @@ -206,7 +208,7 @@ else if (taskCommunication.getState() == State.SUCCEEDED) { if (now - failedTime > taskMaxWaitInMs) { markCommunicationFailed(taskId); reportTaskGroupCommunication(lastTaskGroupContainerCommunication, taskCountInThisTaskGroup); - throw AddaxException.asAddaxException(CommonErrorCode.WAIT_TIME_EXCEED, "The task failover wait timed out."); + throw AddaxException.asAddaxException(ErrorCode.WAIT_TIME_EXCEED, "The task failover wait timed out."); } else { lastExecutor.shutdown(); //try to close again @@ -272,7 +274,7 @@ else if (taskCommunication.getState() == State.SUCCEEDED) { this.containerCommunicator.report(nowTaskGroupContainerCommunication); throw AddaxException.asAddaxException( - FrameworkErrorCode.RUNTIME_ERROR, e); + RUNTIME_ERROR, e); } } @@ -415,7 +417,7 @@ public void doStart() // reader没有起来，writer不可能结束 if (!this.writerThread.isAlive() || this.taskCommunication.getState() == State.FAILED) { - throw AddaxException.asAddaxException(FrameworkErrorCode.RUNTIME_ERROR, this.taskCommunication.getThrowable()); + throw AddaxException.asAddaxException(RUNTIME_ERROR, this.taskCommunication.getThrowable()); } this.readerThread.start(); @@ -423,7 +425,7 @@ public void doStart() // 这里reader可能很快结束 if (!this.readerThread.isAlive() && this.taskCommunication.getState() == State.FAILED) { // 这里有可能出现Reader线上启动即挂情况 对于这类情况 需要立刻抛出异常 - throw AddaxException.asAddaxException(FrameworkErrorCode.RUNTIME_ERROR, this.taskCommunication.getThrowable()); + throw AddaxException.asAddaxException(RUNTIME_ERROR, this.taskCommunication.getThrowable()); } } @@ -471,7 +473,7 @@ private AbstractRunner generateRunner(PluginType pluginType, List rs) notEmpty.signalAll(); } catch (InterruptedException e) { - throw AddaxException.asAddaxException(FrameworkErrorCode.RUNTIME_ERROR, e); + throw AddaxException.asAddaxException(RUNTIME_ERROR, e); } finally { lock.unlock(); @@ -151,7 +152,7 @@ protected void doPullAll(Collection rs) notInsufficient.signalAll(); } catch (InterruptedException e) { - throw AddaxException.asAddaxException(FrameworkErrorCode.RUNTIME_ERROR, e); + throw AddaxException.asAddaxException(RUNTIME_ERROR, e); } finally { lock.unlock(); diff --git a/core/src/main/java/com/wgzhao/addax/core/transport/exchanger/BufferedRecordExchanger.java b/core/src/main/java/com/wgzhao/addax/core/transport/exchanger/BufferedRecordExchanger.java index 96bd8e21d..f38634461 100644 --- a/core/src/main/java/com/wgzhao/addax/core/transport/exchanger/BufferedRecordExchanger.java +++ b/core/src/main/java/com/wgzhao/addax/core/transport/exchanger/BufferedRecordExchanger.java @@ -20,7 +20,7 @@ package com.wgzhao.addax.core.transport.exchanger; import com.wgzhao.addax.common.element.Record; -import com.wgzhao.addax.common.exception.CommonErrorCode; +import com.wgzhao.addax.common.spi.ErrorCode; import com.wgzhao.addax.common.exception.AddaxException; import com.wgzhao.addax.common.plugin.RecordReceiver; import com.wgzhao.addax.common.plugin.RecordSender; @@ -28,7 +28,6 @@ import com.wgzhao.addax.common.util.Configuration; import com.wgzhao.addax.core.transport.channel.Channel; import com.wgzhao.addax.core.transport.record.TerminateRecord; -import com.wgzhao.addax.core.util.FrameworkErrorCode; import com.wgzhao.addax.core.util.container.CoreConstant; import org.apache.commons.lang3.Validate; import org.slf4j.Logger; @@ -38,6 +37,8 @@ import java.util.List; import java.util.concurrent.atomic.AtomicInteger; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; + public class BufferedRecordExchanger implements RecordSender, RecordReceiver { @@ -78,7 +79,7 @@ public BufferedRecordExchanger(Channel channel, TaskPluginCollector pluginCollec "com.wgzhao.addax.core.transport.record.DefaultRecord"))); } catch (Exception e) { - throw AddaxException.asAddaxException(FrameworkErrorCode.CONFIG_ERROR, e); + throw AddaxException.asAddaxException(CONFIG_ERROR, e); } } @@ -89,7 +90,7 @@ public Record createRecord() return BufferedRecordExchanger.recordClass.getConstructor().newInstance(); } catch (Exception e) { - throw AddaxException.asAddaxException(FrameworkErrorCode.CONFIG_ERROR, e); + throw AddaxException.asAddaxException(CONFIG_ERROR, e); } } @@ -97,7 +98,7 @@ public Record createRecord() public void sendToWriter(Record record) { if (shutdown) { - throw AddaxException.asAddaxException(CommonErrorCode.SHUT_DOWN_TASK, ""); + throw AddaxException.asAddaxException(ErrorCode.SHUT_DOWN_TASK, ""); } Validate.notNull(record, "The record cannot be empty."); @@ -123,7 +124,7 @@ public void sendToWriter(Record record) public void flush() { if (shutdown) { - throw AddaxException.asAddaxException(CommonErrorCode.SHUT_DOWN_TASK, ""); + throw AddaxException.asAddaxException(ErrorCode.SHUT_DOWN_TASK, ""); } this.channel.pushAll(this.buffer); this.buffer.clear(); @@ -135,7 +136,7 @@ public void flush() public void terminate() { if (shutdown) { - throw AddaxException.asAddaxException(CommonErrorCode.SHUT_DOWN_TASK, ""); + throw AddaxException.asAddaxException(ErrorCode.SHUT_DOWN_TASK, ""); } flush(); this.channel.pushTerminate(TerminateRecord.get()); @@ -145,7 +146,7 @@ public void terminate() public Record getFromReader() { if (shutdown) { - throw AddaxException.asAddaxException(CommonErrorCode.SHUT_DOWN_TASK, ""); + throw AddaxException.asAddaxException(ErrorCode.SHUT_DOWN_TASK, ""); } boolean isEmpty = (this.bufferIndex >= this.buffer.size()); if (isEmpty) { diff --git a/core/src/main/java/com/wgzhao/addax/core/transport/exchanger/BufferedRecordTransformerExchanger.java b/core/src/main/java/com/wgzhao/addax/core/transport/exchanger/BufferedRecordTransformerExchanger.java index e2a68a9e7..36c5c94be 100644 --- a/core/src/main/java/com/wgzhao/addax/core/transport/exchanger/BufferedRecordTransformerExchanger.java +++ b/core/src/main/java/com/wgzhao/addax/core/transport/exchanger/BufferedRecordTransformerExchanger.java @@ -20,7 +20,7 @@ package com.wgzhao.addax.core.transport.exchanger; import com.wgzhao.addax.common.element.Record; -import com.wgzhao.addax.common.exception.CommonErrorCode; +import com.wgzhao.addax.common.spi.ErrorCode; import com.wgzhao.addax.common.exception.AddaxException; import com.wgzhao.addax.common.plugin.RecordReceiver; import com.wgzhao.addax.common.plugin.RecordSender; @@ -30,7 +30,6 @@ import com.wgzhao.addax.core.transport.channel.Channel; import com.wgzhao.addax.core.transport.record.TerminateRecord; import com.wgzhao.addax.core.transport.transformer.TransformerExecution; -import com.wgzhao.addax.core.util.FrameworkErrorCode; import com.wgzhao.addax.core.util.container.CoreConstant; import org.apache.commons.lang3.Validate; @@ -38,6 +37,8 @@ import java.util.List; import java.util.concurrent.atomic.AtomicInteger; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; + public class BufferedRecordTransformerExchanger extends TransformerExchanger implements RecordSender, RecordReceiver @@ -79,8 +80,7 @@ public BufferedRecordTransformerExchanger(int taskGroupId, int taskId, "com.wgzhao.addax.core.transport.record.DefaultRecord"))); } catch (Exception e) { - throw AddaxException.asAddaxException( - FrameworkErrorCode.CONFIG_ERROR, e); + throw AddaxException.asAddaxException(CONFIG_ERROR, e); } } @@ -91,8 +91,7 @@ public Record createRecord() return BufferedRecordTransformerExchanger.RECORD_CLASS.getConstructor().newInstance(); } catch (Exception e) { - throw AddaxException.asAddaxException( - FrameworkErrorCode.CONFIG_ERROR, e); + throw AddaxException.asAddaxException(CONFIG_ERROR, e); } } @@ -100,7 +99,7 @@ public Record createRecord() public void sendToWriter(Record record) { if (shutdown) { - throw AddaxException.asAddaxException(CommonErrorCode.SHUT_DOWN_TASK, ""); + throw AddaxException.asAddaxException(ErrorCode.SHUT_DOWN_TASK, ""); } Validate.notNull(record, "The record cannot be empty."); @@ -132,7 +131,7 @@ record = doTransformer(record); public void flush() { if (shutdown) { - throw AddaxException.asAddaxException(CommonErrorCode.SHUT_DOWN_TASK, ""); + throw AddaxException.asAddaxException(ErrorCode.SHUT_DOWN_TASK, ""); } this.channel.pushAll(this.buffer); //和channel的统计保持同步 @@ -146,7 +145,7 @@ public void flush() public void terminate() { if (shutdown) { - throw AddaxException.asAddaxException(CommonErrorCode.SHUT_DOWN_TASK, ""); + throw AddaxException.asAddaxException(ErrorCode.SHUT_DOWN_TASK, ""); } flush(); this.channel.pushTerminate(TerminateRecord.get()); @@ -156,7 +155,7 @@ public void terminate() public Record getFromReader() { if (shutdown) { - throw AddaxException.asAddaxException(CommonErrorCode.SHUT_DOWN_TASK, ""); + throw AddaxException.asAddaxException(ErrorCode.SHUT_DOWN_TASK, ""); } boolean isEmpty = (this.bufferIndex >= this.buffer.size()); if (isEmpty) { diff --git a/core/src/main/java/com/wgzhao/addax/core/transport/exchanger/RecordExchanger.java b/core/src/main/java/com/wgzhao/addax/core/transport/exchanger/RecordExchanger.java index a301a6c9c..b73863671 100644 --- a/core/src/main/java/com/wgzhao/addax/core/transport/exchanger/RecordExchanger.java +++ b/core/src/main/java/com/wgzhao/addax/core/transport/exchanger/RecordExchanger.java @@ -20,7 +20,7 @@ package com.wgzhao.addax.core.transport.exchanger; import com.wgzhao.addax.common.element.Record; -import com.wgzhao.addax.common.exception.CommonErrorCode; +import com.wgzhao.addax.common.spi.ErrorCode; import com.wgzhao.addax.common.exception.AddaxException; import com.wgzhao.addax.common.plugin.RecordReceiver; import com.wgzhao.addax.common.plugin.RecordSender; @@ -30,11 +30,12 @@ import com.wgzhao.addax.core.transport.channel.Channel; import com.wgzhao.addax.core.transport.record.TerminateRecord; import com.wgzhao.addax.core.transport.transformer.TransformerExecution; -import com.wgzhao.addax.core.util.FrameworkErrorCode; import com.wgzhao.addax.core.util.container.CoreConstant; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; + public class RecordExchanger extends TransformerExchanger implements RecordSender, RecordReceiver @@ -59,8 +60,7 @@ public RecordExchanger(int taskGroupId, int taskId, Channel channel, Communicati "com.wgzhao.addax.core.transport.record.DefaultRecord")); } catch (ClassNotFoundException e) { - throw AddaxException.asAddaxException( - FrameworkErrorCode.CONFIG_ERROR, e); + throw AddaxException.asAddaxException(CONFIG_ERROR, e); } } @@ -68,7 +68,7 @@ public RecordExchanger(int taskGroupId, int taskId, Channel channel, Communicati public Record getFromReader() { if (shutdown) { - throw AddaxException.asAddaxException(CommonErrorCode.SHUT_DOWN_TASK, ""); + throw AddaxException.asAddaxException(ErrorCode.SHUT_DOWN_TASK, ""); } Record record = this.channel.pull(); return (record instanceof TerminateRecord ? null : record); @@ -81,8 +81,7 @@ public Record createRecord() return RECORD_CLASS.getConstructor().newInstance(); } catch (Exception e) { - throw AddaxException.asAddaxException( - FrameworkErrorCode.CONFIG_ERROR, e); + throw AddaxException.asAddaxException(CONFIG_ERROR, e); } } @@ -90,7 +89,7 @@ public Record createRecord() public void sendToWriter(Record record) { if (shutdown) { - throw AddaxException.asAddaxException(CommonErrorCode.SHUT_DOWN_TASK, ""); + throw AddaxException.asAddaxException(ErrorCode.SHUT_DOWN_TASK, ""); } record = doTransformer(record); if (record == null) { @@ -110,7 +109,7 @@ public void flush() public void terminate() { if (shutdown) { - throw AddaxException.asAddaxException(CommonErrorCode.SHUT_DOWN_TASK, ""); + throw AddaxException.asAddaxException(ErrorCode.SHUT_DOWN_TASK, ""); } this.channel.pushTerminate(TerminateRecord.get()); //和channel的统计保持同步 diff --git a/core/src/main/java/com/wgzhao/addax/core/transport/exchanger/TransformerExchanger.java b/core/src/main/java/com/wgzhao/addax/core/transport/exchanger/TransformerExchanger.java index bdaf8e9e6..38ad98221 100644 --- a/core/src/main/java/com/wgzhao/addax/core/transport/exchanger/TransformerExchanger.java +++ b/core/src/main/java/com/wgzhao/addax/core/transport/exchanger/TransformerExchanger.java @@ -24,12 +24,13 @@ import com.wgzhao.addax.common.plugin.TaskPluginCollector; import com.wgzhao.addax.core.statistics.communication.Communication; import com.wgzhao.addax.core.statistics.communication.CommunicationTool; -import com.wgzhao.addax.core.transport.transformer.TransformerErrorCode; import com.wgzhao.addax.core.transport.transformer.TransformerExecution; import com.wgzhao.addax.core.util.container.ClassLoaderSwapper; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; + /** * no comments. * Created by liqiang on 16/3/9. @@ -87,7 +88,7 @@ public Record doTransformer(Record record) if (transformerInfoExec.getColumnIndex() != null && transformerInfoExec.getColumnIndex() >= record.getColumnNumber()) { - throw AddaxException.asAddaxException(TransformerErrorCode.TRANSFORMER_ILLEGAL_PARAMETER, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, String.format("columnIndex[%s] out of bound[%s]. name=%s", transformerInfoExec.getColumnIndex(), record.getColumnNumber(), transformerInfoExec.getTransformerName())); diff --git a/core/src/main/java/com/wgzhao/addax/core/transport/record/DefaultRecord.java b/core/src/main/java/com/wgzhao/addax/core/transport/record/DefaultRecord.java index f94aa97e5..9197f93e3 100644 --- a/core/src/main/java/com/wgzhao/addax/core/transport/record/DefaultRecord.java +++ b/core/src/main/java/com/wgzhao/addax/core/transport/record/DefaultRecord.java @@ -24,13 +24,14 @@ import com.wgzhao.addax.common.element.Record; import com.wgzhao.addax.common.exception.AddaxException; import com.wgzhao.addax.core.util.ClassSize; -import com.wgzhao.addax.core.util.FrameworkErrorCode; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; + /** * Created by jingxing on 14-8-24. */ @@ -75,7 +76,7 @@ public Column getColumn(int i) public void setColumn(int i, Column column) { if (i < 0) { - throw AddaxException.asAddaxException(FrameworkErrorCode.ARGUMENT_ERROR, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "Cannot set a value for a column with an index less than 0"); } diff --git a/core/src/main/java/com/wgzhao/addax/core/transport/transformer/FilterTransformer.java b/core/src/main/java/com/wgzhao/addax/core/transport/transformer/FilterTransformer.java index cd04444af..7bfb4c046 100644 --- a/core/src/main/java/com/wgzhao/addax/core/transport/transformer/FilterTransformer.java +++ b/core/src/main/java/com/wgzhao/addax/core/transport/transformer/FilterTransformer.java @@ -33,6 +33,9 @@ import java.util.Arrays; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; + /** * no comments. * Created by liqiang on 16/3/4. @@ -67,7 +70,7 @@ public Record evaluate(Record record, Object... paras) } } catch (Exception e) { - throw AddaxException.asAddaxException(TransformerErrorCode.TRANSFORMER_ILLEGAL_PARAMETER, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "paras:" + Arrays.asList(paras) + " => " + e.getMessage()); } @@ -105,7 +108,7 @@ else if ("<=".equalsIgnoreCase(code)) { } catch (Exception e) { throw AddaxException.asAddaxException( - TransformerErrorCode.TRANSFORMER_RUN_EXCEPTION, e.getMessage(), e); + RUNTIME_ERROR, e.getMessage(), e); } } diff --git a/core/src/main/java/com/wgzhao/addax/core/transport/transformer/GroovyTransformer.java b/core/src/main/java/com/wgzhao/addax/core/transport/transformer/GroovyTransformer.java index 01ba9afbd..9a1cea4a3 100644 --- a/core/src/main/java/com/wgzhao/addax/core/transport/transformer/GroovyTransformer.java +++ b/core/src/main/java/com/wgzhao/addax/core/transport/transformer/GroovyTransformer.java @@ -29,6 +29,9 @@ import java.util.Arrays; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; + /** * no comments. * Created by liqiang on 16/3/4. @@ -51,7 +54,7 @@ public Record evaluate(Record record, Object... paras) //全局唯一 if (paras.length < 1 || paras.length > 2) { throw AddaxException.asAddaxException( - TransformerErrorCode.TRANSFORMER_ILLEGAL_PARAMETER, + ILLEGAL_VALUE, "The dx_groovy parameters must be 1 or 2. The current parameter is: " + Arrays.asList(paras)); } synchronized (this) { @@ -79,21 +82,21 @@ private void initGroovyTransformer(String code, List extraPackage) } catch (CompilationFailedException cfe) { throw AddaxException.asAddaxException( - TransformerErrorCode.TRANSFORMER_GROOVY_INIT_EXCEPTION, cfe); + RUNTIME_ERROR, cfe); } try { Object t = groovyClass.getConstructor().newInstance(); if (!(t instanceof Transformer)) { throw AddaxException.asAddaxException( - TransformerErrorCode.TRANSFORMER_GROOVY_INIT_EXCEPTION, + RUNTIME_ERROR, "Addax bug! "); } this.groovyTransformer = (Transformer) t; } catch (Throwable ex) { throw AddaxException.asAddaxException( - TransformerErrorCode.TRANSFORMER_GROOVY_INIT_EXCEPTION, ex); + RUNTIME_ERROR, ex); } } diff --git a/core/src/main/java/com/wgzhao/addax/core/transport/transformer/MapTransformer.java b/core/src/main/java/com/wgzhao/addax/core/transport/transformer/MapTransformer.java index 096557283..5ff7198f7 100644 --- a/core/src/main/java/com/wgzhao/addax/core/transport/transformer/MapTransformer.java +++ b/core/src/main/java/com/wgzhao/addax/core/transport/transformer/MapTransformer.java @@ -27,6 +27,8 @@ import java.util.Arrays; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; import static com.wgzhao.addax.common.util.MathUtil.add; import static com.wgzhao.addax.common.util.MathUtil.divide; import static com.wgzhao.addax.common.util.MathUtil.mod; @@ -75,7 +77,7 @@ public Record evaluate(Record record, Object... paras) } catch (Exception e) { throw AddaxException.asAddaxException( - TransformerErrorCode.TRANSFORMER_ILLEGAL_PARAMETER, + ILLEGAL_VALUE, "paras:" + Arrays.asList(paras) + " => " + e.getMessage()); } @@ -111,7 +113,7 @@ public Record evaluate(Record record, Object... paras) } catch (Exception e) { throw AddaxException.asAddaxException( - TransformerErrorCode.TRANSFORMER_RUN_EXCEPTION, e.getMessage(), e); + RUNTIME_ERROR, e.getMessage(), e); } } } diff --git a/core/src/main/java/com/wgzhao/addax/core/transport/transformer/PadTransformer.java b/core/src/main/java/com/wgzhao/addax/core/transport/transformer/PadTransformer.java index ac6ffafed..8a49c7598 100644 --- a/core/src/main/java/com/wgzhao/addax/core/transport/transformer/PadTransformer.java +++ b/core/src/main/java/com/wgzhao/addax/core/transport/transformer/PadTransformer.java @@ -27,6 +27,9 @@ import java.util.Arrays; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; + /** * no comments. * Created by liqiang on 16/3/4. @@ -60,7 +63,7 @@ public Record evaluate(Record record, Object... paras) } catch (Exception e) { throw AddaxException.asAddaxException( - TransformerErrorCode.TRANSFORMER_ILLEGAL_PARAMETER, + ILLEGAL_VALUE, "paras:" + Arrays.asList(paras) + " => " + e.getMessage()); } @@ -90,7 +93,7 @@ public Record evaluate(Record record, Object... paras) } catch (Exception e) { throw AddaxException.asAddaxException( - TransformerErrorCode.TRANSFORMER_RUN_EXCEPTION, e.getMessage(), e); + RUNTIME_ERROR, e.getMessage(), e); } return record; } diff --git a/core/src/main/java/com/wgzhao/addax/core/transport/transformer/ReplaceTransformer.java b/core/src/main/java/com/wgzhao/addax/core/transport/transformer/ReplaceTransformer.java index ca785dc35..de04b7307 100644 --- a/core/src/main/java/com/wgzhao/addax/core/transport/transformer/ReplaceTransformer.java +++ b/core/src/main/java/com/wgzhao/addax/core/transport/transformer/ReplaceTransformer.java @@ -27,6 +27,9 @@ import java.util.Arrays; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; + /** * no comments. * Created by liqiang on 16/3/4. @@ -58,7 +61,7 @@ public Record evaluate(Record record, Object... paras) replaceString = (String) paras[3]; } catch (Exception e) { - throw AddaxException.asAddaxException(TransformerErrorCode.TRANSFORMER_ILLEGAL_PARAMETER, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "paras:" + Arrays.asList(paras) + " => " + e.getMessage()); } @@ -88,7 +91,7 @@ public Record evaluate(Record record, Object... paras) } catch (Exception e) { throw AddaxException.asAddaxException( - TransformerErrorCode.TRANSFORMER_RUN_EXCEPTION, e.getMessage(), e); + RUNTIME_ERROR, e.getMessage(), e); } return record; } diff --git a/core/src/main/java/com/wgzhao/addax/core/transport/transformer/SubstrTransformer.java b/core/src/main/java/com/wgzhao/addax/core/transport/transformer/SubstrTransformer.java index 1ead8a2e9..9bdd93b67 100644 --- a/core/src/main/java/com/wgzhao/addax/core/transport/transformer/SubstrTransformer.java +++ b/core/src/main/java/com/wgzhao/addax/core/transport/transformer/SubstrTransformer.java @@ -27,6 +27,9 @@ import java.util.Arrays; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; + /** * no comments. * Created by liqiang on 16/3/4. @@ -57,7 +60,7 @@ public Record evaluate(Record record, Object... paras) length = Integer.parseInt((String) paras[2]); } catch (Exception e) { - throw AddaxException.asAddaxException(TransformerErrorCode.TRANSFORMER_ILLEGAL_PARAMETER, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "paras:" + Arrays.asList(paras) + " => " + e.getMessage()); } @@ -85,7 +88,7 @@ public Record evaluate(Record record, Object... paras) } catch (Exception e) { throw AddaxException.asAddaxException( - TransformerErrorCode.TRANSFORMER_RUN_EXCEPTION, e.getMessage(), e); + RUNTIME_ERROR, e.getMessage(), e); } return record; } diff --git a/core/src/main/java/com/wgzhao/addax/core/transport/transformer/TransformerErrorCode.java b/core/src/main/java/com/wgzhao/addax/core/transport/transformer/TransformerErrorCode.java deleted file mode 100644 index c38acebec..000000000 --- a/core/src/main/java/com/wgzhao/addax/core/transport/transformer/TransformerErrorCode.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.core.transport.transformer; - -import com.wgzhao.addax.common.spi.ErrorCode; - -public enum TransformerErrorCode - implements ErrorCode -{ - //重复命名 - TRANSFORMER_NAME_ERROR("TransformerErrorCode-01", "Transformer name illegal"), - TRANSFORMER_DUPLICATE_ERROR("TransformerErrorCode-02", "Transformer name has existed"), - TRANSFORMER_NOTFOUND_ERROR("TransformerErrorCode-03", "Transformer name not found"), - TRANSFORMER_CONFIGURATION_ERROR("TransformerErrorCode-04", "Transformer configuration error"), - TRANSFORMER_ILLEGAL_PARAMETER("TransformerErrorCode-05", "Transformer parameter illegal"), - TRANSFORMER_RUN_EXCEPTION("TransformerErrorCode-06", "Transformer run exception"), - TRANSFORMER_GROOVY_INIT_EXCEPTION("TransformerErrorCode-07", "Transformer Groovy init exception"), - ; - - private final String code; - - private final String description; - - TransformerErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s]. ", this.code, - this.description); - } -} diff --git a/core/src/main/java/com/wgzhao/addax/core/transport/transformer/TransformerRegistry.java b/core/src/main/java/com/wgzhao/addax/core/transport/transformer/TransformerRegistry.java index 18d1efa27..45bd67c9e 100644 --- a/core/src/main/java/com/wgzhao/addax/core/transport/transformer/TransformerRegistry.java +++ b/core/src/main/java/com/wgzhao/addax/core/transport/transformer/TransformerRegistry.java @@ -36,6 +36,8 @@ import java.util.List; import java.util.Map; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; + /** * no comments. * Created by liqiang on 16/3/3. @@ -144,7 +146,7 @@ public static synchronized void registryTransformer(Transformer transformer, if (registeredTransformer.containsKey(transformer.getTransformerName())) { throw AddaxException.asAddaxException( - TransformerErrorCode.TRANSFORMER_DUPLICATE_ERROR, + CONFIG_ERROR, " name=" + transformer.getTransformerName()); } @@ -161,7 +163,7 @@ public static synchronized void registryComplexTransformer(ComplexTransformer co if (registeredTransformer.containsKey(complexTransformer.getTransformerName())) { throw AddaxException.asAddaxException( - TransformerErrorCode.TRANSFORMER_DUPLICATE_ERROR, + CONFIG_ERROR, " name=" + complexTransformer.getTransformerName()); } @@ -185,7 +187,7 @@ private static void checkName(String functionName, boolean isNative) if (!checkResult) { throw AddaxException.asAddaxException( - TransformerErrorCode.TRANSFORMER_NAME_ERROR, + CONFIG_ERROR, " name=" + functionName + ": isNative=" + isNative); } } diff --git a/core/src/main/java/com/wgzhao/addax/core/util/ConfigParser.java b/core/src/main/java/com/wgzhao/addax/core/util/ConfigParser.java index 63a3dd4bf..e207e58c0 100644 --- a/core/src/main/java/com/wgzhao/addax/core/util/ConfigParser.java +++ b/core/src/main/java/com/wgzhao/addax/core/util/ConfigParser.java @@ -39,6 +39,9 @@ import org.slf4j.LoggerFactory; import static com.wgzhao.addax.common.base.Key.CONNECTION; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.PLUGIN_INIT_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; import static com.wgzhao.addax.core.util.container.CoreConstant.CONF_PATH; import static com.wgzhao.addax.core.util.container.CoreConstant.CORE_SERVER_TIMEOUT_SEC; import static com.wgzhao.addax.core.util.container.CoreConstant.JOB_CONTENT; @@ -54,17 +57,19 @@ import static com.wgzhao.addax.core.util.container.CoreConstant.PLUGIN_READER_HOME; import static com.wgzhao.addax.core.util.container.CoreConstant.PLUGIN_WRITER_HOME; - -public final class ConfigParser { +public final class ConfigParser +{ private static final Logger LOG = LoggerFactory.getLogger(ConfigParser.class); - private ConfigParser() { + private ConfigParser() + { } /* * 指定Job配置路径，ConfigParser会解析Job、Plugin、Core全部信息，并以Configuration返回 */ - public static Configuration parse(String jobPath) { + public static Configuration parse(String jobPath) + { Configuration configuration = ConfigParser.parseJobConfig(jobPath); // Upgrade the new job format to the old one @@ -90,18 +95,7 @@ public static Configuration parse(String jobPath) { if (StringUtils.isNotEmpty(postHandlerName)) { pluginList.add(postHandlerName); } - try { - configuration.merge(parsePluginConfig(new ArrayList<>(pluginList)), false); - } catch (Exception e) { - //吞掉异常，保持log干净。这里message足够。 - LOG.warn("Failed to load plugin(s) [{},{}]: {}, try after 1 second.", readerPluginName, writerPluginName, e.getMessage()); - try { - Thread.sleep(1000); - } catch (InterruptedException e1) { - // - } - configuration.merge(parsePluginConfig(new ArrayList<>(pluginList)), false); - } + configuration.merge(parsePluginConfig(new ArrayList<>(pluginList)), false); return configuration; } @@ -112,7 +106,9 @@ public static Configuration parse(String jobPath) { * * @param configuration {@link Configuration} */ - private static void upgradeJobConfig(Configuration configuration) { + private static void upgradeJobConfig(Configuration configuration) + { + configuration.getNecessaryValue(JOB_CONTENT); if (configuration.getString(JOB_CONTENT).startsWith("[")) { // get the first element List contentList = configuration.getList(JOB_CONTENT, Map.class); @@ -135,16 +131,19 @@ private static void upgradeJobConfig(Configuration configuration) { } } - private static Configuration parseCoreConfig() { + private static Configuration parseCoreConfig() + { return Configuration.from(new File(CONF_PATH)); } - public static Configuration parseJobConfig(String path) { + public static Configuration parseJobConfig(String path) + { String jobContent = getJobContent(path); return Configuration.from(jobContent); } - private static String getJobContent(String jobResource) { + private static String getJobContent(String jobResource) + { String jobContent; boolean isJobResourceFromHttp = jobResource.trim().toLowerCase().startsWith("http"); @@ -162,25 +161,29 @@ private static String getJobContent(String jobResource) { httpGet.setURI(url.toURI()); jobContent = httpClientUtil.executeAndGetWithFailedRetry(httpGet, 1, 1000L); - } catch (Exception e) { - throw AddaxException.asAddaxException(FrameworkErrorCode.CONFIG_ERROR, "Failed to obtain job configuration:" + jobResource, e); } - } else { + catch (Exception e) { + throw AddaxException.asAddaxException(CONFIG_ERROR, "Failed to obtain job configuration:" + jobResource, e); + } + } + else { // jobResource 是本地文件绝对路径 try { jobContent = FileUtils.readFileToString(new File(jobResource), StandardCharsets.UTF_8); - } catch (IOException e) { - throw AddaxException.asAddaxException(FrameworkErrorCode.CONFIG_ERROR, "Failed to obtain job configuration:" + jobResource, e); + } + catch (IOException e) { + throw AddaxException.asAddaxException(CONFIG_ERROR, "Failed to obtain job configuration:" + jobResource, e); } } if (jobContent == null) { - throw AddaxException.asAddaxException(FrameworkErrorCode.CONFIG_ERROR, "Failed to obtain job configuration:" + jobResource); + throw AddaxException.asAddaxException(CONFIG_ERROR, "Failed to obtain job configuration:" + jobResource); } return jobContent; } - public static Configuration parsePluginConfig(List wantPluginNames) { + public static Configuration parsePluginConfig(List wantPluginNames) + { Configuration configuration = Configuration.newDefault(); int complete = 0; @@ -190,13 +193,19 @@ public static Configuration parsePluginConfig(List wantPluginNames) { if (plugin.endsWith("reader")) { pluginType = "reader"; pluginPath = PLUGIN_READER_HOME + File.separator + plugin; - } else { + } + else { pluginType = "writer"; pluginPath = PLUGIN_WRITER_HOME + File.separator + plugin; } String filePath = pluginPath + File.separator + "plugin.json"; - Configuration pluginConf = Configuration.from(new File(filePath)); + // check if the plugin.json file exists + File file = new File(filePath); + if (!file.exists()) { + throw AddaxException.asAddaxException(PLUGIN_INIT_ERROR, "The plugin '" + plugin + "' has not installed yet" ); + } + Configuration pluginConf = Configuration.from(file); if (StringUtils.isBlank(pluginConf.getString("path"))) { pluginConf.set("path", pluginPath); } @@ -207,25 +216,26 @@ public static Configuration parsePluginConfig(List wantPluginNames) { } if (!wantPluginNames.isEmpty() && wantPluginNames.size() != complete) { - throw AddaxException.asAddaxException(FrameworkErrorCode.PLUGIN_INIT_ERROR, "Plugin loading failed. The specified plugin was not loaded: " + wantPluginNames); + throw AddaxException.asAddaxException(PLUGIN_INIT_ERROR, "Plugin loading failed. The specified plugin was not loaded: " + wantPluginNames); } return configuration; } - private static void validateJob(Configuration conf) { + private static void validateJob(Configuration conf) + { final Map content = conf.getMap(JOB_CONTENT); - String[] validPaths = new String[]{JOB_CONTENT_READER, JOB_CONTENT_WRITER, JOB_CONTENT_READER_NAME, + String[] validPaths = new String[] {JOB_CONTENT_READER, JOB_CONTENT_WRITER, JOB_CONTENT_READER_NAME, JOB_CONTENT_READER_PARAMETER, JOB_CONTENT_WRITER_NAME, JOB_CONTENT_WRITER_PARAMETER}; if (content == null || content.isEmpty()) { - throw AddaxException.asAddaxException(FrameworkErrorCode.JOB_ERROR, + throw AddaxException.asAddaxException(REQUIRED_VALUE, "The configuration item '" + JOB_CONTENT + "' is required"); } for (String path : validPaths) { if (conf.get(path) == null) { - throw AddaxException.asAddaxException(FrameworkErrorCode.JOB_ERROR, + throw AddaxException.asAddaxException(REQUIRED_VALUE, "The configuration item '" + path + "' is required"); } } diff --git a/core/src/main/java/com/wgzhao/addax/core/util/ErrorRecordChecker.java b/core/src/main/java/com/wgzhao/addax/core/util/ErrorRecordChecker.java index ac8189e4b..a985b4e2f 100644 --- a/core/src/main/java/com/wgzhao/addax/core/util/ErrorRecordChecker.java +++ b/core/src/main/java/com/wgzhao/addax/core/util/ErrorRecordChecker.java @@ -28,6 +28,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static com.wgzhao.addax.common.spi.ErrorCode.OVER_LIMIT_ERROR; + /** * 检查任务是否到达错误记录限制。有检查条数（recordLimit）和百分比(percentageLimit)两种方式。 * 1. errorRecord表示出错条数不能大于限制数，当超过时任务失败。比如errorRecord为0表示不容许任何脏数据。 @@ -75,7 +77,7 @@ public void checkRecordLimit(Communication communication) if (recordLimit < errorNumber) { LOG.debug("The error limit is set to {}%. The error counter was checked.", recordLimit); throw AddaxException.asAddaxException( - FrameworkErrorCode.PLUGIN_DIRTY_DATA_LIMIT_EXCEED, + OVER_LIMIT_ERROR, String.format("The number of dirty data records did not pass the check. " + "The limit is [%d] records, but [%d] records were actually captured.", recordLimit, errorNumber)); @@ -94,7 +96,7 @@ public void checkPercentageLimit(Communication communication) if (total > 0 && ((double) error / (double) total) > percentageLimit) { throw AddaxException.asAddaxException( - FrameworkErrorCode.PLUGIN_DIRTY_DATA_LIMIT_EXCEED, + OVER_LIMIT_ERROR, String.format("The dirty data percentage check failed. The limit is [%f], but [%f] was actually captured", percentageLimit, ((double) error / (double) total))); } diff --git a/core/src/main/java/com/wgzhao/addax/core/util/FrameworkErrorCode.java b/core/src/main/java/com/wgzhao/addax/core/util/FrameworkErrorCode.java deleted file mode 100644 index ee1dd8b08..000000000 --- a/core/src/main/java/com/wgzhao/addax/core/util/FrameworkErrorCode.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.core.util; - -import com.wgzhao.addax.common.spi.ErrorCode; - -/** - *

请不要格式化本类代码

- */ -public enum FrameworkErrorCode - implements ErrorCode -{ - - ARGUMENT_ERROR("Framework-01", "Addax 引擎运行错误，该问题通常是由于内部编程错误引起，请联系Addax 开发团队解决 ."), - RUNTIME_ERROR("Framework-02", "Addax 引擎运行过程出错，具体原因请参看Addax 运行结束时的错误诊断信息 ."), - CONFIG_ERROR("Framework-03", "Addax 引擎配置错误，该问题通常是由于Addax 安装错误引起，请联系您的运维解决 ."), - JOB_ERROR("Framework-04", "job 配置文件错误，通常是由于不正确的配置项导致的，要特别仔细检查括号匹配位置是否正确."), - - PLUGIN_INSTALL_ERROR("Framework-10", "Addax 插件安装错误, 该问题通常是由于Addax 安装错误引起，请联系您的运维解决 ."), - PLUGIN_INIT_ERROR("Framework-12", "Addax 插件初始化错误, 该问题通常是由于Addax 安装错误引起，请联系您的运维解决 ."), - PLUGIN_RUNTIME_ERROR("Framework-13", "Addax 插件运行时出错, 具体原因请参看Addax 运行结束时的错误诊断信息 ."), - PLUGIN_DIRTY_DATA_LIMIT_EXCEED("Framework-14", - "Addax 传输脏数据超过用户预期，该错误通常是由于源端数据存在较多业务脏数据导致，请仔细检查Addax 汇报的脏数据日志信息, 或者您可以适当调大脏数据阈值 ."), - PLUGIN_SPLIT_ERROR("Framework-15", "Addax 插件切分出错, 该问题通常是由于Addax 各个插件编程错误引起，请联系Addax 开发团队解决"), - CALL_REMOTE_FAILED("Framework-19", "远程调用失败"), - KILLED_EXIT_VALUE("Framework-143", "Job 收到了 Kill 命令."); - - private final String code; - - private final String description; - - FrameworkErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s]. ", this.code, this.description); - } - - /* - * 通过 "Framework-143" 来标示 任务是 Killed 状态 - */ - public int toExitValue() - { - if (this == FrameworkErrorCode.KILLED_EXIT_VALUE) { - return 143; - } - else { - return 1; - } - } -} diff --git a/core/src/main/java/com/wgzhao/addax/core/util/HttpClientUtil.java b/core/src/main/java/com/wgzhao/addax/core/util/HttpClientUtil.java index f1c75b070..f0e1e6855 100644 --- a/core/src/main/java/com/wgzhao/addax/core/util/HttpClientUtil.java +++ b/core/src/main/java/com/wgzhao/addax/core/util/HttpClientUtil.java @@ -36,6 +36,8 @@ import java.io.IOException; import java.util.concurrent.ThreadPoolExecutor; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; + public class HttpClientUtil { @@ -144,14 +146,14 @@ public String executeAndGetWithFailedRetry(HttpRequestBase httpRequestBase, String result = executeAndGet(httpRequestBase); if (result != null && result.startsWith("{\"result\":-1")) { throw AddaxException.asAddaxException( - FrameworkErrorCode.CALL_REMOTE_FAILED, "The return code is -1, try again."); + RUNTIME_ERROR, "The return code is -1, try again."); } return result; }, retryTimes, retryInterval, true, HTTP_TIMEOUT_MILLISECONDS + 1000, asyncExecutor); } catch (Exception e) { - throw AddaxException.asAddaxException(FrameworkErrorCode.RUNTIME_ERROR, e); + throw AddaxException.asAddaxException(RUNTIME_ERROR, e); } } } diff --git a/core/src/main/java/com/wgzhao/addax/core/util/TransformerUtil.java b/core/src/main/java/com/wgzhao/addax/core/util/TransformerUtil.java index 227d6236c..4a0379192 100644 --- a/core/src/main/java/com/wgzhao/addax/core/util/TransformerUtil.java +++ b/core/src/main/java/com/wgzhao/addax/core/util/TransformerUtil.java @@ -21,7 +21,6 @@ import com.wgzhao.addax.common.exception.AddaxException; import com.wgzhao.addax.common.util.Configuration; -import com.wgzhao.addax.core.transport.transformer.TransformerErrorCode; import com.wgzhao.addax.core.transport.transformer.TransformerExecution; import com.wgzhao.addax.core.transport.transformer.TransformerExecutionParas; import com.wgzhao.addax.core.transport.transformer.TransformerInfo; @@ -38,6 +37,10 @@ import java.util.ArrayList; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.IO_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; + /** * no comments. * Created by liqiang on 16/3/9. @@ -61,12 +64,12 @@ public static List buildTransformerInfo(Configuration task for (Configuration configuration : tfConfigs) { String functionName = configuration.getString("name"); if (StringUtils.isEmpty(functionName)) { - throw AddaxException.asAddaxException(TransformerErrorCode.TRANSFORMER_CONFIGURATION_ERROR, + throw AddaxException.asAddaxException(CONFIG_ERROR, "config=" + configuration.toJSON()); } if ("dx_groovy".equals(functionName) && functionNames.contains("dx_groovy")) { - throw AddaxException.asAddaxException(TransformerErrorCode.TRANSFORMER_CONFIGURATION_ERROR, + throw AddaxException.asAddaxException(CONFIG_ERROR, "dx_groovy can be invoke once only."); } functionNames.add(functionName); @@ -84,7 +87,7 @@ public static List buildTransformerInfo(Configuration task String functionName = configuration.getString("name"); TransformerInfo transformerInfo = TransformerRegistry.getTransformer(functionName); if (transformerInfo == null) { - throw AddaxException.asAddaxException(TransformerErrorCode.TRANSFORMER_NOTFOUND_ERROR, + throw AddaxException.asAddaxException(REQUIRED_VALUE, "name=" + functionName); } @@ -99,7 +102,7 @@ public static List buildTransformerInfo(Configuration task Integer columnIndex = configuration.getInt(CoreConstant.TRANSFORMER_PARAMETER_COLUMN_INDEX); if (columnIndex == null) { - throw AddaxException.asAddaxException(TransformerErrorCode.TRANSFORMER_ILLEGAL_PARAMETER, + throw AddaxException.asAddaxException(REQUIRED_VALUE, "columnIndex must be set by UDF: name=" + functionName); } @@ -113,7 +116,7 @@ public static List buildTransformerInfo(Configuration task String code = configuration.getString(CoreConstant.TRANSFORMER_PARAMETER_CODE); String codeFile = configuration.getString(CoreConstant.TRANSFORMER_PARAMETER_CODE_FILE); if (StringUtils.isAllEmpty(code, codeFile)) { - throw AddaxException.asAddaxException(TransformerErrorCode.TRANSFORMER_ILLEGAL_PARAMETER, + throw AddaxException.asAddaxException(REQUIRED_VALUE, "groovy code or codeFile must be set by UDF: name=" + functionName); } // code and codeFile both setup, prefers to code , ignore codeFile @@ -125,13 +128,13 @@ public static List buildTransformerInfo(Configuration task // the codeFile default relative path is the same of addax.home properties File file = new File(codeFile); if (! file.exists() || ! file.isFile()) { - throw AddaxException.asAddaxException(TransformerErrorCode.TRANSFORMER_CONFIGURATION_ERROR, + throw AddaxException.asAddaxException(CONFIG_ERROR, "the codeFile [" + codeFile + "]does not exists or is unreadable!"); } try { code = FileUtils.readFileToString(file, Charset.defaultCharset()); } catch (IOException e) { - throw AddaxException.asAddaxException(TransformerErrorCode.TRANSFORMER_RUN_EXCEPTION, + throw AddaxException.asAddaxException(IO_ERROR, "read codeFile [" + codeFile + "] failure:", e); } } diff --git a/core/src/main/java/com/wgzhao/addax/core/util/container/JarLoader.java b/core/src/main/java/com/wgzhao/addax/core/util/container/JarLoader.java index 9e89567df..6a00e358f 100644 --- a/core/src/main/java/com/wgzhao/addax/core/util/container/JarLoader.java +++ b/core/src/main/java/com/wgzhao/addax/core/util/container/JarLoader.java @@ -20,7 +20,6 @@ package com.wgzhao.addax.core.util.container; import com.wgzhao.addax.common.exception.AddaxException; -import com.wgzhao.addax.core.util.FrameworkErrorCode; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.Validate; @@ -32,6 +31,8 @@ import java.util.List; import java.util.Objects; +import static com.wgzhao.addax.common.spi.ErrorCode.PLUGIN_INIT_ERROR; + /** * 提供Jar隔离的加载机制，会把传入的路径、及其子路径、以及路径中的jar文件加入到class path。 */ @@ -109,7 +110,7 @@ private static List doGetURLs(final String path) jarURLs.add(allJar.toURI().toURL()); } catch (Exception e) { - throw AddaxException.asAddaxException(FrameworkErrorCode.PLUGIN_INIT_ERROR, + throw AddaxException.asAddaxException(PLUGIN_INIT_ERROR, "Exception occurred when load the jar package.", e); } } diff --git a/core/src/main/java/com/wgzhao/addax/core/util/container/LoadUtil.java b/core/src/main/java/com/wgzhao/addax/core/util/container/LoadUtil.java index aec2dc058..e9bb48065 100644 --- a/core/src/main/java/com/wgzhao/addax/core/util/container/LoadUtil.java +++ b/core/src/main/java/com/wgzhao/addax/core/util/container/LoadUtil.java @@ -28,12 +28,13 @@ import com.wgzhao.addax.core.taskgroup.runner.AbstractRunner; import com.wgzhao.addax.core.taskgroup.runner.ReaderRunner; import com.wgzhao.addax.core.taskgroup.runner.WriterRunner; -import com.wgzhao.addax.core.util.FrameworkErrorCode; import org.apache.commons.lang3.StringUtils; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; + +import static com.wgzhao.addax.common.spi.ErrorCode.PLUGIN_INSTALL_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; /** * Created by jingxing on 14-8-24. @@ -77,7 +78,7 @@ private static Configuration getPluginConf(PluginType pluginType, String pluginN if (null == pluginConf) { throw AddaxException.asAddaxException( - FrameworkErrorCode.PLUGIN_INSTALL_ERROR, + PLUGIN_INSTALL_ERROR, String.format("Can not find the configure of plugin [%s].", pluginName)); } @@ -98,7 +99,7 @@ public static AbstractJobPlugin loadJobPlugin(PluginType pluginType, String plug } catch (Exception e) { throw AddaxException.asAddaxException( - FrameworkErrorCode.RUNTIME_ERROR, + RUNTIME_ERROR, String.format("Exception occurred when load job plugin [%s].", pluginName), e); } } @@ -116,7 +117,7 @@ public static AbstractTaskPlugin loadTaskPlugin(PluginType pluginType, String pl return taskPlugin; } catch (Exception e) { - throw AddaxException.asAddaxException(FrameworkErrorCode.RUNTIME_ERROR, + throw AddaxException.asAddaxException(RUNTIME_ERROR, String.format("Can not find the configure of task plugin [%s].", pluginName), e); } } @@ -134,7 +135,7 @@ public static AbstractRunner loadPluginRunner(PluginType pluginType, String plug case WRITER: return new WriterRunner(taskPlugin); default: - throw AddaxException.asAddaxException(FrameworkErrorCode.RUNTIME_ERROR, + throw AddaxException.asAddaxException(RUNTIME_ERROR, String.format("The plugin type must be reader or writer, [%s] is unsupported.", pluginName)); } } @@ -154,7 +155,7 @@ private static synchronized Class loadPluginClass( + pluginRunType.value()); } catch (Exception e) { - throw AddaxException.asAddaxException(FrameworkErrorCode.RUNTIME_ERROR, e); + throw AddaxException.asAddaxException(RUNTIME_ERROR, e); } } @@ -165,7 +166,7 @@ public static synchronized JarLoader getJarLoader(PluginType pluginType, String if (null == jarLoader) { String pluginPath = pluginConf.getString("path"); if (StringUtils.isBlank(pluginPath)) { - throw AddaxException.asAddaxException(FrameworkErrorCode.RUNTIME_ERROR, + throw AddaxException.asAddaxException(RUNTIME_ERROR, String.format("Illegal path of plugin [%s] for [%s].", pluginName, pluginType)); } jarLoader = new JarLoader(new String[] {pluginPath}); diff --git a/lib/addax-rdbms/src/main/java/com/wgzhao/addax/rdbms/reader/util/OriginalConfPretreatmentUtil.java b/lib/addax-rdbms/src/main/java/com/wgzhao/addax/rdbms/reader/util/OriginalConfPretreatmentUtil.java index dbc6e149a..23154ffa7 100644 --- a/lib/addax-rdbms/src/main/java/com/wgzhao/addax/rdbms/reader/util/OriginalConfPretreatmentUtil.java +++ b/lib/addax-rdbms/src/main/java/com/wgzhao/addax/rdbms/reader/util/OriginalConfPretreatmentUtil.java @@ -28,7 +28,6 @@ import com.wgzhao.addax.common.util.EncryptUtil; import com.wgzhao.addax.common.util.ListUtil; import com.wgzhao.addax.rdbms.util.DBUtil; -import com.wgzhao.addax.rdbms.util.DBUtilErrorCode; import com.wgzhao.addax.rdbms.util.DataBaseType; import com.wgzhao.addax.rdbms.util.TableExpandUtil; import org.apache.commons.lang3.StringUtils; @@ -38,6 +37,10 @@ import java.util.ArrayList; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.EXECUTE_FAIL; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; + public final class OriginalConfPretreatmentUtil { private static final Logger LOG = LoggerFactory.getLogger(OriginalConfPretreatmentUtil.class); @@ -49,7 +52,7 @@ private OriginalConfPretreatmentUtil() {} public static void doPretreatment(Configuration originalConfig) { // 检查 username 配置（必填） - originalConfig.getNecessaryValue(Key.USERNAME, DBUtilErrorCode.REQUIRED_VALUE); + originalConfig.getNecessaryValue(Key.USERNAME, REQUIRED_VALUE); /* * 有些数据库没有密码，因此密码不再作为必选项 */ @@ -117,12 +120,12 @@ private static void dealJdbcAndTable(Configuration originalConfig) LOG.warn("use specified driver class: {}", driverClass); dataBaseType.setDriverClassName(driverClass); } - connConf.getNecessaryValue(Key.JDBC_URL, DBUtilErrorCode.REQUIRED_VALUE); + connConf.getNecessaryValue(Key.JDBC_URL, REQUIRED_VALUE); String jdbcUrl = connConf.getString(Key.JDBC_URL); if (StringUtils.isBlank(jdbcUrl)) { - throw AddaxException.asAddaxException(DBUtilErrorCode.REQUIRED_VALUE, "The parameter [connection.jdbcUrl] is not set."); + throw AddaxException.asAddaxException(REQUIRED_VALUE, "The parameter [connection.jdbcUrl] is not set."); } if (isPreCheck) { @@ -146,7 +149,7 @@ private static void dealJdbcAndTable(Configuration originalConfig) if (expandedTables.isEmpty()) { throw AddaxException.asAddaxException( - DBUtilErrorCode.ILLEGAL_VALUE, String.format("Failed to obtain the table [%s].", StringUtils.join(tables, ","))); + EXECUTE_FAIL, String.format("Failed to obtain the table [%s].", StringUtils.join(tables, ","))); } tableNum += expandedTables.size(); @@ -166,7 +169,7 @@ private static void dealColumnConf(Configuration originalConfig) if (isTableMode) { if (null == userConfiguredColumns || userConfiguredColumns.isEmpty()) { - throw AddaxException.asAddaxException(DBUtilErrorCode.REQUIRED_VALUE, "The item column is required."); + throw AddaxException.asAddaxException(REQUIRED_VALUE, "The item column is required."); } else { String splitPk = originalConfig.getString(Key.SPLIT_PK, null); @@ -195,7 +198,7 @@ private static void dealColumnConf(Configuration originalConfig) for (String column : userConfiguredColumns) { if ("*".equals(column)) { - throw AddaxException.asAddaxException(DBUtilErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(CONFIG_ERROR, "The item column your configured is invalid, because it includes multiply asterisk('*')."); } @@ -206,7 +209,7 @@ private static void dealColumnConf(Configuration originalConfig) originalConfig.set(Key.COLUMN_LIST, quotedColumns); originalConfig.set(Key.COLUMN, StringUtils.join(quotedColumns, ",")); if (StringUtils.isNotBlank(splitPk) && !allColumns.contains(splitPk.toLowerCase())) { - throw AddaxException.asAddaxException(DBUtilErrorCode.ILLEGAL_SPLIT_PK, + throw AddaxException.asAddaxException(CONFIG_ERROR, String.format("The table [%s] has not the primary key [%s].", tableName, splitPk)); } } @@ -255,11 +258,11 @@ private static boolean recognizeTableOrQuerySqlMode(Configuration originalConfig if (!isTableMode && !isQuerySqlMode) { // table 和 querySql 二者均未配置 throw AddaxException.asAddaxException( - DBUtilErrorCode.TABLE_QUERY_SQL_MISSING, "You must configure either table or querySql."); + REQUIRED_VALUE, "You must configure either table or querySql."); } else if (isTableMode && isQuerySqlMode) { // table 和 querySql 二者均配置 - throw AddaxException.asAddaxException(DBUtilErrorCode.TABLE_QUERY_SQL_MIXED, + throw AddaxException.asAddaxException(CONFIG_ERROR, "You ca not configure both table and querySql at the same time."); } diff --git a/lib/addax-rdbms/src/main/java/com/wgzhao/addax/rdbms/reader/util/SingleTableSplitUtil.java b/lib/addax-rdbms/src/main/java/com/wgzhao/addax/rdbms/reader/util/SingleTableSplitUtil.java index 59d4a4e18..5b865a6be 100644 --- a/lib/addax-rdbms/src/main/java/com/wgzhao/addax/rdbms/reader/util/SingleTableSplitUtil.java +++ b/lib/addax-rdbms/src/main/java/com/wgzhao/addax/rdbms/reader/util/SingleTableSplitUtil.java @@ -27,7 +27,6 @@ import com.wgzhao.addax.common.exception.AddaxException; import com.wgzhao.addax.common.util.Configuration; import com.wgzhao.addax.rdbms.util.DBUtil; -import com.wgzhao.addax.rdbms.util.DBUtilErrorCode; import com.wgzhao.addax.rdbms.util.DataBaseType; import com.wgzhao.addax.rdbms.util.RdbmsException; import com.wgzhao.addax.rdbms.util.RdbmsRangeSplitWrap; @@ -47,6 +46,9 @@ import java.util.ArrayList; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.NOT_SUPPORT_TYPE; + public class SingleTableSplitUtil { private static final Logger LOG = LoggerFactory.getLogger(SingleTableSplitUtil.class); @@ -73,7 +75,7 @@ public static List splitSingleTable(Configuration configuration, else { Pair minMaxPK = getPkRange(configuration); if (null == minMaxPK) { - throw AddaxException.asAddaxException(DBUtilErrorCode.ILLEGAL_SPLIT_PK, + throw AddaxException.asAddaxException(CONFIG_ERROR, "Primary key-based table splitting failed. The key type ONLY supports integer and string."); } @@ -96,7 +98,7 @@ else if (isLongType) { new BigInteger(minMaxPK.getRight().toString()), adviceNum, splitPkName); } else { - throw AddaxException.asAddaxException(DBUtilErrorCode.ILLEGAL_SPLIT_PK, + throw AddaxException.asAddaxException(NOT_SUPPORT_TYPE, "the splitPk[" + splitPkName + "] type is unsupported, it only support int and string"); } } @@ -177,7 +179,7 @@ public static void preCheckSplitPk(Connection conn, String pkRangeSQL, int fetch { Pair minMaxPK = checkSplitPk(conn, pkRangeSQL, fetchSize, table, username, null); if (null == minMaxPK) { - throw AddaxException.asAddaxException(DBUtilErrorCode.ILLEGAL_SPLIT_PK, + throw AddaxException.asAddaxException(CONFIG_ERROR, "The split key should be single column, and the type is either integer or string."); } } @@ -228,23 +230,23 @@ else if (isLongType(rsMetaData.getColumnType(1))) { // check: string shouldn't contain '.', for oracle String minMax = rs.getString(1) + rs.getString(2); if (StringUtils.contains(minMax, '.')) { - throw AddaxException.asAddaxException(DBUtilErrorCode.ILLEGAL_SPLIT_PK, errorMsg); + throw AddaxException.asAddaxException(CONFIG_ERROR, errorMsg); } } } else { - throw AddaxException.asAddaxException(DBUtilErrorCode.ILLEGAL_SPLIT_PK, errorMsg); + throw AddaxException.asAddaxException(CONFIG_ERROR, errorMsg); } } else { - throw AddaxException.asAddaxException(DBUtilErrorCode.ILLEGAL_SPLIT_PK, errorMsg); + throw AddaxException.asAddaxException(CONFIG_ERROR, errorMsg); } } catch (AddaxException e) { throw e; } catch (Exception e) { - throw AddaxException.asAddaxException(DBUtilErrorCode.ILLEGAL_SPLIT_PK, "Failed to split the table.", e); + throw AddaxException.asAddaxException(CONFIG_ERROR, "Failed to split the table.", e); } finally { DBUtil.closeDBResources(rs, null, null); @@ -267,7 +269,7 @@ private static boolean isPKTypeValid(ResultSetMetaData rsMetaData) } } catch (Exception e) { - throw AddaxException.asAddaxException(DBUtilErrorCode.ILLEGAL_SPLIT_PK, + throw AddaxException.asAddaxException(CONFIG_ERROR, "Failed to obtain the type of split key."); } return ret; @@ -361,7 +363,7 @@ else if (adviceNum == 1) { throw e; } catch (Exception e) { - throw AddaxException.asAddaxException(DBUtilErrorCode.ILLEGAL_SPLIT_PK, "Failed to split table by split key.", e); + throw AddaxException.asAddaxException(CONFIG_ERROR, "Failed to split table by split key.", e); } finally { DBUtil.closeDBResources(rs, null, null); @@ -395,7 +397,7 @@ else if (isStringType(splitRange.get(0).getRight())) { splitPK, "'", dataBaseType)); } else { - throw AddaxException.asAddaxException(DBUtilErrorCode.ILLEGAL_SPLIT_PK, + throw AddaxException.asAddaxException(CONFIG_ERROR, "the data type of split key is unsupported. it ONLY supports integer and string."); } } diff --git a/lib/addax-rdbms/src/main/java/com/wgzhao/addax/rdbms/util/DBUtil.java b/lib/addax-rdbms/src/main/java/com/wgzhao/addax/rdbms/util/DBUtil.java index a7bb72547..0b46954b6 100644 --- a/lib/addax-rdbms/src/main/java/com/wgzhao/addax/rdbms/util/DBUtil.java +++ b/lib/addax-rdbms/src/main/java/com/wgzhao/addax/rdbms/util/DBUtil.java @@ -49,7 +49,13 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -public final class DBUtil { +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.CONNECT_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.EXECUTE_FAIL; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; + +public final class DBUtil +{ private static final Logger LOG = LoggerFactory.getLogger(DBUtil.class); private static final int DEFAULT_SOCKET_TIMEOUT_SEC = 20_000; @@ -58,39 +64,47 @@ public final class DBUtil { .setDaemon(true) .build())); - private DBUtil() { + private DBUtil() + { } - public static void validJdbcUrl(DataBaseType dataBaseType, String jdbcUrl, String username, String password, List preSql) { + public static void validJdbcUrl(DataBaseType dataBaseType, String jdbcUrl, String username, String password, List preSql) + { try { RetryUtil.executeWithRetry(() -> { if (null == preSql || preSql.isEmpty()) { testConnWithoutRetry(dataBaseType, jdbcUrl, username, password); - } else { + } + else { testConnWithoutRetry(dataBaseType, jdbcUrl, username, password, preSql); } return null; }, 3, 1000L, true); - } catch (Exception e) { - throw AddaxException.asAddaxException(DBUtilErrorCode.CONN_DB_ERROR, + } + catch (Exception e) { + throw AddaxException.asAddaxException(CONNECT_ERROR, "Failed to connect the database server using " + jdbcUrl, e); } } - public static void validJdbcUrlWithoutRetry(DataBaseType dataBaseType, String jdbcUrl, String username, String password, List preSql) { + public static void validJdbcUrlWithoutRetry(DataBaseType dataBaseType, String jdbcUrl, String username, String password, List preSql) + { if (null != preSql && !preSql.isEmpty()) { testConnWithoutRetry(dataBaseType, jdbcUrl, username, password, preSql); - } else { + } + else { try { testConnWithoutRetry(dataBaseType, jdbcUrl, username, password); - } catch (Exception e) { + } + catch (Exception e) { throw AddaxException.asAddaxException( - DBUtilErrorCode.CONN_DB_ERROR, "Failed to connect the server using jdbcUrl " + jdbcUrl, e); + CONNECT_ERROR, "Failed to connect the server using jdbcUrl " + jdbcUrl, e); } } } - public static boolean checkInsertPrivilege(DataBaseType dataBaseType, String jdbcURL, String userName, String password, List tableList) { + public static boolean checkInsertPrivilege(DataBaseType dataBaseType, String jdbcURL, String userName, String password, List tableList) + { Connection connection = connect(dataBaseType, jdbcURL, userName, password); String insertTemplate = "INSERT INTO %s (SELECT * FROM %s WHERE 1 = 2)"; @@ -101,7 +115,8 @@ public static boolean checkInsertPrivilege(DataBaseType dataBaseType, String jdb try { insertStmt = connection.createStatement(); insertStmt.execute(checkInsertPrivilegeSql); - } catch (Exception e) { + } + catch (Exception e) { hasInsertPrivilege = false; LOG.warn("Failed to insert into table [{}] with user [{}]: {}.", userName, tableName, e.getMessage()); } @@ -111,7 +126,8 @@ public static boolean checkInsertPrivilege(DataBaseType dataBaseType, String jdb return hasInsertPrivilege; } - public static boolean checkDeletePrivilege(DataBaseType dataBaseType, String jdbcURL, String userName, String password, List tableList) { + public static boolean checkDeletePrivilege(DataBaseType dataBaseType, String jdbcURL, String userName, String password, List tableList) + { Connection connection = connect(dataBaseType, jdbcURL, userName, password); String deleteTemplate = "DELETE FROM %s WHERE 1 = 2"; @@ -122,7 +138,8 @@ public static boolean checkDeletePrivilege(DataBaseType dataBaseType, String jdb try { deleteStmt = connection.createStatement(); deleteStmt.execute(checkDeletePrivilegeSQL); - } catch (Exception e) { + } + catch (Exception e) { hasInsertPrivilege = false; LOG.warn("Failed to delete from table [{}] with user [{}]: {}.", userName, tableName, e.getMessage()); } @@ -132,7 +149,8 @@ public static boolean checkDeletePrivilege(DataBaseType dataBaseType, String jdb return hasInsertPrivilege; } - public static boolean needCheckDeletePrivilege(Configuration originalConfig) { + public static boolean needCheckDeletePrivilege(Configuration originalConfig) + { List allSqls = new ArrayList<>(); List preSQLs = originalConfig.getList(Key.PRE_SQL, String.class); List postSQLs = originalConfig.getList(Key.POST_SQL, String.class); @@ -157,23 +175,26 @@ public static boolean needCheckDeletePrivilege(Configuration originalConfig) { *

* * @param dataBaseType database type. - * @param jdbcUrl java jdbc url. - * @param username User for login. - * @param password Password to use when connecting to server. + * @param jdbcUrl java jdbc url. + * @param username User for login. + * @param password Password to use when connecting to server. * @return Connection class {@link Connection} */ - public static Connection getConnection(DataBaseType dataBaseType, String jdbcUrl, String username, String password) { + public static Connection getConnection(DataBaseType dataBaseType, String jdbcUrl, String username, String password) + { return getConnection(dataBaseType, jdbcUrl, username, password, DEFAULT_SOCKET_TIMEOUT_SEC); } - public static Connection getConnection(DataBaseType dataBaseType, String jdbcUrl, String username, String password, int socketTimeout) { + public static Connection getConnection(DataBaseType dataBaseType, String jdbcUrl, String username, String password, int socketTimeout) + { try { return RetryUtil.executeWithRetry(() -> DBUtil.connect(dataBaseType, jdbcUrl, username, password, socketTimeout), 3, 1000L, true); - } catch (Exception e) { - throw AddaxException.asAddaxException(DBUtilErrorCode.CONN_DB_ERROR, + } + catch (Exception e) { + throw AddaxException.asAddaxException(CONNECT_ERROR, String.format("Failed to connect the database with [%s].", jdbcUrl), e); } } @@ -185,24 +206,28 @@ public static Connection getConnection(DataBaseType dataBaseType, String jdbcUrl *

* * @param dataBaseType The database's type - * @param jdbcUrl jdbc url - * @param username User for login - * @param password Password to use when connecting to server + * @param jdbcUrl jdbc url + * @param username User for login + * @param password Password to use when connecting to server * @return Connection class {@link Connection} */ - public static Connection getConnectionWithoutRetry(DataBaseType dataBaseType, String jdbcUrl, String username, String password) { + public static Connection getConnectionWithoutRetry(DataBaseType dataBaseType, String jdbcUrl, String username, String password) + { return getConnectionWithoutRetry(dataBaseType, jdbcUrl, username, password, DEFAULT_SOCKET_TIMEOUT_SEC); } - public static Connection getConnectionWithoutRetry(DataBaseType dataBaseType, String jdbcUrl, String username, String password, int socketTimeout) { + public static Connection getConnectionWithoutRetry(DataBaseType dataBaseType, String jdbcUrl, String username, String password, int socketTimeout) + { return DBUtil.connect(dataBaseType, jdbcUrl, username, password, socketTimeout); } - private static synchronized Connection connect(DataBaseType dataBaseType, String url, String user, String pass) { + private static synchronized Connection connect(DataBaseType dataBaseType, String url, String user, String pass) + { return connect(dataBaseType, url, user, pass, DEFAULT_SOCKET_TIMEOUT_SEC); } - private static synchronized Connection connect(DataBaseType dataBaseType, String url, String user, String pass, int socketTimeout) { + private static synchronized Connection connect(DataBaseType dataBaseType, String url, String user, String pass, int socketTimeout) + { try (BasicDataSource bds = new BasicDataSource()) { bds.setUrl(url); @@ -219,7 +244,8 @@ private static synchronized Connection connect(DataBaseType dataBaseType, String url = url.replace("inceptor2", "hive2"); bds.setUrl(url); bds.setDriverClassName("org.apache.hive.jdbc.HiveDriver"); - } else { + } + else { LOG.debug("Connecting to database with driver {}", dataBaseType.getDriverClassName()); bds.setDriverClassName(dataBaseType.getDriverClassName()); } @@ -227,7 +253,8 @@ private static synchronized Connection connect(DataBaseType dataBaseType, String bds.setMaxIdle(5); bds.setMaxOpenPreparedStatements(200); return bds.getConnection(); - } catch (Exception e) { + } + catch (Exception e) { LOG.error("An exception occurred while attempting to connect to the database using jdbcUrl '{}': {}'", url, e.toString()); throw RdbmsException.asConnException(e); } @@ -236,14 +263,15 @@ private static synchronized Connection connect(DataBaseType dataBaseType, String /** * a wrapped method to execute select-like sql statement . * - * @param conn Database connection . - * @param sql sql statement to be executed + * @param conn Database connection . + * @param sql sql statement to be executed * @param fetchSize fetch size * @return a {@link ResultSet} * @throws SQLException if occurs SQLException. */ public static ResultSet query(Connection conn, String sql, int fetchSize) - throws SQLException { + throws SQLException + { // 默认3600 s 的query Timeout return query(conn, sql, fetchSize, DEFAULT_SOCKET_TIMEOUT_SEC); } @@ -251,26 +279,29 @@ public static ResultSet query(Connection conn, String sql, int fetchSize) /** * a wrapped method to execute select-like sql statement . * - * @param conn Database connection . - * @param sql sql statement to be executed - * @param fetchSize fetch size each batch + * @param conn Database connection . + * @param sql sql statement to be executed + * @param fetchSize fetch size each batch * @param queryTimeout unit:second * @return A {@link ResultSet} * @throws SQLException if failed to execute sql statement */ public static ResultSet query(Connection conn, String sql, int fetchSize, int queryTimeout) - throws SQLException { + throws SQLException + { Statement stmt; try { // make sure autocommit is off conn.setAutoCommit(false); - } catch (SQLFeatureNotSupportedException ignore) { + } + catch (SQLFeatureNotSupportedException ignore) { LOG.warn("The current database does not support AUTO_COMMIT property"); } try { stmt = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); //NOSONAR - } catch (SQLException ignore) { + } + catch (SQLException ignore) { // some database does not support TYPE_FORWARD_ONLY/CONCUR_READ_ONLY LOG.warn("The current database does not support TYPE_FORWARD_ONLY/CONCUR_READ_ONLY"); stmt = conn.createStatement(); //NOSONAR @@ -280,11 +311,13 @@ public static ResultSet query(Connection conn, String sql, int fetchSize, int qu return stmt.executeQuery(sql); } - public static void closeDBResources(ResultSet rs, Statement stmt, Connection conn) { + public static void closeDBResources(ResultSet rs, Statement stmt, Connection conn) + { if (null != rs) { try { rs.close(); - } catch (SQLException ignored) { + } + catch (SQLException ignored) { // } } @@ -292,7 +325,8 @@ public static void closeDBResources(ResultSet rs, Statement stmt, Connection con if (null != stmt) { try { stmt.close(); - } catch (SQLException ignored) { + } + catch (SQLException ignored) { // } } @@ -300,22 +334,26 @@ public static void closeDBResources(ResultSet rs, Statement stmt, Connection con if (null != conn) { try { conn.close(); - } catch (SQLException ignored) { + } + catch (SQLException ignored) { // } } } - public static void closeDBResources(Statement stmt, Connection conn) { + public static void closeDBResources(Statement stmt, Connection conn) + { closeDBResources(null, stmt, conn); } - public static List getTableColumns(DataBaseType dataBaseType, String jdbcUrl, String user, String pass, String tableName) { + public static List getTableColumns(DataBaseType dataBaseType, String jdbcUrl, String user, String pass, String tableName) + { Connection conn = getConnection(dataBaseType, jdbcUrl, user, pass); return getTableColumnsByConn(conn, tableName); } - public static List getTableColumnsByConn(Connection conn, String tableName) { + public static List getTableColumnsByConn(Connection conn, String tableName) + { List columns = new ArrayList<>(); List> rsMetaData = getColumnMetaData(conn, tableName, "*"); @@ -328,12 +366,13 @@ public static List getTableColumnsByConn(Connection conn, String tableNa /** * get column description * - * @param conn database connection + * @param conn database connection * @param tableName The table name - * @param column table column + * @param column table column * @return {@link List} */ - public static List> getColumnMetaData(Connection conn, String tableName, String column) { + public static List> getColumnMetaData(Connection conn, String tableName, String column) + { List> result = new ArrayList<>(); // skip index 0, compliant with jdbc resultSet and resultMetaData result.add(null); @@ -343,7 +382,8 @@ public static List> getColumnMetaData(Connection conn, Strin if (DataBaseType.TDengine.getDriverClassName().equals(conn.getMetaData().getDriverName())) { // TDengine does not support 1=2 clause queryColumnSql = "SELECT " + column + " FROM " + tableName + " LIMIT 0"; - } else { + } + else { queryColumnSql = "SELECT " + column + " FROM " + tableName + " WHERE 1 = 2"; } ResultSetMetaData metaData = statement.executeQuery(queryColumnSql).getMetaData(); @@ -359,51 +399,60 @@ public static List> getColumnMetaData(Connection conn, Strin } statement.close(); return result; - } catch (SQLException e) { - throw AddaxException.asAddaxException(DBUtilErrorCode.GET_COLUMN_INFO_FAILED, + } + catch (SQLException e) { + throw AddaxException.asAddaxException(EXECUTE_FAIL, String.format("Failed to obtain the fields of table [%s].", tableName), e); } } - public static void testConnWithoutRetry(DataBaseType dataBaseType, String url, String user, String pass) { + public static void testConnWithoutRetry(DataBaseType dataBaseType, String url, String user, String pass) + { Connection ignored = null; try { ignored = connect(dataBaseType, url, user, pass); - } catch (Exception e) { - throw AddaxException.asAddaxException(DBUtilErrorCode.CONN_DB_ERROR, + } + catch (Exception e) { + throw AddaxException.asAddaxException(CONNECT_ERROR, String.format("Failed to connect the database using '%s': %s.", url, e.getMessage()), e); - } finally { + } + finally { if (null != ignored) { try { ignored.close(); - } catch (SQLException e) { + } + catch (SQLException e) { LOG.warn("Failed to close the connection."); } } } } - public static void testConnWithoutRetry(DataBaseType dataBaseType, String url, String user, String pass, List preSql) { + public static void testConnWithoutRetry(DataBaseType dataBaseType, String url, String user, String pass, List preSql) + { try (Connection connection = connect(dataBaseType, url, user, pass)) { for (String pre : preSql) { if (!doPreCheck(connection, pre)) { LOG.warn("Failed to doPreCheck."); } } - } catch (Exception e) { + } + catch (Exception e) { LOG.warn("Failed to connect the database using '{}': {}.", url, e.getMessage()); } } public static ResultSet query(Connection conn, String sql) - throws SQLException { + throws SQLException + { try (Statement stmt = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY)) { stmt.setQueryTimeout(DEFAULT_SOCKET_TIMEOUT_SEC); return stmt.executeQuery(sql); } } - private static boolean doPreCheck(Connection conn, String pre) { + private static boolean doPreCheck(Connection conn, String pre) + { try (ResultSet rs = query(conn, pre)) { int checkResult = -1; if (DBUtil.asyncResultSetNext(rs)) { @@ -417,14 +466,16 @@ private static boolean doPreCheck(Connection conn, String pre) { return true; } LOG.warn("Failed to pre-check with [{}]. It should return 0.", pre); - } catch (Exception e) { + } + catch (Exception e) { LOG.warn("Failed to pre-check with [{}], errorMessage: [{}].", pre, e.getMessage()); } return false; } // warn:until now, only oracle need to handle session config. - public static void dealWithSessionConfig(Connection conn, Configuration config, DataBaseType databaseType, String message) { + public static void dealWithSessionConfig(Connection conn, Configuration config, DataBaseType databaseType, String message) + { List sessionConfig; switch (databaseType) { case Oracle: @@ -438,7 +489,8 @@ public static void dealWithSessionConfig(Connection conn, Configuration config, } } - private static void doDealWithSessionConfig(Connection conn, List sessions, String message) { + private static void doDealWithSessionConfig(Connection conn, List sessions, String message) + { if (null == sessions || sessions.isEmpty()) { return; } @@ -446,8 +498,9 @@ private static void doDealWithSessionConfig(Connection conn, List sessio Statement stmt; try { stmt = conn.createStatement(); - } catch (SQLException e) { - throw AddaxException.asAddaxException(DBUtilErrorCode.SET_SESSION_ERROR, + } + catch (SQLException e) { + throw AddaxException.asAddaxException(CONFIG_ERROR, String.format("Failed to set session with [%s]", message), e); } @@ -455,15 +508,17 @@ private static void doDealWithSessionConfig(Connection conn, List sessio LOG.info("Executing SQL:[{}]", sessionSql); try { stmt.execute(sessionSql); - } catch (SQLException e) { - throw AddaxException.asAddaxException(DBUtilErrorCode.SET_SESSION_ERROR, + } + catch (SQLException e) { + throw AddaxException.asAddaxException(CONFIG_ERROR, String.format("Failed to set session with [%s].", message), e); } } DBUtil.closeDBResources(stmt, null); } - public static void sqlValid(String sql, DataBaseType dataBaseType) { + public static void sqlValid(String sql, DataBaseType dataBaseType) + { SQLStatementParser statementParser = SQLParserUtils.createSQLStatementParser(sql, dataBaseType.getTypeName()); statementParser.parseStatementList(); } @@ -474,23 +529,27 @@ public static void sqlValid(String sql, DataBaseType dataBaseType) { * @param resultSet result set * @return boolean */ - public static boolean asyncResultSetNext(ResultSet resultSet) { + public static boolean asyncResultSetNext(ResultSet resultSet) + { return asyncResultSetNext(resultSet, 3600); } - public static boolean asyncResultSetNext(ResultSet resultSet, int timeout) { + public static boolean asyncResultSetNext(ResultSet resultSet, int timeout) + { Future future = rsExecutors.get().submit(resultSet::next); try { return future.get(timeout, TimeUnit.SECONDS); - } catch (Exception e) { - throw AddaxException.asAddaxException(DBUtilErrorCode.RS_ASYNC_ERROR, "Asynchronous retrieval of ResultSet failed.", e); + } + catch (Exception e) { + throw AddaxException.asAddaxException(RUNTIME_ERROR, "Asynchronous retrieval of ResultSet failed.", e); } } - public static void loadDriverClass(String pluginType, String pluginName) { + public static void loadDriverClass(String pluginType, String pluginName) + { try { String pluginJsonPath = StringUtils.join( - new String[]{ + new String[] { System.getProperty("addax.home"), "plugin", pluginType, @@ -501,8 +560,9 @@ public static void loadDriverClass(String pluginType, String pluginName) { for (String driver : drivers) { Class.forName(driver); } - } catch (ClassNotFoundException e) { - throw AddaxException.asAddaxException(DBUtilErrorCode.CONF_ERROR, + } + catch (ClassNotFoundException e) { + throw AddaxException.asAddaxException(CONFIG_ERROR, "Error loading database driver. Please confirm that the libs directory has the driver jar package " + "and the drivers configuration in plugin.json is correct.", e); } diff --git a/lib/addax-rdbms/src/main/java/com/wgzhao/addax/rdbms/util/DBUtilErrorCode.java b/lib/addax-rdbms/src/main/java/com/wgzhao/addax/rdbms/util/DBUtilErrorCode.java deleted file mode 100644 index 966441927..000000000 --- a/lib/addax-rdbms/src/main/java/com/wgzhao/addax/rdbms/util/DBUtilErrorCode.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, - * * software distributed under the License is distributed on an - * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * * KIND, either express or implied. See the License for the - * * specific language governing permissions and limitations - * * under the License. - * - */ - -package com.wgzhao.addax.rdbms.util; - -import com.wgzhao.addax.common.spi.ErrorCode; - -public enum DBUtilErrorCode - implements ErrorCode -{ - JDBC_NULL("db-1001", "JDBC URL为空，请检查配置"), - CONF_ERROR("db-1002", "您的配置错误."), - CONN_DB_ERROR("db-1003", "Failed to connect the database."), - GET_COLUMN_INFO_FAILED("db-2001", "获取表字段相关信息失败."), - UNSUPPORTED_TYPE("db-3001", "不支持的数据库类型. 请注意查看已经支持的数据库类型以及数据库版本."), - COLUMN_SPLIT_ERROR("db-4001", "根据主键进行切分失败."), - SET_SESSION_ERROR("db-5001", "设置 session 失败."), - RS_ASYNC_ERROR("db-5002", "异步获取ResultSet next失败."), - - REQUIRED_VALUE("db-0001", "您缺失了必须填写的参数值."), - ILLEGAL_VALUE("db-0002", "您填写的参数值不合法."), - ILLEGAL_SPLIT_PK("DBUtilErrorCode-04", "您填写的主键列不合法, 仅支持切分主键为一个,并且类型为整数或者字符串类型."), - SPLIT_FAILED_ILLEGAL_SQL("DBUtilErrorCode-15", "尝试切分表时, 执行数据库 Sql 失败. 请检查您的配置 table/splitPk/where 并作出修改."), - SQL_EXECUTE_FAIL("DBUtilErrorCode-06", "执行数据库 Sql 失败, 请检查您的配置的 column/table/where/querySql或者向 DBA 寻求帮助."), - - // only for reader - READ_RECORD_FAIL("DBUtilErrorCode-07", "读取数据库数据失败. 请检查您的配置的 column/table/where/querySql或者向 DBA 寻求帮助."), - TABLE_QUERY_SQL_MIXED("DBUtilErrorCode-08", "您配置凌乱了. 不能同时既配置table又配置querySql"), - TABLE_QUERY_SQL_MISSING("DBUtilErrorCode-09", "您配置错误. table和querySql 应该并且只能配置一个."), - - // only for writer - WRITE_DATA_ERROR("DBUtilErrorCode-05", "往您配置的写入表中写入数据时失败."), - NO_INSERT_PRIVILEGE("DBUtilErrorCode-11", "数据库没有写权限，请联系DBA"), - NO_DELETE_PRIVILEGE("DBUtilErrorCode-16", "数据库没有DELETE权限，请联系DBA"), - ; - - private final String code; - - private final String description; - - DBUtilErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("error code: %s", this.code); - } -} diff --git a/lib/addax-rdbms/src/main/java/com/wgzhao/addax/rdbms/util/RdbmsException.java b/lib/addax-rdbms/src/main/java/com/wgzhao/addax/rdbms/util/RdbmsException.java index e09a40704..f8522ca5e 100644 --- a/lib/addax-rdbms/src/main/java/com/wgzhao/addax/rdbms/util/RdbmsException.java +++ b/lib/addax-rdbms/src/main/java/com/wgzhao/addax/rdbms/util/RdbmsException.java @@ -24,6 +24,10 @@ import com.wgzhao.addax.common.exception.AddaxException; import com.wgzhao.addax.common.spi.ErrorCode; +import static com.wgzhao.addax.common.spi.ErrorCode.CONNECT_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.EXECUTE_FAIL; +import static com.wgzhao.addax.common.spi.ErrorCode.PERMISSION_ERROR; + /** * Created by judy.lt on 2015/6/5. */ @@ -37,42 +41,41 @@ public RdbmsException(ErrorCode errorCode, String message) public static AddaxException asConnException(Exception e) { - return asAddaxException(DBUtilErrorCode.CONN_DB_ERROR, e.getMessage()); + return asAddaxException(CONNECT_ERROR, e.getMessage()); } public static AddaxException asQueryException(Exception e, String querySql) { - return asAddaxException(DBUtilErrorCode.SQL_EXECUTE_FAIL, e.getMessage()); + return asAddaxException(EXECUTE_FAIL, e.getMessage()); } public static AddaxException asSqlParserException(Exception e, String querySql) { - throw asAddaxException(DBUtilErrorCode.READ_RECORD_FAIL, e.getMessage()); + throw asAddaxException(EXECUTE_FAIL, e.getMessage()); } public static AddaxException asPreSQLParserException(Exception e, String querySql) { - throw asAddaxException(DBUtilErrorCode.READ_RECORD_FAIL, e.getMessage()); + throw asAddaxException(EXECUTE_FAIL, e.getMessage()); } public static AddaxException asPostSQLParserException(Exception e, String querySql) { - throw asAddaxException(DBUtilErrorCode.READ_RECORD_FAIL, e.getMessage()); + throw asAddaxException(EXECUTE_FAIL, e.getMessage()); } public static AddaxException asInsertPriException(String userName, String jdbcUrl) { - throw asAddaxException(DBUtilErrorCode.NO_INSERT_PRIVILEGE, ""); + throw asAddaxException(PERMISSION_ERROR, ""); } public static AddaxException asDeletePriException(String userName, String jdbcUrl) { - throw asAddaxException(DBUtilErrorCode.NO_DELETE_PRIVILEGE, ""); + throw asAddaxException(PERMISSION_ERROR, ""); } public static AddaxException asSplitPKException(Exception e, String splitSql, String splitPkID) { -// return asAddaxException(DBUtilErrorCode.READ_RECORD_FAIL, "配置的SplitPK为: " + splitPkID + ", 执行的SQL为: " + splitSql + " 具体错误信息为：" + e); - return asAddaxException(DBUtilErrorCode.READ_RECORD_FAIL, e.getMessage()); + return asAddaxException(EXECUTE_FAIL, e.getMessage()); } } diff --git a/lib/addax-rdbms/src/main/java/com/wgzhao/addax/rdbms/writer/CommonRdbmsWriter.java b/lib/addax-rdbms/src/main/java/com/wgzhao/addax/rdbms/writer/CommonRdbmsWriter.java index 81c8a0d11..80aa8c4d9 100644 --- a/lib/addax-rdbms/src/main/java/com/wgzhao/addax/rdbms/writer/CommonRdbmsWriter.java +++ b/lib/addax-rdbms/src/main/java/com/wgzhao/addax/rdbms/writer/CommonRdbmsWriter.java @@ -30,7 +30,6 @@ import com.wgzhao.addax.common.plugin.TaskPluginCollector; import com.wgzhao.addax.common.util.Configuration; import com.wgzhao.addax.rdbms.util.DBUtil; -import com.wgzhao.addax.rdbms.util.DBUtilErrorCode; import com.wgzhao.addax.rdbms.util.DataBaseType; import com.wgzhao.addax.rdbms.util.RdbmsException; import com.wgzhao.addax.rdbms.writer.util.OriginalConfPretreatmentUtil; @@ -40,7 +39,6 @@ import org.slf4j.LoggerFactory; import java.math.BigDecimal; -import java.sql.Clob; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; @@ -50,6 +48,10 @@ import java.util.List; import java.util.Map; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.EXECUTE_FAIL; +import static com.wgzhao.addax.common.spi.ErrorCode.NOT_SUPPORT_TYPE; + public class CommonRdbmsWriter { public static class Job @@ -303,7 +305,7 @@ public void startWriteWithConnection(RecordReceiver recordReceiver, TaskPluginCo if (record.getColumnNumber() != this.columnNumber) { // 源头读取字段列数与目的表字段写入列数不相等，直接报错 throw AddaxException.asAddaxException( - DBUtilErrorCode.CONF_ERROR, + CONFIG_ERROR, String.format( "The item column number [%d] in source file not equals the column number [%d] in table.", record.getColumnNumber(), @@ -326,7 +328,7 @@ public void startWriteWithConnection(RecordReceiver recordReceiver, TaskPluginCo } catch (Exception e) { throw AddaxException.asAddaxException( - DBUtilErrorCode.WRITE_DATA_ERROR, e); + EXECUTE_FAIL, e); } finally { writeBuffer.clear(); @@ -411,7 +413,7 @@ protected void doBatchInsert(Connection connection, List buffer) } catch (Exception e) { throw AddaxException.asAddaxException( - DBUtilErrorCode.WRITE_DATA_ERROR, e); + EXECUTE_FAIL, e); } finally { DBUtil.closeDBResources(preparedStatement, null); @@ -441,7 +443,7 @@ protected void doOneInsert(Connection connection, List buffer) } } catch (Exception e) { - throw AddaxException.asAddaxException(DBUtilErrorCode.WRITE_DATA_ERROR, e); + throw AddaxException.asAddaxException(EXECUTE_FAIL, e); } finally { DBUtil.closeDBResources(preparedStatement, null); @@ -564,7 +566,7 @@ protected PreparedStatement fillPreparedStatementColumnType(PreparedStatement pr default: Map map = this.resultSetMetaData.get(columnIndex); throw AddaxException.asAddaxException( - DBUtilErrorCode.UNSUPPORTED_TYPE, + NOT_SUPPORT_TYPE, String.format( "您的配置文件中的列配置信息有误. 不支持数据库写入这种字段类型. 字段名:[%s], " + "字段SQL类型编号:[%s], 字段Java类型:[%s]. 请修改表中该字段的类型或者不同步该字段.", diff --git a/lib/addax-rdbms/src/main/java/com/wgzhao/addax/rdbms/writer/util/OriginalConfPretreatmentUtil.java b/lib/addax-rdbms/src/main/java/com/wgzhao/addax/rdbms/writer/util/OriginalConfPretreatmentUtil.java index 054c21320..5b08cf996 100644 --- a/lib/addax-rdbms/src/main/java/com/wgzhao/addax/rdbms/writer/util/OriginalConfPretreatmentUtil.java +++ b/lib/addax-rdbms/src/main/java/com/wgzhao/addax/rdbms/writer/util/OriginalConfPretreatmentUtil.java @@ -29,7 +29,6 @@ import com.wgzhao.addax.common.util.ListUtil; import com.wgzhao.addax.rdbms.util.ConnectionFactory; import com.wgzhao.addax.rdbms.util.DBUtil; -import com.wgzhao.addax.rdbms.util.DBUtilErrorCode; import com.wgzhao.addax.rdbms.util.DataBaseType; import com.wgzhao.addax.rdbms.util.JdbcConnectionFactory; import com.wgzhao.addax.rdbms.util.TableExpandUtil; @@ -41,6 +40,10 @@ import java.util.ArrayList; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; + public final class OriginalConfPretreatmentUtil { private static final Logger LOG = LoggerFactory.getLogger(OriginalConfPretreatmentUtil.class); @@ -52,7 +55,7 @@ private OriginalConfPretreatmentUtil() {} public static void doPretreatment(Configuration originalConfig, DataBaseType dataBaseType) { // 检查 username 配置（必填） - originalConfig.getNecessaryValue(Key.USERNAME, DBUtilErrorCode.REQUIRED_VALUE); + originalConfig.getNecessaryValue(Key.USERNAME, REQUIRED_VALUE); // 有些数据库没有密码，因此密码不再作为必选项 if (originalConfig.getString(Key.PASSWORD) == null) { @@ -77,7 +80,7 @@ public static void doCheckBatchSize(Configuration originalConfig) // 检查batchSize 配置（选填，如果未填写，则设置为默认值） int batchSize = originalConfig.getInt(Key.BATCH_SIZE, Constant.DEFAULT_BATCH_SIZE); if (batchSize < 1) { - throw AddaxException.asAddaxException(DBUtilErrorCode.ILLEGAL_VALUE, String.format( + throw AddaxException.asAddaxException(ILLEGAL_VALUE, String.format( "The item batchSize [%s] must be greater than 1. recommended value range is [100,1000].", batchSize)); } @@ -100,7 +103,7 @@ public static void simplifyConf(Configuration originalConfig) } String jdbcUrl = connConf.getString(Key.JDBC_URL); if (StringUtils.isBlank(jdbcUrl)) { - throw AddaxException.asAddaxException(DBUtilErrorCode.REQUIRED_VALUE, "The item jdbcUrl is required."); + throw AddaxException.asAddaxException(REQUIRED_VALUE, "The item jdbcUrl is required."); } jdbcUrl = dataBaseType.appendJDBCSuffixForWriter(jdbcUrl); @@ -109,7 +112,7 @@ public static void simplifyConf(Configuration originalConfig) List tables = connConf.getList(Key.TABLE, String.class); if (null == tables || tables.isEmpty()) { - throw AddaxException.asAddaxException(DBUtilErrorCode.REQUIRED_VALUE, + throw AddaxException.asAddaxException(REQUIRED_VALUE, "The item table is required."); } @@ -117,8 +120,7 @@ public static void simplifyConf(Configuration originalConfig) List expandedTables = TableExpandUtil.expandTableConf(dataBaseType, tables); if (expandedTables.isEmpty()) { - throw AddaxException.asAddaxException(DBUtilErrorCode.CONF_ERROR, - "The item table is required."); + throw AddaxException.asAddaxException(REQUIRED_VALUE, "The item table is required."); } tableNum += expandedTables.size(); @@ -133,7 +135,7 @@ public static void dealColumnConf(Configuration originalConfig, ConnectionFactor { List userConfiguredColumns = originalConfig.getList(Key.COLUMN, String.class); if (null == userConfiguredColumns || userConfiguredColumns.isEmpty()) { - throw AddaxException.asAddaxException(DBUtilErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "The item column is required and can not be empty."); } else { @@ -157,7 +159,7 @@ public static void dealColumnConf(Configuration originalConfig, ConnectionFactor originalConfig.set(Key.COLUMN, allColumns); } else if (userConfiguredColumns.size() > allColumns.size()) { - throw AddaxException.asAddaxException(DBUtilErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(CONFIG_ERROR, String.format("The number of columns your configured [%d] are greater than the number of table columns [%d].", userConfiguredColumns.size(), allColumns.size())); } diff --git a/lib/addax-rdbms/src/main/java/com/wgzhao/addax/rdbms/writer/util/WriterUtil.java b/lib/addax-rdbms/src/main/java/com/wgzhao/addax/rdbms/writer/util/WriterUtil.java index ad1248825..481da026f 100644 --- a/lib/addax-rdbms/src/main/java/com/wgzhao/addax/rdbms/writer/util/WriterUtil.java +++ b/lib/addax-rdbms/src/main/java/com/wgzhao/addax/rdbms/writer/util/WriterUtil.java @@ -27,7 +27,6 @@ import com.wgzhao.addax.common.exception.AddaxException; import com.wgzhao.addax.common.util.Configuration; import com.wgzhao.addax.rdbms.util.DBUtil; -import com.wgzhao.addax.rdbms.util.DBUtilErrorCode; import com.wgzhao.addax.rdbms.util.DataBaseType; import com.wgzhao.addax.rdbms.util.RdbmsException; import org.apache.commons.lang3.StringUtils; @@ -40,6 +39,9 @@ import java.util.Arrays; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; + public final class WriterUtil { private static final Logger LOG = LoggerFactory.getLogger(WriterUtil.class); @@ -64,7 +66,7 @@ public static List doSplit(Configuration simplifiedConf, int advi } if (tableNumber != adviceNumber) { - throw AddaxException.asAddaxException(DBUtilErrorCode.CONF_ERROR, + throw AddaxException.asAddaxException(CONFIG_ERROR, String.format("您的配置文件中的列配置信息有误. 您要写入的目的端的表个数是:%s , 但是根据系统建议需要切分的份数是：%s. 请检查您的配置并作出修改.", tableNumber, adviceNumber)); } @@ -139,7 +141,7 @@ public static String getWriteTemplate(List columnHolders, List v boolean isWriteModeLegal = mode.startsWith("insert") || mode.startsWith("replace") || mode.startsWith("update"); if (!isWriteModeLegal) { - throw AddaxException.asAddaxException(DBUtilErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, String.format("您所配置的 writeMode:%s 错误. 目前仅支持replace,update 或 insert 方式. 请检查您的配置并作出修改.", writeMode)); } String writeDataSqlTemplate; @@ -161,7 +163,7 @@ else if (dataBaseType == DataBaseType.SQLServer) { "INSERT (" + columns + ") VALUES ( " + placeHolders + " );"; } else { - throw AddaxException.asAddaxException(DBUtilErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, String.format("当前数据库不支持 writeMode:%s 模式.", writeMode)); } } diff --git a/lib/addax-storage/src/main/java/com/wgzhao/addax/storage/reader/StorageReaderErrorCode.java b/lib/addax-storage/src/main/java/com/wgzhao/addax/storage/reader/StorageReaderErrorCode.java deleted file mode 100644 index 042ca6574..000000000 --- a/lib/addax-storage/src/main/java/com/wgzhao/addax/storage/reader/StorageReaderErrorCode.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, - * * software distributed under the License is distributed on an - * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * * KIND, either express or implied. See the License for the - * * specific language governing permissions and limitations - * * under the License. - * - */ - -package com.wgzhao.addax.storage.reader; - -import com.wgzhao.addax.common.spi.ErrorCode; - -/** - * Created by haiwei.luo on 14-9-20. - */ -public enum StorageReaderErrorCode - implements ErrorCode -{ - CONFIG_INVALID_EXCEPTION("StorageReader-00", "您的参数配置错误."), - NOT_SUPPORT_TYPE("StorageReader-01", "您配置的列类型暂不支持."), - REQUIRED_VALUE("StorageReader-02", "您缺失了必须填写的参数值."), - ILLEGAL_VALUE("StorageReader-03", "您填写的参数值不合法."), - MIXED_INDEX_VALUE("StorageReader-04", "您的列信息配置同时包含了index,value."), - NO_INDEX_VALUE("StorageReader-05", "您明确的配置列信息,但未填写相应的index,value."), - FILE_NOT_EXISTS("StorageReader-06", "您配置的源路径不存在."), - OPEN_FILE_WITH_CHARSET_ERROR("StorageReader-07", "您配置的编码和实际存储编码不符合."), -// OPEN_FILE_ERROR("StorageReader-08", "您配置的源在打开时异常,建议您检查源源是否有隐藏实体,管道文件等特殊文件."), - READ_FILE_IO_ERROR("StorageReader-09", "您配置的文件在读取时出现IO异常."), - SECURITY_NOT_ENOUGH("StorageReader-10", "您缺少权限执行相应的文件读取操作."), - RUNTIME_EXCEPTION("StorageReader-11", "出现运行时异常, 请联系我们"); - - private final String code; - private final String description; - - StorageReaderErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s].", this.code, this.description); - } -} diff --git a/lib/addax-storage/src/main/java/com/wgzhao/addax/storage/reader/StorageReaderUtil.java b/lib/addax-storage/src/main/java/com/wgzhao/addax/storage/reader/StorageReaderUtil.java index e58bb6e57..816220a67 100644 --- a/lib/addax-storage/src/main/java/com/wgzhao/addax/storage/reader/StorageReaderUtil.java +++ b/lib/addax-storage/src/main/java/com/wgzhao/addax/storage/reader/StorageReaderUtil.java @@ -63,6 +63,14 @@ import java.util.List; import java.util.Objects; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.ENCODING_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.IO_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.NOT_SUPPORT_TYPE; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; + public class StorageReaderUtil { private static final Logger LOG = LoggerFactory.getLogger(StorageReaderUtil.class); @@ -118,20 +126,18 @@ else if ("lzo".equalsIgnoreCase(compress)) { } catch (UnsupportedEncodingException uee) { throw AddaxException.asAddaxException( - StorageReaderErrorCode.OPEN_FILE_WITH_CHARSET_ERROR, + ENCODING_ERROR, String.format("%s is unsupported", encoding), uee); } catch (NullPointerException e) { - throw AddaxException.asAddaxException( - StorageReaderErrorCode.RUNTIME_EXCEPTION, e); + throw AddaxException.asAddaxException(RUNTIME_ERROR, e); } catch (IOException e) { throw AddaxException.asAddaxException( - StorageReaderErrorCode.READ_FILE_IO_ERROR, String.format("Failed to read stream [%s].", fileName), e); + IO_ERROR, String.format("Failed to read stream [%s].", fileName), e); } catch (CompressorException e) { - throw AddaxException.asAddaxException( - StorageReaderErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(NOT_SUPPORT_TYPE, "The compress algorithm [" + compress + "] is unsupported yet" ); } @@ -151,7 +157,7 @@ public static void doReadFromStream(BufferedReader reader, String fileName, .getString(Key.FIELD_DELIMITER); if (null != delimiterInStr && 1 != delimiterInStr.length()) { throw AddaxException.asAddaxException( - StorageReaderErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, String.format("The delimiter ONLY has one char, [%s] is illegal", delimiterInStr)); } if (null == delimiterInStr) { @@ -180,23 +186,23 @@ public static void doReadFromStream(BufferedReader reader, String fileName, } catch (UnsupportedEncodingException uee) { throw AddaxException.asAddaxException( - StorageReaderErrorCode.OPEN_FILE_WITH_CHARSET_ERROR, + ENCODING_ERROR, String.format("The encoding: [%s] is unsupported", encoding), uee); } catch (FileNotFoundException fnfe) { throw AddaxException.asAddaxException( - StorageReaderErrorCode.FILE_NOT_EXISTS, String.format("The file [%s] does not exists ", fileName), fnfe); + IO_ERROR, String.format("The file [%s] does not exists ", fileName), fnfe); } catch (IOException ioe) { throw AddaxException.asAddaxException( - StorageReaderErrorCode.READ_FILE_IO_ERROR, String.format("Failed to ead file [%s]", fileName), ioe); + IO_ERROR, String.format("Failed to ead file [%s]", fileName), ioe); } catch (Exception e) { throw AddaxException.asAddaxException( - StorageReaderErrorCode.RUNTIME_EXCEPTION, e); + RUNTIME_ERROR, e); } finally { - + IOUtils.closeQuietly(reader, null); } } @@ -248,12 +254,12 @@ public static void transportOneRecord(RecordSender recordSender, List columns = readerConfiguration.getListConfiguration(Key.COLUMN); if (null == columns || columns.isEmpty()) { - throw AddaxException.asAddaxException(StorageReaderErrorCode.REQUIRED_VALUE, "The item columns is required."); + throw AddaxException.asAddaxException(REQUIRED_VALUE, "The item columns is required."); } // handle ["*"] if (1 == columns.size()) { @@ -423,21 +429,21 @@ public static void validateColumn(Configuration readerConfiguration) if (null != columns && !columns.isEmpty()) { for (Configuration eachColumnConf : columns) { - eachColumnConf.getNecessaryValue(Key.TYPE, StorageReaderErrorCode.REQUIRED_VALUE); + eachColumnConf.getNecessaryValue(Key.TYPE, REQUIRED_VALUE); Integer columnIndex = eachColumnConf.getInt(Key.INDEX); String columnValue = eachColumnConf.getString(Key.VALUE); if (null == columnIndex && null == columnValue) { - throw AddaxException.asAddaxException(StorageReaderErrorCode.NO_INDEX_VALUE, + throw AddaxException.asAddaxException(CONFIG_ERROR, "You must configure one of index or name or value"); } if (null != columnIndex && null != columnValue) { - throw AddaxException.asAddaxException(StorageReaderErrorCode.MIXED_INDEX_VALUE, + throw AddaxException.asAddaxException(CONFIG_ERROR, "You both configure index, value, or name, you can ONLY specify the one each column"); } if (null != columnIndex && columnIndex < 0) { - throw AddaxException.asAddaxException(StorageReaderErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, String.format("The value of index must be greater than 0, %s is illegal", columnIndex)); } } @@ -456,7 +462,7 @@ public static String getRegexPathParentPath(String regexPath) String parentPath; parentPath = regexPath.substring(0, lastDirSeparator + 1); if (parentPath.contains("*") || parentPath.contains("?")) { - throw AddaxException.asAddaxException(StorageReaderErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, String.format("The path '%s' is illegal, ONLY the trail folder can container wildcard * or ? ", regexPath)); } diff --git a/lib/addax-storage/src/main/java/com/wgzhao/addax/storage/writer/StorageWriterErrorCode.java b/lib/addax-storage/src/main/java/com/wgzhao/addax/storage/writer/StorageWriterErrorCode.java deleted file mode 100644 index f4a8809f4..000000000 --- a/lib/addax-storage/src/main/java/com/wgzhao/addax/storage/writer/StorageWriterErrorCode.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, - * * software distributed under the License is distributed on an - * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * * KIND, either express or implied. See the License for the - * * specific language governing permissions and limitations - * * under the License. - * - */ - -package com.wgzhao.addax.storage.writer; - -import com.wgzhao.addax.common.spi.ErrorCode; - -public enum StorageWriterErrorCode - implements ErrorCode -{ - ILLEGAL_VALUE("UnstructuredStorageWriter-00", "您填写的参数值不合法."), - WRITE_FILE_WITH_CHARSET_ERROR("UnstructuredStorageWriter-01", "您配置的编码未能正常写入."), - WRITE_FILE_IO_ERROR("UnstructuredStorageWriter-02", "您配置的文件在写入时出现IO异常."), - RUNTIME_EXCEPTION("UnstructuredStorageWriter-03", "出现运行时异常, 请联系我们"), - REQUIRED_VALUE("UnstructuredStorageWriter-04", "您缺失了必须填写的参数值."), - SQL_REQUIRED_TABLE_NAME("UnstructuredStorageWriter-05", "sql format required table name."), - ; - - private final String code; - private final String description; - - StorageWriterErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s].", this.code, - this.description); - } -} diff --git a/lib/addax-storage/src/main/java/com/wgzhao/addax/storage/writer/StorageWriterUtil.java b/lib/addax-storage/src/main/java/com/wgzhao/addax/storage/writer/StorageWriterUtil.java index c0adc2a12..886578ed0 100644 --- a/lib/addax-storage/src/main/java/com/wgzhao/addax/storage/writer/StorageWriterUtil.java +++ b/lib/addax-storage/src/main/java/com/wgzhao/addax/storage/writer/StorageWriterUtil.java @@ -34,7 +34,6 @@ import com.wgzhao.addax.common.plugin.RecordReceiver; import com.wgzhao.addax.common.plugin.TaskPluginCollector; import com.wgzhao.addax.common.util.Configuration; -import com.wgzhao.addax.storage.reader.StorageReaderErrorCode; import com.wgzhao.addax.storage.util.FileHelper; import org.apache.commons.compress.compressors.CompressorException; import org.apache.commons.compress.compressors.CompressorOutputStream; @@ -61,27 +60,36 @@ import java.util.Objects; import java.util.Set; -import static com.wgzhao.addax.storage.writer.StorageWriterErrorCode.SQL_REQUIRED_TABLE_NAME; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.ENCODING_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.IO_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.NOT_SUPPORT_TYPE; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; -public class StorageWriterUtil { +public class StorageWriterUtil +{ private static final Logger LOG = LoggerFactory.getLogger(StorageWriterUtil.class); private static final Set supportedWriteModes = new HashSet<>(Arrays.asList("truncate", "append", "nonConflict", "overwrite")); - private StorageWriterUtil() { + private StorageWriterUtil() + { } /* * check parameter: writeMode, encoding, compress, filedDelimiter */ - public static void validateParameter(Configuration writerConfiguration) { + public static void validateParameter(Configuration writerConfiguration) + { // writeMode check - String writeMode = writerConfiguration.getNecessaryValue(Key.WRITE_MODE, StorageWriterErrorCode.REQUIRED_VALUE); + String writeMode = writerConfiguration.getNecessaryValue(Key.WRITE_MODE, REQUIRED_VALUE); writeMode = writeMode.trim(); if (!supportedWriteModes.contains(writeMode)) { throw AddaxException .asAddaxException( - StorageWriterErrorCode.ILLEGAL_VALUE, + NOT_SUPPORT_TYPE, String.format( "The writeMode [%s] is unsupported, it only supports [%s]", writeMode, StringUtils.join(supportedWriteModes, ","))); @@ -94,14 +102,16 @@ public static void validateParameter(Configuration writerConfiguration) { // like " ", null LOG.warn(String.format("The item encoding is empty, uses [%s] as default.", Constant.DEFAULT_ENCODING)); writerConfiguration.set(Key.ENCODING, Constant.DEFAULT_ENCODING); - } else { + } + else { try { encoding = encoding.trim(); writerConfiguration.set(Key.ENCODING, encoding); Charsets.toCharset(encoding); - } catch (Exception e) { + } + catch (Exception e) { throw AddaxException.asAddaxException( - StorageWriterErrorCode.ILLEGAL_VALUE, + NOT_SUPPORT_TYPE, String.format("The encoding [%s] is unsupported.", encoding), e); } } @@ -117,7 +127,7 @@ public static void validateParameter(Configuration writerConfiguration) { // warn: if it has, length must be one if (null != delimiterInStr && 1 != delimiterInStr.length()) { throw AddaxException.asAddaxException( - StorageWriterErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, String.format("The delimiter only supports single character, [%s] is invalid.", delimiterInStr)); } if (null == delimiterInStr) { @@ -129,12 +139,13 @@ public static void validateParameter(Configuration writerConfiguration) { String fileFormat = writerConfiguration.getString(Key.FILE_FORMAT, Constant.DEFAULT_FILE_FORMAT); if (!Constant.SUPPORTED_FILE_FORMAT.contains(fileFormat)) { throw AddaxException.asAddaxException( - StorageWriterErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, String.format("The fileFormat [%s] you configured is invalid, it only supports %s.", fileFormat, Constant.SUPPORTED_FILE_FORMAT)); } } - public static List split(Configuration writerSliceConfig, Set originAllFileExists, int mandatoryNumber) { + public static List split(Configuration writerSliceConfig, Set originAllFileExists, int mandatoryNumber) + { List writerSplitConfigs = new ArrayList<>(); LOG.info("Begin to split..."); if (mandatoryNumber == 1) { @@ -163,7 +174,8 @@ public static List split(Configuration writerSliceConfig, Set recordToList(Record record, String nullFormat, DateFormat dateParse, TaskPluginCollector taskPluginCollector) { + public static List recordToList(Record record, String nullFormat, DateFormat dateParse, TaskPluginCollector taskPluginCollector) + { try { List splitRows = new ArrayList<>(); int recordLength = record.getColumnNumber(); @@ -308,15 +332,18 @@ public static List recordToList(Record record, String nullFormat, DateFo if (null == column || null == column.getRawData() || column.asString().equals(nullFormat)) { // warn: it's all ok if nullFormat is null splitRows.add(nullFormat); - } else { + } + else { // warn: it's all ok if nullFormat is null boolean isDateColumn = column instanceof DateColumn || column instanceof TimestampColumn; if (!isDateColumn) { splitRows.add(column.asString()); - } else { + } + else { if (null != dateParse) { splitRows.add(dateParse.format(column.asDate())); - } else { + } + else { splitRows.add(column.asString()); } } @@ -324,16 +351,19 @@ public static List recordToList(Record record, String nullFormat, DateFo } } return splitRows; - } catch (Exception e) { + } + catch (Exception e) { // warn: dirty data taskPluginCollector.collectDirtyRecord(record, e); return null; } } - public static void writeToSql(RecordReceiver lineReceiver, BufferedWriter writer, Configuration config) throws IOException { + public static void writeToSql(RecordReceiver lineReceiver, BufferedWriter writer, Configuration config) + throws IOException + { // sql format required table and column name and optional extendedInsert and optional batchSize - String tableName = config.getNecessaryValue(Key.TABLE, SQL_REQUIRED_TABLE_NAME); + String tableName = config.getNecessaryValue(Key.TABLE, REQUIRED_VALUE); String existColumns = config.getString(Key.COLUMN, null); List columns = null; if (existColumns != null) { @@ -350,18 +380,19 @@ public static void writeToSql(RecordReceiver lineReceiver, BufferedWriter writer StringBuilder sb = new StringBuilder(); sb.append(sqlHeader).append(" VALUES ("); while ((record = lineReceiver.getFromReader()) != null) { - if (columns!= null && record.getColumnNumber() != columns.size()) { + if (columns != null && record.getColumnNumber() != columns.size()) { throw AddaxException.asAddaxException( - StorageWriterErrorCode.ILLEGAL_VALUE, + CONFIG_ERROR, String.format("The column number [%d] of record is not equal to the column number [%d] of table.", record.getColumnNumber(), columns.size())); } Column column; - for (int i =0; i < record.getColumnNumber(); i++ ) { + for (int i = 0; i < record.getColumnNumber(); i++) { column = record.getColumn(i); if (column instanceof LongColumn || column instanceof BoolColumn) { sb.append(column.asString()); - } else { + } + else { sb.append("'").append(column.asString()).append("'"); } if (i < record.getColumnNumber() - 1) { @@ -379,11 +410,13 @@ public static void writeToSql(RecordReceiver lineReceiver, BufferedWriter writer sb.append(sqlHeader).append(" VALUES ("); // reset counter curNum = 0; - } else { + } + else { sb.append("), ("); curNum++; } - } else { + } + else { sb.append(");\n"); //write to file writer.write(sb.toString()); diff --git a/plugin/reader/cassandrareader/src/main/java/com/wgzhao/addax/plugin/reader/cassandrareader/CassandraReader.java b/plugin/reader/cassandrareader/src/main/java/com/wgzhao/addax/plugin/reader/cassandrareader/CassandraReader.java index c5c287850..952cd760e 100644 --- a/plugin/reader/cassandrareader/src/main/java/com/wgzhao/addax/plugin/reader/cassandrareader/CassandraReader.java +++ b/plugin/reader/cassandrareader/src/main/java/com/wgzhao/addax/plugin/reader/cassandrareader/CassandraReader.java @@ -37,8 +37,7 @@ public class CassandraReader extends Reader { - private static final Logger LOG = LoggerFactory - .getLogger(CassandraReader.class); + private static final Logger LOG = LoggerFactory.getLogger(CassandraReader.class); public static class Job extends Reader.Job diff --git a/plugin/reader/cassandrareader/src/main/java/com/wgzhao/addax/plugin/reader/cassandrareader/CassandraReaderErrorCode.java b/plugin/reader/cassandrareader/src/main/java/com/wgzhao/addax/plugin/reader/cassandrareader/CassandraReaderErrorCode.java deleted file mode 100644 index 023426c11..000000000 --- a/plugin/reader/cassandrareader/src/main/java/com/wgzhao/addax/plugin/reader/cassandrareader/CassandraReaderErrorCode.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.reader.cassandrareader; - -import com.wgzhao.addax.common.spi.ErrorCode; - -public enum CassandraReaderErrorCode - implements ErrorCode -{ - CONF_ERROR("CassandraReader-00", "配置错误."), - ; - - private final String code; - private final String description; - - CassandraReaderErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s]. ", this.code, - this.description); - } -} diff --git a/plugin/reader/cassandrareader/src/main/java/com/wgzhao/addax/plugin/reader/cassandrareader/CassandraReaderHelper.java b/plugin/reader/cassandrareader/src/main/java/com/wgzhao/addax/plugin/reader/cassandrareader/CassandraReaderHelper.java index 7fdf42632..07a64da8d 100644 --- a/plugin/reader/cassandrareader/src/main/java/com/wgzhao/addax/plugin/reader/cassandrareader/CassandraReaderHelper.java +++ b/plugin/reader/cassandrareader/src/main/java/com/wgzhao/addax/plugin/reader/cassandrareader/CassandraReaderHelper.java @@ -61,6 +61,8 @@ import java.util.Map; import java.util.Set; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; + /** * Created by mazhenlin on 2019/8/21. */ @@ -418,7 +420,7 @@ static Record buildRecord(Record record, Row rs, ColumnDefinitions metaData, int default: throw AddaxException .asAddaxException( - CassandraReaderErrorCode.CONF_ERROR, + CONFIG_ERROR, String.format( "您的配置文件中的列配置信息有误. 不支持数据库读取这种字段类型. 字段名:[%s], " + "字段类型:[%s]. ", @@ -429,7 +431,7 @@ static Record buildRecord(Record record, Row rs, ColumnDefinitions metaData, int catch (TypeNotSupported t) { throw AddaxException .asAddaxException( - CassandraReaderErrorCode.CONF_ERROR, + CONFIG_ERROR, String.format( "您的配置文件中的列配置信息有误. 不支持数据库读取这种字段类型. 字段名:[%s], " + "字段类型:[%s]. ", @@ -574,7 +576,7 @@ public static void checkConfig(Configuration jobConfig, Cluster cluster) if (cluster.getMetadata().getKeyspace(keyspace) == null) { throw AddaxException .asAddaxException( - CassandraReaderErrorCode.CONF_ERROR, + CONFIG_ERROR, String.format( "配置信息有错误.keyspace'%s'不存在 .", keyspace)); @@ -584,7 +586,7 @@ public static void checkConfig(Configuration jobConfig, Cluster cluster) if (tableMetadata == null) { throw AddaxException .asAddaxException( - CassandraReaderErrorCode.CONF_ERROR, + CONFIG_ERROR, String.format( "配置信息有错误.表'%s'不存在 .", table)); @@ -594,7 +596,7 @@ public static void checkConfig(Configuration jobConfig, Cluster cluster) if (name == null || name.isEmpty()) { throw AddaxException .asAddaxException( - CassandraReaderErrorCode.CONF_ERROR, + CONFIG_ERROR, String.format( "配置信息有错误.列信息中需要包含'%s'字段 .", MyKey.COLUMN_NAME)); } @@ -606,7 +608,7 @@ static void ensureExists(Configuration jobConfig, String keyword) if (jobConfig.get(keyword) == null) { throw AddaxException .asAddaxException( - CassandraReaderErrorCode.CONF_ERROR, + CONFIG_ERROR, String.format( "配置信息有错误.参数'%s'为必填项 .", keyword)); @@ -619,7 +621,7 @@ static void ensureStringExists(Configuration jobConfig, String keyword) if (jobConfig.getString(keyword).isEmpty()) { throw AddaxException .asAddaxException( - CassandraReaderErrorCode.CONF_ERROR, + CONFIG_ERROR, String.format( "配置信息有错误.参数'%s'不能为空 .", keyword)); diff --git a/plugin/reader/cassandrareader/src/main/java/com/wgzhao/addax/plugin/reader/cassandrareader/LocalStrings.properties b/plugin/reader/cassandrareader/src/main/java/com/wgzhao/addax/plugin/reader/cassandrareader/LocalStrings.properties deleted file mode 100644 index 2245fb7d6..000000000 --- a/plugin/reader/cassandrareader/src/main/java/com/wgzhao/addax/plugin/reader/cassandrareader/LocalStrings.properties +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -errorcode.config_invalid_exception=\u914D\u7F6E\u9519\u8BEF \ No newline at end of file diff --git a/plugin/reader/cassandrareader/src/main/java/com/wgzhao/addax/plugin/reader/cassandrareader/LocalStrings_en_US.properties b/plugin/reader/cassandrareader/src/main/java/com/wgzhao/addax/plugin/reader/cassandrareader/LocalStrings_en_US.properties deleted file mode 100644 index d8a500d9d..000000000 --- a/plugin/reader/cassandrareader/src/main/java/com/wgzhao/addax/plugin/reader/cassandrareader/LocalStrings_en_US.properties +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - diff --git a/plugin/reader/cassandrareader/src/main/java/com/wgzhao/addax/plugin/reader/cassandrareader/LocalStrings_ja_JP.properties b/plugin/reader/cassandrareader/src/main/java/com/wgzhao/addax/plugin/reader/cassandrareader/LocalStrings_ja_JP.properties deleted file mode 100644 index 2245fb7d6..000000000 --- a/plugin/reader/cassandrareader/src/main/java/com/wgzhao/addax/plugin/reader/cassandrareader/LocalStrings_ja_JP.properties +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -errorcode.config_invalid_exception=\u914D\u7F6E\u9519\u8BEF \ No newline at end of file diff --git a/plugin/reader/cassandrareader/src/main/java/com/wgzhao/addax/plugin/reader/cassandrareader/LocalStrings_zh_CN.properties b/plugin/reader/cassandrareader/src/main/java/com/wgzhao/addax/plugin/reader/cassandrareader/LocalStrings_zh_CN.properties deleted file mode 100644 index 2245fb7d6..000000000 --- a/plugin/reader/cassandrareader/src/main/java/com/wgzhao/addax/plugin/reader/cassandrareader/LocalStrings_zh_CN.properties +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -errorcode.config_invalid_exception=\u914D\u7F6E\u9519\u8BEF \ No newline at end of file diff --git a/plugin/reader/cassandrareader/src/main/java/com/wgzhao/addax/plugin/reader/cassandrareader/LocalStrings_zh_HK.properties b/plugin/reader/cassandrareader/src/main/java/com/wgzhao/addax/plugin/reader/cassandrareader/LocalStrings_zh_HK.properties deleted file mode 100644 index 2245fb7d6..000000000 --- a/plugin/reader/cassandrareader/src/main/java/com/wgzhao/addax/plugin/reader/cassandrareader/LocalStrings_zh_HK.properties +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -errorcode.config_invalid_exception=\u914D\u7F6E\u9519\u8BEF \ No newline at end of file diff --git a/plugin/reader/cassandrareader/src/main/java/com/wgzhao/addax/plugin/reader/cassandrareader/LocalStrings_zh_TW.properties b/plugin/reader/cassandrareader/src/main/java/com/wgzhao/addax/plugin/reader/cassandrareader/LocalStrings_zh_TW.properties deleted file mode 100644 index 2245fb7d6..000000000 --- a/plugin/reader/cassandrareader/src/main/java/com/wgzhao/addax/plugin/reader/cassandrareader/LocalStrings_zh_TW.properties +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -errorcode.config_invalid_exception=\u914D\u7F6E\u9519\u8BEF \ No newline at end of file diff --git a/plugin/reader/cassandrareader/src/main/java/com/wgzhao/addax/plugin/reader/cassandrareader/MyKey.java b/plugin/reader/cassandrareader/src/main/java/com/wgzhao/addax/plugin/reader/cassandrareader/MyKey.java index 4dcba0533..4cc1ba546 100644 --- a/plugin/reader/cassandrareader/src/main/java/com/wgzhao/addax/plugin/reader/cassandrareader/MyKey.java +++ b/plugin/reader/cassandrareader/src/main/java/com/wgzhao/addax/plugin/reader/cassandrareader/MyKey.java @@ -24,7 +24,8 @@ /** * Created by mazhenlin on 2019/8/19. */ -public class MyKey extends Key +public class MyKey + extends Key { public final static String HOST = "host"; diff --git a/plugin/reader/datareader/src/main/java/com/wgzhao/addax/plugin/reader/datareader/DataReader.java b/plugin/reader/datareader/src/main/java/com/wgzhao/addax/plugin/reader/datareader/DataReader.java index f86f3cce8..c1db5f8a3 100644 --- a/plugin/reader/datareader/src/main/java/com/wgzhao/addax/plugin/reader/datareader/DataReader.java +++ b/plugin/reader/datareader/src/main/java/com/wgzhao/addax/plugin/reader/datareader/DataReader.java @@ -67,6 +67,9 @@ import static com.wgzhao.addax.common.base.Key.SLICE_RECORD_COUNT; import static com.wgzhao.addax.common.base.Key.TYPE; import static com.wgzhao.addax.common.base.Key.VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.NOT_SUPPORT_TYPE; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; import static com.wgzhao.addax.plugin.reader.datareader.DataKey.RULE; public class DataReader @@ -107,10 +110,10 @@ public void init() Long sliceRecordCount = this.originalConfig.getLong(SLICE_RECORD_COUNT); if (null == sliceRecordCount) { - throw AddaxException.asAddaxException(DataReaderErrorCode.REQUIRED_VALUE, "sliceRecordCount is required"); + throw AddaxException.asAddaxException(REQUIRED_VALUE, "sliceRecordCount is required"); } else if (sliceRecordCount < 1) { - throw AddaxException.asAddaxException(DataReaderErrorCode.ILLEGAL_VALUE, "sliceRecordCount must be greater than 1"); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "sliceRecordCount must be greater than 1"); } } @@ -118,7 +121,7 @@ private void dealColumn(Configuration originalConfig) { List columns = originalConfig.getList(COLUMN, JSONObject.class); if (null == columns || columns.isEmpty()) { - throw AddaxException.asAddaxException(DataReaderErrorCode.REQUIRED_VALUE, "column is required and NOT be empty"); + throw AddaxException.asAddaxException(REQUIRED_VALUE, "column is required and NOT be empty"); } List dealColumns = new ArrayList<>(); @@ -128,7 +131,7 @@ private void dealColumn(Configuration originalConfig) this.parseMixupFunctions(eachColumnConfig); } catch (Exception e) { - throw AddaxException.asAddaxException(DataReaderErrorCode.NOT_SUPPORT_TYPE, + throw AddaxException.asAddaxException(NOT_SUPPORT_TYPE, String.format("Failed to parse column: %s", e.getMessage()), e); } String typeName = eachColumnConfig.getString(TYPE); @@ -145,7 +148,7 @@ private void dealColumn(Configuration originalConfig) } if (!Type.isTypeIllegal(typeName)) { throw AddaxException.asAddaxException( - DataReaderErrorCode.NOT_SUPPORT_TYPE, + NOT_SUPPORT_TYPE, String.format("不支持类型[%s]", typeName)); } } @@ -213,7 +216,7 @@ private void validateIncrRule(Configuration eachColumnConfig) } catch (NumberFormatException e) { throw AddaxException.asAddaxException( - DataReaderErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, value + " is illegal, it must be a digital string" ); } @@ -229,7 +232,7 @@ else if (fields.length == 2) { } catch (NumberFormatException e) { throw AddaxException.asAddaxException( - DataReaderErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, "The second field must be numeric, value [" + fields[1] + "] is not valid" ); } @@ -245,7 +248,7 @@ else if (fields.length == 2) { } else { throw AddaxException.asAddaxException( - DataReaderErrorCode.NOT_SUPPORT_TYPE, + NOT_SUPPORT_TYPE, "递增序列当前仅支持整数类型(long)和日期类型(date)" ); } @@ -256,7 +259,7 @@ private void validateRandom(Configuration config) String value = config.getString(VALUE); String[] split = value.split(","); if (split.length < 2) { - throw AddaxException.asAddaxException(DataReaderErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, String.format("Illegal random value [%s], supported random value like 'minVal, MaxVal[,scale]'", value)); } String param1 = split[0]; @@ -265,7 +268,7 @@ private void validateRandom(Configuration config) long param2Int; if (StringUtils.isBlank(param1) && StringUtils.isBlank(param2)) { throw AddaxException.asAddaxException( - DataReaderErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, String.format("random混淆函数不合法[%s], 混淆函数random的参数不能为空:%s, %s", value, param1, param2)); } @@ -281,7 +284,7 @@ private void validateRandom(Configuration config) } catch (ParseException e) { throw AddaxException.asAddaxException( - DataReaderErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, String.format("dateFormat参数[%s]和混淆函数random的参数不匹配，解析错误:%s, %s", dateFormat, param1, param2), e); } @@ -292,13 +295,13 @@ private void validateRandom(Configuration config) } if (param1Int < 0 || param2Int < 0) { throw AddaxException.asAddaxException( - DataReaderErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, String.format("random 函数不合法[%s], 混淆函数random的参数不能为负数:%s, %s", value, param1, param2)); } if (!Type.BOOL.name().equalsIgnoreCase(typeName) && param1Int > param2Int) { throw AddaxException.asAddaxException( - DataReaderErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, String.format("random 函数不合法[%s], 混淆函数random的参数需要第一个小于等于第二个:%s, %s", value, param1, param2)); } @@ -337,7 +340,7 @@ else if (!validUnits.contains(unit.toLowerCase())) { } if (!isOK) { throw AddaxException.asAddaxException( - DataReaderErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, unit + " is NOT valid interval unit，for more details, please refer to the documentation"); } } @@ -524,7 +527,7 @@ else if (columnType == Type.DATE) { } catch (java.text.ParseException e) { throw AddaxException.asAddaxException( - DataReaderErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, String.format("can not parse date value [%s] with date format [%s]", fields[0], datePattern) ); } @@ -534,7 +537,7 @@ else if (columnType == Type.DATE) { } else { throw AddaxException.asAddaxException( - DataReaderErrorCode.NOT_SUPPORT_TYPE, + NOT_SUPPORT_TYPE, columnType + " can not support for increment" ); } @@ -577,7 +580,7 @@ else if (columnType == Type.DATE) { case ZIP_CODE: return new LongColumn(RandomSource.XO_RO_SHI_RO_1024_PP.create().nextLong(1000000, 699000)); default: - throw AddaxException.asAddaxException(DataReaderErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, columnRule + " is unsupported"); } } @@ -608,7 +611,7 @@ private Date dateIncrement(Date curDate, int step, String unit) case "s": return DateUtils.addSeconds(curDate, step); default: - throw AddaxException.asAddaxException(DataReaderErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "The date interval unit '" + unit + "' is unsupported"); } } @@ -632,7 +635,7 @@ private Record buildOneRecord(RecordSender recordSender, } } catch (Exception e) { - throw AddaxException.asAddaxException(DataReaderErrorCode.ILLEGAL_VALUE, "构造一个record失败.", e); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "构造一个record失败.", e); } return record; } diff --git a/plugin/reader/datareader/src/main/java/com/wgzhao/addax/plugin/reader/datareader/DataReaderErrorCode.java b/plugin/reader/datareader/src/main/java/com/wgzhao/addax/plugin/reader/datareader/DataReaderErrorCode.java deleted file mode 100644 index 39eefc41c..000000000 --- a/plugin/reader/datareader/src/main/java/com/wgzhao/addax/plugin/reader/datareader/DataReaderErrorCode.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.reader.datareader; - -import com.wgzhao.addax.common.spi.ErrorCode; - -public enum DataReaderErrorCode - implements ErrorCode -{ - REQUIRED_VALUE("DataReader-00", "缺失必要的值"), - ILLEGAL_VALUE("DataReader-01", "值非法"), - NOT_SUPPORT_TYPE("DataReader-02", "不支持的column类型"), - ; - - private final String code; - private final String description; - - DataReaderErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s]. ", this.code, - this.description); - } -} diff --git a/plugin/reader/dbfreader/src/main/java/com/wgzhao/addax/plugin/reader/dbfreader/DbfReader.java b/plugin/reader/dbfreader/src/main/java/com/wgzhao/addax/plugin/reader/dbfreader/DbfReader.java index 4fc975beb..39bb6e4cf 100644 --- a/plugin/reader/dbfreader/src/main/java/com/wgzhao/addax/plugin/reader/dbfreader/DbfReader.java +++ b/plugin/reader/dbfreader/src/main/java/com/wgzhao/addax/plugin/reader/dbfreader/DbfReader.java @@ -30,6 +30,7 @@ import com.wgzhao.addax.common.util.Configuration; import com.wgzhao.addax.storage.reader.StorageReaderUtil; import com.wgzhao.addax.storage.util.FileHelper; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,6 +41,9 @@ import java.util.ArrayList; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; + /** * Created by zhongtian.hu on 19-8-8. */ @@ -57,17 +61,6 @@ public static class Job private List sourceFiles; - private boolean isBlank(Object object) - { - if (null == object) { - return true; - } - if ((object instanceof String)) { - return "".equals(((String) object).trim()); - } - return false; - } - @Override public void init() { @@ -79,12 +72,7 @@ public void init() private void validateParameter() { // Compatible with the old version, path is a string before - String pathInString = this.originConfig.getNecessaryValue(Key.PATH, DbfReaderErrorCode.REQUIRED_VALUE); - if (isBlank(pathInString)) { - throw AddaxException.asAddaxException( - DbfReaderErrorCode.REQUIRED_VALUE, - "您需要指定待读取的源目录或文件"); - } + String pathInString = this.originConfig.getNecessaryValue(Key.PATH, REQUIRED_VALUE); if (!pathInString.startsWith("[") && !pathInString.endsWith("]")) { path = new ArrayList<>(); path.add(pathInString); @@ -93,8 +81,8 @@ private void validateParameter() path = this.originConfig.getList(Key.PATH, String.class); if (null == path || path.isEmpty()) { throw AddaxException.asAddaxException( - DbfReaderErrorCode.REQUIRED_VALUE, - "您需要指定待读取的源目录或文件"); + REQUIRED_VALUE, + "Nothing to read due to no file path specified."); } } } @@ -106,7 +94,7 @@ public void prepare() this.sourceFiles = FileHelper.buildSourceTargets(this.path); - LOG.info("您即将读取的文件数为: [{}]", this.sourceFiles.size()); + LOG.info("{} file(s) to be read", this.sourceFiles.size()); } @Override @@ -132,7 +120,7 @@ public List split(int adviceNumber) int splitNumber = this.sourceFiles.size(); if (0 == splitNumber) { throw AddaxException.asAddaxException( - DbfReaderErrorCode.EMPTY_DIR_EXCEPTION, + RUNTIME_ERROR, String.format("未能找到待读取的文件,请确认您的配置项path: %s", this.originConfig.getString(Key.PATH))); } @@ -193,13 +181,17 @@ public void startRead(RecordSender recordSender) } if (column == null) { throw AddaxException.asAddaxException( - DbfReaderErrorCode.RUNTIME_EXCEPTION, + RUNTIME_ERROR, "无法从指定的DBF文件(" + this.sourceFiles.get(0) + ")获取字段信息" ); } int colNum = column.size(); DBFRow row; for (String fileName : this.sourceFiles) { + if (StringUtils.isBlank(fileName)) { + LOG.warn("source file name is blank, continue..."); + continue; + } LOG.info("begin reading file : [{}]", fileName); try (DBFReader reader = new DBFReader(new FileInputStream(fileName), Charset.forName(encode))) { while ((row = reader.nextRow()) != null) { diff --git a/plugin/reader/dbfreader/src/main/java/com/wgzhao/addax/plugin/reader/dbfreader/DbfReaderErrorCode.java b/plugin/reader/dbfreader/src/main/java/com/wgzhao/addax/plugin/reader/dbfreader/DbfReaderErrorCode.java deleted file mode 100644 index 78c1ac628..000000000 --- a/plugin/reader/dbfreader/src/main/java/com/wgzhao/addax/plugin/reader/dbfreader/DbfReaderErrorCode.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.reader.dbfreader; - -import com.wgzhao.addax.common.spi.ErrorCode; - -/** - * Created by zhongtian.hu on 19-8-8. - */ -public enum DbfReaderErrorCode - implements ErrorCode -{ - REQUIRED_VALUE("DbfReader-00", "您缺失了必须填写的参数值."), - ILLEGAL_VALUE("DbfReader-01", "您填写的参数值不合法."), - MIXED_INDEX_VALUE("DbfReader-02", "您的列信息配置同时包含了index,value."), - NO_INDEX_VALUE("DbfReader-03", "您明确的配置列信息,但未填写相应的index,value."), - FILE_NOT_EXISTS("DbfReader-04", "您配置的目录文件路径不存在."), - OPEN_FILE_WITH_CHARSET_ERROR("DbfReader-05", "您配置的文件编码和实际文件编码不符合."), - OPEN_FILE_ERROR("DbfReader-06", "您配置的文件在打开时异常,建议您检查源目录是否有隐藏文件,管道文件等特殊文件."), - READ_FILE_IO_ERROR("DbfReader-07", "您配置的文件在读取时出现IO异常."), - SECURITY_NOT_ENOUGH("DbfReader-08", "您缺少权限执行相应的文件操作."), - CONFIG_INVALID_EXCEPTION("DbfReader-09", "您的参数配置错误."), - RUNTIME_EXCEPTION("DbfReader-10", "出现运行时异常, 请联系我们"), - EMPTY_DIR_EXCEPTION("DbfReader-11", "您尝试读取的文件目录为空."), - ; - - private final String code; - private final String description; - - private DbfReaderErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s].", this.code, - this.description); - } -} diff --git a/plugin/reader/elasticsearchreader/src/main/java/com/wgzhao/addax/plugin/reader/elasticsearchreader/ESKey.java b/plugin/reader/elasticsearchreader/src/main/java/com/wgzhao/addax/plugin/reader/elasticsearchreader/ESKey.java index bfb3b2898..2c6c6cbf7 100644 --- a/plugin/reader/elasticsearchreader/src/main/java/com/wgzhao/addax/plugin/reader/elasticsearchreader/ESKey.java +++ b/plugin/reader/elasticsearchreader/src/main/java/com/wgzhao/addax/plugin/reader/elasticsearchreader/ESKey.java @@ -27,6 +27,8 @@ import java.util.List; import java.util.Map; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; + public final class ESKey { // ---------------------------------------- @@ -45,7 +47,7 @@ public static SearchType getSearchType(Configuration conf) public static String getEndpoint(Configuration conf) { - return conf.getNecessaryValue("endpoint", ESReaderErrorCode.BAD_CONFIG_VALUE); + return conf.getNecessaryValue("endpoint", CONFIG_ERROR); } public static String getAccessID(Configuration conf) @@ -95,7 +97,7 @@ public static boolean isMultiThread(Configuration conf) public static String getIndexName(Configuration conf) { - return conf.getNecessaryValue("index", ESReaderErrorCode.BAD_CONFIG_VALUE); + return conf.getNecessaryValue("index", CONFIG_ERROR); } public static String getTypeName(Configuration conf) diff --git a/plugin/reader/elasticsearchreader/src/main/java/com/wgzhao/addax/plugin/reader/elasticsearchreader/ESReaderErrorCode.java b/plugin/reader/elasticsearchreader/src/main/java/com/wgzhao/addax/plugin/reader/elasticsearchreader/ESReaderErrorCode.java deleted file mode 100644 index 7ac497174..000000000 --- a/plugin/reader/elasticsearchreader/src/main/java/com/wgzhao/addax/plugin/reader/elasticsearchreader/ESReaderErrorCode.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.reader.elasticsearchreader; - -import com.wgzhao.addax.common.spi.ErrorCode; - -public enum ESReaderErrorCode - implements ErrorCode -{ - BAD_CONFIG_VALUE("ESReader-00", "您配置的值不合法."), - ES_SEARCH_ERROR("ESReader-01", "search出错."), - ES_INDEX_NOT_EXISTS("ESReader-02", "index不存在."), - UNKNOWN_DATA_TYPE("ESReader-03", "无法识别的数据类型."), - COLUMN_CANT_BE_EMPTY("ESReader-04", "column不能为空."), - ; - - private final String code; - private final String description; - - ESReaderErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s]. ", this.code, - this.description); - } -} \ No newline at end of file diff --git a/plugin/reader/elasticsearchreader/src/main/java/com/wgzhao/addax/plugin/reader/elasticsearchreader/EsReader.java b/plugin/reader/elasticsearchreader/src/main/java/com/wgzhao/addax/plugin/reader/elasticsearchreader/EsReader.java index 2b7144eb5..03fd1aaa9 100644 --- a/plugin/reader/elasticsearchreader/src/main/java/com/wgzhao/addax/plugin/reader/elasticsearchreader/EsReader.java +++ b/plugin/reader/elasticsearchreader/src/main/java/com/wgzhao/addax/plugin/reader/elasticsearchreader/EsReader.java @@ -55,6 +55,12 @@ import java.util.List; import java.util.Map; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.EXECUTE_FAIL; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.NOT_SUPPORT_TYPE; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; + /** * @author kesc mail:492167585@qq.com * @since 2020-04-14 10:32 @@ -96,7 +102,7 @@ public void prepare() } } catch (Exception ex) { - throw AddaxException.asAddaxException(ESReaderErrorCode.ES_INDEX_NOT_EXISTS, ex.toString()); + throw AddaxException.asAddaxException(CONFIG_ERROR, ex.toString()); } esClient.closeJestClient(); } @@ -177,11 +183,11 @@ public void init() this.filter = ESKey.getFilter(conf); this.column = ESKey.getColumn(conf); if (column == null || column.isEmpty()) { - throw AddaxException.asAddaxException(ESReaderErrorCode.COLUMN_CANT_BE_EMPTY, "column必须配置"); + throw AddaxException.asAddaxException(REQUIRED_VALUE, "column必须配置"); } if (column.size() == 1 && "*".equals(column.get(0))) { // TODO get column from record - throw AddaxException.asAddaxException(ESReaderErrorCode.COLUMN_CANT_BE_EMPTY, "column暂不支持*配置"); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "column暂不支持*配置"); } } @@ -196,10 +202,10 @@ public void startRead(RecordSender recordSender) searchResult = esClient.search(query, searchType, index, type, scroll, headers, this.column); } catch (Exception e) { - throw AddaxException.asAddaxException(ESReaderErrorCode.ES_SEARCH_ERROR, e); + throw AddaxException.asAddaxException(EXECUTE_FAIL, e); } if (!searchResult.isSucceeded()) { - throw AddaxException.asAddaxException(ESReaderErrorCode.ES_SEARCH_ERROR, searchResult.getResponseCode() + ":" + searchResult.getErrorMessage()); + throw AddaxException.asAddaxException(EXECUTE_FAIL, searchResult.getResponseCode() + ":" + searchResult.getErrorMessage()); } queryPerfRecord.end(); //transport records @@ -221,7 +227,7 @@ public void startRead(RecordSender recordSender) JestResult currScroll = esClient.scroll(scrollId, this.scroll); queryPerfRecord.end(); if (!currScroll.isSucceeded()) { - throw AddaxException.asAddaxException(ESReaderErrorCode.ES_SEARCH_ERROR, + throw AddaxException.asAddaxException(EXECUTE_FAIL, String.format("scroll[id=%s] search error,code:%s,msg:%s", scrollId, currScroll.getResponseCode(), currScroll.getErrorMessage())); } allResultPerfRecord.start(); @@ -233,7 +239,7 @@ public void startRead(RecordSender recordSender) throw dxe; } catch (Exception e) { - throw AddaxException.asAddaxException(ESReaderErrorCode.ES_SEARCH_ERROR, e); + throw AddaxException.asAddaxException(EXECUTE_FAIL, e); } finally { esClient.clearScroll(scrollId); @@ -368,7 +374,7 @@ else if (value instanceof Array) { col = new StringColumn(JSON.toJSONString(value)); } else { - throw AddaxException.asAddaxException(ESReaderErrorCode.UNKNOWN_DATA_TYPE, "type:" + value.getClass().getName()); + throw AddaxException.asAddaxException(NOT_SUPPORT_TYPE, "type:" + value.getClass().getName()); } return col; } diff --git a/plugin/reader/excelreader/src/main/java/com/wgzhao/addax/plugin/reader/excelreader/ExcelHelper.java b/plugin/reader/excelreader/src/main/java/com/wgzhao/addax/plugin/reader/excelreader/ExcelHelper.java index f8d24df3a..4d78bef56 100644 --- a/plugin/reader/excelreader/src/main/java/com/wgzhao/addax/plugin/reader/excelreader/ExcelHelper.java +++ b/plugin/reader/excelreader/src/main/java/com/wgzhao/addax/plugin/reader/excelreader/ExcelHelper.java @@ -42,6 +42,8 @@ import java.io.IOException; import java.util.Iterator; +import static com.wgzhao.addax.common.spi.ErrorCode.IO_ERROR; + public class ExcelHelper { public boolean header; @@ -81,10 +83,10 @@ public void open(String filePath) } } catch (FileNotFoundException e) { - throw AddaxException.asAddaxException(ExcelReaderErrorCode.OPEN_FILE_ERROR, e); + throw AddaxException.asAddaxException(IO_ERROR, e); } catch (IOException e) { - throw AddaxException.asAddaxException(ExcelReaderErrorCode.OPEN_FILE_ERROR, + throw AddaxException.asAddaxException(IO_ERROR, "IOException occurred when open '" + filePath + "':" + e.getMessage()); } } diff --git a/plugin/reader/excelreader/src/main/java/com/wgzhao/addax/plugin/reader/excelreader/ExcelReader.java b/plugin/reader/excelreader/src/main/java/com/wgzhao/addax/plugin/reader/excelreader/ExcelReader.java index 4bed57a11..b938827d7 100644 --- a/plugin/reader/excelreader/src/main/java/com/wgzhao/addax/plugin/reader/excelreader/ExcelReader.java +++ b/plugin/reader/excelreader/src/main/java/com/wgzhao/addax/plugin/reader/excelreader/ExcelReader.java @@ -35,6 +35,9 @@ import java.util.ArrayList; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; + public class ExcelReader extends Reader { @@ -53,9 +56,9 @@ public void init() { this.originConfig = this.getPluginJobConf(); // Compatible with the old version, path is a string before - String pathInString = this.originConfig.getNecessaryValue(Key.PATH, ExcelReaderErrorCode.REQUIRED_VALUE); + String pathInString = this.originConfig.getNecessaryValue(Key.PATH, REQUIRED_VALUE); if (StringUtils.isBlank(pathInString)) { - throw AddaxException.asAddaxException(ExcelReaderErrorCode.REQUIRED_VALUE, "the path is required"); + throw AddaxException.asAddaxException(REQUIRED_VALUE, "the path is required"); } if (!pathInString.startsWith("[") && !pathInString.endsWith("]")) { path = new ArrayList<>(); @@ -64,7 +67,7 @@ public void init() else { path = this.originConfig.getList(Key.PATH, String.class); if (null == path || path.isEmpty()) { - throw AddaxException.asAddaxException(ExcelReaderErrorCode.REQUIRED_VALUE, "the path is required"); + throw AddaxException.asAddaxException(REQUIRED_VALUE, "the path is required"); } } @@ -90,7 +93,7 @@ public List split(int adviceNumber) int splitNumber = this.sourceFiles.size(); if (0 == splitNumber) { throw AddaxException.asAddaxException( - ExcelReaderErrorCode.EMPTY_DIR_EXCEPTION, + RUNTIME_ERROR, "Nothing found in the directory " + this.originConfig.getString(Key.PATH) + ". Please check it"); } diff --git a/plugin/reader/excelreader/src/main/java/com/wgzhao/addax/plugin/reader/excelreader/ExcelReaderErrorCode.java b/plugin/reader/excelreader/src/main/java/com/wgzhao/addax/plugin/reader/excelreader/ExcelReaderErrorCode.java deleted file mode 100644 index 229bfc54f..000000000 --- a/plugin/reader/excelreader/src/main/java/com/wgzhao/addax/plugin/reader/excelreader/ExcelReaderErrorCode.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.reader.excelreader; - -import com.wgzhao.addax.common.spi.ErrorCode; - -/** - * Created by haiwei.luo on 14-9-20. - */ -public enum ExcelReaderErrorCode - implements ErrorCode -{ - REQUIRED_VALUE("ExcelReader-00", "您缺失了必须填写的参数值."), - ILLEGAL_VALUE("ExcelReader-01", "您填写的参数值不合法."), - MIXED_INDEX_VALUE("ExcelReader-02", "您的列信息配置同时包含了index,value."), - NO_INDEX_VALUE("ExcelReader-03", "您明确的配置列信息,但未填写相应的index,value."), - FILE_NOT_EXISTS("ExcelReader-04", "您配置的目录文件路径不存在."), -// OPEN_FILE_WITH_CHARSET_ERROR("ExcelReader-05", "您配置的文件编码和实际文件编码不符合."), - OPEN_FILE_ERROR("ExcelReader-06", "您配置的文件在打开时异常,建议您检查源目录是否有隐藏文件,管道文件等特殊文件."), -// READ_FILE_IO_ERROR("ExcelReader-07", "您配置的文件在读取时出现IO异常."), - SECURITY_NOT_ENOUGH("ExcelReader-08", "您缺少权限执行相应的文件操作."), - CONFIG_INVALID_EXCEPTION("ExcelReader-09", "您的参数配置错误."), -// RUNTIME_EXCEPTION("ExcelReader-10", "出现运行时异常, 请联系我们"), - EMPTY_DIR_EXCEPTION("ExcelReader-11", "您尝试读取的文件目录为空."), - NOT_SUPPORT_TYPE("ExcelReader-12", "您配置的列类型暂不支持."), - OPEN_FILE_WITH_CHARSET_ERROR("ExcelReader-13", "您配置的编码和实际存储编码不符合."), - // OPEN_FILE_ERROR("UnstructuredStorageReader-08", "您配置的源在打开时异常,建议您检查源源是否有隐藏实体,管道文件等特殊文件."), - READ_FILE_IO_ERROR("ExcelReader-14", "您配置的文件在读取时出现IO异常."), - RUNTIME_EXCEPTION("ExcelReader-15", "出现运行时异常, 请联系我们") - ; - - private final String code; - private final String description; - - ExcelReaderErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s].", this.code, - this.description); - } -} diff --git a/plugin/reader/ftpreader/src/main/java/com/wgzhao/addax/plugin/reader/ftpreader/FtpReader.java b/plugin/reader/ftpreader/src/main/java/com/wgzhao/addax/plugin/reader/ftpreader/FtpReader.java index ac1db6f28..baa034316 100644 --- a/plugin/reader/ftpreader/src/main/java/com/wgzhao/addax/plugin/reader/ftpreader/FtpReader.java +++ b/plugin/reader/ftpreader/src/main/java/com/wgzhao/addax/plugin/reader/ftpreader/FtpReader.java @@ -34,6 +34,10 @@ import java.util.HashSet; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; + public class FtpReader extends Reader { @@ -88,14 +92,14 @@ else if ("ftp".equals(protocol)) { private void validateParameter() { - this.protocol = this.originConfig.getNecessaryValue(FtpKey.PROTOCOL, FtpReaderErrorCode.REQUIRED_VALUE).toLowerCase(); + this.protocol = this.originConfig.getNecessaryValue(FtpKey.PROTOCOL, REQUIRED_VALUE).toLowerCase(); boolean protocolTag = "ftp".equals(this.protocol) || "sftp".equals(this.protocol); if (!protocolTag) { - throw AddaxException.asAddaxException(FtpReaderErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, String.format("仅支持 ftp和sftp 传输协议 , 不支持您配置的传输协议: [%s]", protocol)); } - this.host = this.originConfig.getNecessaryValue(FtpKey.HOST, FtpReaderErrorCode.REQUIRED_VALUE); - this.username = this.originConfig.getNecessaryValue(FtpKey.USERNAME, FtpReaderErrorCode.REQUIRED_VALUE); + this.host = this.originConfig.getNecessaryValue(FtpKey.HOST, REQUIRED_VALUE); + this.username = this.originConfig.getNecessaryValue(FtpKey.USERNAME, REQUIRED_VALUE); this.password = this.originConfig.getString(FtpKey.PASSWORD, null); this.timeout = originConfig.getInt(FtpKey.TIME_OUT, FtpConstant.DEFAULT_TIMEOUT); this.maxTraversalLevel = originConfig.getInt(FtpKey.MAX_TRAVERSAL_LEVEL, FtpConstant.DEFAULT_MAX_TRAVERSAL_LEVEL); @@ -104,7 +108,7 @@ private void validateParameter() this.connectPattern = this.originConfig.getUnnecessaryValue(FtpKey.CONNECT_PATTERN, FtpConstant.DEFAULT_FTP_CONNECT_PATTERN); boolean connectPatternTag = "PORT".equals(connectPattern) || "PASV".equals(connectPattern); if (!connectPatternTag) { - throw AddaxException.asAddaxException(FtpReaderErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, String.format("不支持您配置的ftp传输模式: [%s]", connectPattern)); } else { @@ -112,7 +116,7 @@ private void validateParameter() } //path check - String pathInString = this.originConfig.getNecessaryValue(Key.PATH, FtpReaderErrorCode.REQUIRED_VALUE); + String pathInString = this.originConfig.getNecessaryValue(Key.PATH, REQUIRED_VALUE); if (!pathInString.startsWith("[") && !pathInString.endsWith("]")) { path = new ArrayList<>(); path.add(pathInString); @@ -120,13 +124,13 @@ private void validateParameter() else { path = this.originConfig.getList(Key.PATH, String.class); if (null == path || path.isEmpty()) { - throw AddaxException.asAddaxException(FtpReaderErrorCode.REQUIRED_VALUE, "您需要指定待读取的源目录或文件"); + throw AddaxException.asAddaxException(REQUIRED_VALUE, "您需要指定待读取的源目录或文件"); } for (String eachPath : path) { if (!eachPath.startsWith("/")) { String message = String.format("请检查参数path:[%s],需要配置为绝对路径", eachPath); LOG.error(message); - throw AddaxException.asAddaxException(FtpReaderErrorCode.ILLEGAL_VALUE, message); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, message); } } } @@ -147,7 +151,7 @@ private void validateParameter() else { String msg = "You have configured to use the key, but neither the configured key file nor the default file(" + DEFAULT_PRIVATE_KEY + " exists"; - throw AddaxException.asAddaxException(FtpReaderErrorCode.ILLEGAL_VALUE, msg); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, msg); } } } @@ -195,7 +199,7 @@ public List split(int adviceNumber) // int splitNumber = adviceNumber; int splitNumber = this.sourceFiles.size(); if (0 == splitNumber) { - throw AddaxException.asAddaxException(FtpReaderErrorCode.EMPTY_DIR_EXCEPTION, + throw AddaxException.asAddaxException(CONFIG_ERROR, String.format("未能找到待读取的文件,请确认您的配置项path: %s", this.originConfig.getString(Key.PATH))); } diff --git a/plugin/reader/ftpreader/src/main/java/com/wgzhao/addax/plugin/reader/ftpreader/FtpReaderErrorCode.java b/plugin/reader/ftpreader/src/main/java/com/wgzhao/addax/plugin/reader/ftpreader/FtpReaderErrorCode.java deleted file mode 100644 index 8b9d6b387..000000000 --- a/plugin/reader/ftpreader/src/main/java/com/wgzhao/addax/plugin/reader/ftpreader/FtpReaderErrorCode.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.reader.ftpreader; - -import com.wgzhao.addax.common.spi.ErrorCode; - -/** - * Created by haiwei.luo on 14-9-20. - */ -public enum FtpReaderErrorCode - implements ErrorCode -{ - REQUIRED_VALUE("FtpReader-00", "您缺失了必须填写的参数值."), - ILLEGAL_VALUE("FtpReader-01", "您填写的参数值不合法."), -// MIXED_INDEX_VALUE("FtpReader-02", "您的列信息配置同时包含了index,value."), -// NO_INDEX_VALUE("FtpReader-03", "您明确的配置列信息,但未填写相应的index,value."), - - FILE_NOT_EXISTS("FtpReader-04", "您配置的目录文件路径不存在或者没有权限读取."), -// OPEN_FILE_WITH_CHARSET_ERROR("FtpReader-05", "您配置的文件编码和实际文件编码不符合."), - OPEN_FILE_ERROR("FtpReader-06", "您配置的文件在打开时异常."), -// READ_FILE_IO_ERROR("FtpReader-07", "您配置的文件在读取时出现IO异常."), -// SECURITY_NOT_ENOUGH("FtpReader-08", "您缺少权限执行相应的文件操作."), -// CONFIG_INVALID_EXCEPTION("FtpReader-09", "您的参数配置错误."), -// RUNTIME_EXCEPTION("FtpReader-10", "出现运行时异常, 请联系我们"), - EMPTY_DIR_EXCEPTION("FtpReader-11", "您尝试读取的文件目录为空."), - - FAIL_LOGIN("FtpReader-12", "登录失败,无法与ftp服务器建立连接."), - FAIL_DISCONNECT("FtpReader-13", "关闭ftp连接失败,无法与ftp服务器断开连接."), - COMMAND_FTP_IO_EXCEPTION("FtpReader-14", "与ftp服务器连接异常."), - OUT_MAX_DIRECTORY_LEVEL("FtpReader-15", "超出允许的最大目录层数."), - LINK_FILE("FtpReader-16", "您尝试读取的文件为链接文件."), - ; - - private final String code; - private final String description; - - FtpReaderErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s].", this.code, - this.description); - } -} diff --git a/plugin/reader/ftpreader/src/main/java/com/wgzhao/addax/plugin/reader/ftpreader/SftpHelper.java b/plugin/reader/ftpreader/src/main/java/com/wgzhao/addax/plugin/reader/ftpreader/SftpHelper.java index 74a3a07dd..e5412b952 100644 --- a/plugin/reader/ftpreader/src/main/java/com/wgzhao/addax/plugin/reader/ftpreader/SftpHelper.java +++ b/plugin/reader/ftpreader/src/main/java/com/wgzhao/addax/plugin/reader/ftpreader/SftpHelper.java @@ -37,6 +37,12 @@ import java.util.HashSet; import java.util.Properties; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.CONNECT_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.IO_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; + public class SftpHelper extends FtpHelper { @@ -61,7 +67,7 @@ public void loginFtpServer(String host, String username, String password, int po } } catch (JSchException e) { - throw AddaxException.asAddaxException(FtpReaderErrorCode.ILLEGAL_VALUE, "Failed to use private key", e); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "Failed to use private key", e); } } try { @@ -69,7 +75,7 @@ public void loginFtpServer(String host, String username, String password, int po // 根据用户名，主机ip，端口获取一个Session对象 // 如果服务器连接不上，则抛出异常 if (session == null) { - throw AddaxException.asAddaxException(FtpReaderErrorCode.FAIL_LOGIN, + throw AddaxException.asAddaxException(CONNECT_ERROR, "session is null,无法通过sftp与服务器建立链接，请检查主机名和用户名是否正确."); } @@ -98,12 +104,12 @@ public void loginFtpServer(String host, String username, String password, int po if (unknownHostException.equals(cause)) { String message = String.format("请确认ftp服务器地址是否正确，无法连接到地址为: [%s] 的ftp服务器", host); LOG.error(message); - throw AddaxException.asAddaxException(FtpReaderErrorCode.FAIL_LOGIN, message, e); + throw AddaxException.asAddaxException(CONNECT_ERROR, message, e); } else if (illegalArgumentException.equals(cause) || wrongPort.equals(cause)) { String message = String.format("请确认连接ftp服务器端口是否正确，错误的端口: [%s] ", port); LOG.error(message); - throw AddaxException.asAddaxException(FtpReaderErrorCode.FAIL_LOGIN, message, e); + throw AddaxException.asAddaxException(CONNECT_ERROR, message, e); } } else { @@ -111,13 +117,13 @@ else if (illegalArgumentException.equals(cause) || wrongPort.equals(cause)) { String message = String.format("与ftp服务器建立连接失败,请检查用户名和密码是否正确: [%s]", "message:host =" + host + ",username = " + username + ",port =" + port); LOG.error(message); - throw AddaxException.asAddaxException(FtpReaderErrorCode.FAIL_LOGIN, message); + throw AddaxException.asAddaxException(CONNECT_ERROR, message); } else { String message = String.format("与ftp服务器建立连接失败 : [%s]", "message:host =" + host + ",username = " + username + ",port =" + port); LOG.error(message); - throw AddaxException.asAddaxException(FtpReaderErrorCode.FAIL_LOGIN, message, e); + throw AddaxException.asAddaxException(CONNECT_ERROR, message, e); } } } @@ -145,11 +151,11 @@ public boolean isDirExist(String directoryPath) if (e.getMessage().equalsIgnoreCase("no such file")) { String message = String.format("请确认您的配置项path:[%s]存在，且配置的用户有权限读取", directoryPath); LOG.error(message); - throw AddaxException.asAddaxException(FtpReaderErrorCode.FILE_NOT_EXISTS, message); + throw AddaxException.asAddaxException(CONFIG_ERROR, message); } String message = String.format("进入目录：[%s]时发生I/O异常,请确认与ftp服务器的连接正常", directoryPath); LOG.error(message); - throw AddaxException.asAddaxException(FtpReaderErrorCode.COMMAND_FTP_IO_EXCEPTION, message, e); + throw AddaxException.asAddaxException(IO_ERROR, message, e); } } @@ -167,12 +173,12 @@ public boolean isFileExist(String filePath) if (e.getMessage().equalsIgnoreCase("no such file")) { String message = String.format("请确认您的配置项path:[%s]存在，且配置的用户有权限读取", filePath); LOG.error(message); - throw AddaxException.asAddaxException(FtpReaderErrorCode.FILE_NOT_EXISTS, message); + throw AddaxException.asAddaxException(CONFIG_ERROR, message); } else { String message = String.format("获取文件：[%s] 属性时发生I/O异常,请确认与ftp服务器的连接正常", filePath); LOG.error(message); - throw AddaxException.asAddaxException(FtpReaderErrorCode.COMMAND_FTP_IO_EXCEPTION, message, e); + throw AddaxException.asAddaxException(IO_ERROR, message, e); } } return isExitFlag; @@ -189,12 +195,12 @@ public boolean isSymbolicLink(String filePath) if (e.getMessage().equalsIgnoreCase("no such file")) { String message = String.format("请确认您的配置项path:[%s]存在，且配置的用户有权限读取", filePath); LOG.error(message); - throw AddaxException.asAddaxException(FtpReaderErrorCode.FILE_NOT_EXISTS, message); + throw AddaxException.asAddaxException(CONFIG_ERROR, message); } else { String message = String.format("获取文件：[%s] 属性时发生I/O异常,请确认与ftp服务器的连接正常", filePath); LOG.error(message); - throw AddaxException.asAddaxException(FtpReaderErrorCode.COMMAND_FTP_IO_EXCEPTION, message, e); + throw AddaxException.asAddaxException(IO_ERROR, message, e); } } } @@ -215,7 +221,7 @@ public HashSet getListFiles(String directoryPath, int parentLevel, int m String message = String.format("不能进入目录：[%s]," + "请确认您的配置项path:[%s]存在，且配置的用户有权限进入", subPath, directoryPath); LOG.error(message); - throw AddaxException.asAddaxException(FtpReaderErrorCode.FILE_NOT_EXISTS, message); + throw AddaxException.asAddaxException(CONFIG_ERROR, message); } } else if (isDirExist(directoryPath)) { @@ -231,7 +237,7 @@ else if (isSymbolicLink(directoryPath)) { //path是链接文件 String message = String.format("文件:[%s]是链接文件，当前不支持链接文件的读取", directoryPath); LOG.error(message); - throw AddaxException.asAddaxException(FtpReaderErrorCode.LINK_FILE, message); + throw AddaxException.asAddaxException(CONFIG_ERROR, message); } else if (isFileExist(directoryPath)) { // path指向具体文件 @@ -241,7 +247,7 @@ else if (isFileExist(directoryPath)) { else { String message = String.format("请确认您的配置项path:[%s]存在，且配置的用户有权限读取", directoryPath); LOG.error(message); - throw AddaxException.asAddaxException(FtpReaderErrorCode.FILE_NOT_EXISTS, message); + throw AddaxException.asAddaxException(CONFIG_ERROR, message); } try { @@ -262,7 +268,7 @@ else if (isSymbolicLink(filePath)) { //是链接文件 String message = String.format("文件:[%s]是链接文件，当前不支持链接文件的读取", filePath); LOG.error(message); - throw AddaxException.asAddaxException(FtpReaderErrorCode.LINK_FILE, message); + throw AddaxException.asAddaxException(CONFIG_ERROR, message); } else if (isFileExist(filePath)) { // 是文件 @@ -271,14 +277,14 @@ else if (isFileExist(filePath)) { else { String message = String.format("请确认path:[%s]存在，且配置的用户有权限读取", filePath); LOG.error(message); - throw AddaxException.asAddaxException(FtpReaderErrorCode.FILE_NOT_EXISTS, message); + throw AddaxException.asAddaxException(CONFIG_ERROR, message); } } // end for vector } catch (SftpException e) { String message = String.format("获取path：[%s] 下文件列表时发生I/O异常,请确认与ftp服务器的连接正常", directoryPath); LOG.error(message); - throw AddaxException.asAddaxException(FtpReaderErrorCode.COMMAND_FTP_IO_EXCEPTION, message, e); + throw AddaxException.asAddaxException(IO_ERROR, message, e); } return sourceFiles; @@ -287,7 +293,7 @@ else if (isFileExist(filePath)) { //超出最大递归层数 String message = String.format("获取path：[%s] 下文件列表时超出最大层数,请确认路径[%s]下不存在软连接文件", directoryPath, directoryPath); LOG.error(message); - throw AddaxException.asAddaxException(FtpReaderErrorCode.OUT_MAX_DIRECTORY_LEVEL, message); + throw AddaxException.asAddaxException(RUNTIME_ERROR, message); } } @@ -300,7 +306,7 @@ public InputStream getInputStream(String filePath) catch (SftpException e) { String message = String.format("读取文件 : [%s] 时出错,请确认文件：[%s]存在且配置的用户有权限读取", filePath, filePath); LOG.error(message); - throw AddaxException.asAddaxException(FtpReaderErrorCode.OPEN_FILE_ERROR, message); + throw AddaxException.asAddaxException(CONFIG_ERROR, message); } } } diff --git a/plugin/reader/ftpreader/src/main/java/com/wgzhao/addax/plugin/reader/ftpreader/StandardFtpHelper.java b/plugin/reader/ftpreader/src/main/java/com/wgzhao/addax/plugin/reader/ftpreader/StandardFtpHelper.java index 9302fa1b9..4f935fa82 100644 --- a/plugin/reader/ftpreader/src/main/java/com/wgzhao/addax/plugin/reader/ftpreader/StandardFtpHelper.java +++ b/plugin/reader/ftpreader/src/main/java/com/wgzhao/addax/plugin/reader/ftpreader/StandardFtpHelper.java @@ -34,6 +34,11 @@ import java.nio.charset.StandardCharsets; import java.util.HashSet; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.CONNECT_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.IO_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; import static org.apache.commons.net.ftp.FTP.BINARY_FILE_TYPE; public class StandardFtpHelper @@ -71,7 +76,7 @@ else if ("PORT".equals(connectMode)) { String message = String.format("与ftp服务器建立连接失败,请检查用户名和密码是否正确: [%s]", "message:host =" + host + ",username = " + username + ",port =" + port); LOG.error(message); - throw AddaxException.asAddaxException(FtpReaderErrorCode.FAIL_LOGIN, message); + throw AddaxException.asAddaxException(CONNECT_ERROR, message); } //设置命令传输编码 String fileEncoding = System.getProperty("file.encoding"); @@ -82,18 +87,18 @@ else if ("PORT".equals(connectMode)) { catch (UnknownHostException e) { String message = String.format("请确认ftp服务器地址是否正确，无法连接到地址为: [%s] 的ftp服务器", host); LOG.error(message); - throw AddaxException.asAddaxException(FtpReaderErrorCode.FAIL_LOGIN, message, e); + throw AddaxException.asAddaxException(CONNECT_ERROR, message, e); } catch (IllegalArgumentException e) { String message = String.format("请确认连接ftp服务器端口是否正确，错误的端口: [%s] ", port); LOG.error(message); - throw AddaxException.asAddaxException(FtpReaderErrorCode.FAIL_LOGIN, message, e); + throw AddaxException.asAddaxException(CONNECT_ERROR, message, e); } catch (Exception e) { String message = String.format("与ftp服务器建立连接失败 : [%s]", "message:host =" + host + ",username = " + username + ",port =" + port); LOG.error(message); - throw AddaxException.asAddaxException(FtpReaderErrorCode.FAIL_LOGIN, message, e); + throw AddaxException.asAddaxException(CONNECT_ERROR, message, e); } } @@ -108,7 +113,7 @@ public void logoutFtpServer() catch (IOException e) { String message = "与ftp服务器断开连接失败"; LOG.error(message); - throw AddaxException.asAddaxException(FtpReaderErrorCode.FAIL_DISCONNECT, message, e); + throw AddaxException.asAddaxException(CONNECT_ERROR, message, e); } } } @@ -122,7 +127,7 @@ public boolean isDirExist(String directoryPath) catch (IOException e) { String message = String.format("进入目录：[%s]时发生I/O异常,请确认与ftp服务器的连接正常", directoryPath); LOG.error(message); - throw AddaxException.asAddaxException(FtpReaderErrorCode.COMMAND_FTP_IO_EXCEPTION, message, e); + throw AddaxException.asAddaxException(IO_ERROR, message, e); } } @@ -139,7 +144,7 @@ public boolean isFileExist(String filePath) catch (IOException e) { String message = String.format("获取文件：[%s] 属性时发生I/O异常,请确认与ftp服务器的连接正常", filePath); LOG.error(message); - throw AddaxException.asAddaxException(FtpReaderErrorCode.COMMAND_FTP_IO_EXCEPTION, message, e); + throw AddaxException.asAddaxException(IO_ERROR, message, e); } return isExitFlag; } @@ -157,7 +162,7 @@ public boolean isSymbolicLink(String filePath) catch (IOException e) { String message = String.format("获取文件：[%s] 属性时发生I/O异常,请确认与ftp服务器的连接正常", filePath); LOG.error(message); - throw AddaxException.asAddaxException(FtpReaderErrorCode.COMMAND_FTP_IO_EXCEPTION, message, e); + throw AddaxException.asAddaxException(IO_ERROR, message, e); } return isExitFlag; } @@ -178,7 +183,7 @@ public HashSet getListFiles(String directoryPath, int parentLevel, int m String message = String.format("不能进入目录：[%s]," + "请确认您的配置项path:[%s]存在，且配置的用户有权限进入", subPath, directoryPath); LOG.error(message); - throw AddaxException.asAddaxException(FtpReaderErrorCode.FILE_NOT_EXISTS, message); + throw AddaxException.asAddaxException(CONFIG_ERROR, message); } } else if (isDirExist(directoryPath)) { @@ -199,12 +204,12 @@ else if (isSymbolicLink(directoryPath)) { //path是链接文件 String message = String.format("文件:[%s]是链接文件，当前不支持链接文件的读取", directoryPath); LOG.error(message); - throw AddaxException.asAddaxException(FtpReaderErrorCode.LINK_FILE, message); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, message); } else { String message = String.format("请确认您的配置项path:[%s]存在，且配置的用户有权限读取", directoryPath); LOG.error(message); - throw AddaxException.asAddaxException(FtpReaderErrorCode.FILE_NOT_EXISTS, message); + throw AddaxException.asAddaxException(CONNECT_ERROR, message); } try { @@ -226,19 +231,19 @@ else if (ff.isSymbolicLink()) { //是链接文件 String message = String.format("文件:[%s]是链接文件，当前不支持链接文件的读取", filePath); LOG.error(message); - throw AddaxException.asAddaxException(FtpReaderErrorCode.LINK_FILE, message); + throw AddaxException.asAddaxException(CONFIG_ERROR, message); } else { String message = String.format("请确认path:[%s]存在，且配置的用户有权限读取", filePath); LOG.error(message); - throw AddaxException.asAddaxException(FtpReaderErrorCode.FILE_NOT_EXISTS, message); + throw AddaxException.asAddaxException(CONNECT_ERROR, message); } } // end for FTPFile } catch (IOException e) { String message = String.format("获取path：[%s] 下文件列表时发生I/O异常,请确认与ftp服务器的连接正常", directoryPath); LOG.error(message); - throw AddaxException.asAddaxException(FtpReaderErrorCode.COMMAND_FTP_IO_EXCEPTION, message, e); + throw AddaxException.asAddaxException(IO_ERROR, message, e); } return sourceFiles; } @@ -246,7 +251,7 @@ else if (ff.isSymbolicLink()) { //超出最大递归层数 String message = String.format("获取path：[%s] 下文件列表时超出最大层数,请确认路径[%s]下不存在软连接文件", directoryPath, directoryPath); LOG.error(message); - throw AddaxException.asAddaxException(FtpReaderErrorCode.OUT_MAX_DIRECTORY_LEVEL, message); + throw AddaxException.asAddaxException(RUNTIME_ERROR, message); } } @@ -259,7 +264,7 @@ public InputStream getInputStream(String filePath) catch (IOException e) { String message = String.format("读取文件 : [%s] 时出错,请确认文件：[%s]存在且配置的用户有权限读取", filePath, filePath); LOG.error(message); - throw AddaxException.asAddaxException(FtpReaderErrorCode.OPEN_FILE_ERROR, message); + throw AddaxException.asAddaxException(IO_ERROR, message); } } } diff --git a/plugin/reader/hbase11xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase11xreader/ColumnType.java b/plugin/reader/hbase11xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase11xreader/ColumnType.java index e363fec68..2b888b41f 100644 --- a/plugin/reader/hbase11xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase11xreader/ColumnType.java +++ b/plugin/reader/hbase11xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase11xreader/ColumnType.java @@ -24,6 +24,8 @@ import java.util.Arrays; +import static com.wgzhao.addax.common.spi.ErrorCode.NOT_SUPPORT_TYPE; + /** * 只对 normal 模式读取时有用，多版本读取时，不存在列类型的 */ @@ -49,7 +51,7 @@ public enum ColumnType public static ColumnType getByTypeName(String typeName) { if (StringUtils.isBlank(typeName)) { - throw AddaxException.asAddaxException(Hbase11xReaderErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(NOT_SUPPORT_TYPE, String.format("Hbasereader 不支持该类型:%s, 目前支持的类型是:%s", typeName, Arrays.asList(values()))); } for (ColumnType columnType : values()) { @@ -58,7 +60,7 @@ public static ColumnType getByTypeName(String typeName) } } - throw AddaxException.asAddaxException(Hbase11xReaderErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(NOT_SUPPORT_TYPE, String.format("Hbasereader 不支持该类型:%s, 目前支持的类型是:%s", typeName, Arrays.asList(values()))); } diff --git a/plugin/reader/hbase11xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase11xreader/Hbase11xHelper.java b/plugin/reader/hbase11xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase11xreader/Hbase11xHelper.java index f2aec7b0c..f7a0d7fff 100644 --- a/plugin/reader/hbase11xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase11xreader/Hbase11xHelper.java +++ b/plugin/reader/hbase11xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase11xreader/Hbase11xHelper.java @@ -48,6 +48,12 @@ import java.util.List; import java.util.Map; +import static com.wgzhao.addax.common.spi.ErrorCode.CONNECT_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.EXECUTE_FAIL; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; + /** * 工具类 * Created by shf on 16/3/7. @@ -66,27 +72,27 @@ public static org.apache.hadoop.hbase.client.Connection getHbaseConnection(Strin return hConnection; } if (StringUtils.isBlank(hbaseConfig)) { - throw AddaxException.asAddaxException(Hbase11xReaderErrorCode.REQUIRED_VALUE, "读 Hbase 时需要配置hbaseConfig，其内容为 Hbase 连接信息，请联系 Hbase PE 获取该信息."); + throw AddaxException.asAddaxException(REQUIRED_VALUE, "读 Hbase 时需要配置hbaseConfig，其内容为 Hbase 连接信息，请联系 Hbase PE 获取该信息."); } org.apache.hadoop.conf.Configuration hConfiguration = HBaseConfiguration.create(); try { Map hbaseConfigMap = JSON.parseObject(hbaseConfig, new TypeReference>() {}); // 用户配置的 key-value 对 来表示 hbaseConfig - Validate.isTrue(hbaseConfigMap != null && hbaseConfigMap.size() != 0, "hbaseConfig不能为空Map结构!"); + Validate.isTrue(hbaseConfigMap != null && !hbaseConfigMap.isEmpty(), "hbaseConfig不能为空Map结构!"); for (Map.Entry entry : hbaseConfigMap.entrySet()) //NOSONAR { hConfiguration.set(entry.getKey(), entry.getValue()); } } catch (Exception e) { - throw AddaxException.asAddaxException(Hbase11xReaderErrorCode.GET_HBASE_CONNECTION_ERROR, e); + throw AddaxException.asAddaxException(CONNECT_ERROR, e); } try { hConnection = ConnectionFactory.createConnection(hConfiguration); } catch (Exception e) { Hbase11xHelper.closeConnection(hConnection); - throw AddaxException.asAddaxException(Hbase11xReaderErrorCode.GET_HBASE_CONNECTION_ERROR, e); + throw AddaxException.asAddaxException(CONNECT_ERROR, e); } return hConnection; } @@ -108,7 +114,7 @@ public static Table getTable(Configuration configuration) Hbase11xHelper.closeTable(hTable); Hbase11xHelper.closeAdmin(admin); Hbase11xHelper.closeConnection(hConnection); - throw AddaxException.asAddaxException(Hbase11xReaderErrorCode.GET_HBASE_TABLE_ERROR, e); + throw AddaxException.asAddaxException(RUNTIME_ERROR, e); } return hTable; } @@ -130,7 +136,7 @@ public static RegionLocator getRegionLocator(Configuration configuration) Hbase11xHelper.closeRegionLocator(regionLocator); Hbase11xHelper.closeAdmin(admin); Hbase11xHelper.closeConnection(hConnection); - throw AddaxException.asAddaxException(Hbase11xReaderErrorCode.GET_HBASE_REGINLOCTOR_ERROR, e); + throw AddaxException.asAddaxException(RUNTIME_ERROR, e); } return regionLocator; } @@ -143,7 +149,7 @@ public static synchronized void closeConnection(Connection hConnection) } } catch (IOException e) { - throw AddaxException.asAddaxException(Hbase11xReaderErrorCode.CLOSE_HBASE_CONNECTION_ERROR, e); + throw AddaxException.asAddaxException(RUNTIME_ERROR, e); } } @@ -155,7 +161,7 @@ public static void closeAdmin(Admin admin) } } catch (IOException e) { - throw AddaxException.asAddaxException(Hbase11xReaderErrorCode.CLOSE_HBASE_ADMIN_ERROR, e); + throw AddaxException.asAddaxException(RUNTIME_ERROR, e); } } @@ -167,7 +173,7 @@ public static void closeTable(Table table) } } catch (IOException e) { - throw AddaxException.asAddaxException(Hbase11xReaderErrorCode.CLOSE_HBASE_TABLE_ERROR, e); + throw AddaxException.asAddaxException(RUNTIME_ERROR, e); } } @@ -186,7 +192,7 @@ public static void closeRegionLocator(RegionLocator regionLocator) } } catch (IOException e) { - throw AddaxException.asAddaxException(Hbase11xReaderErrorCode.CLOSE_HBASE_REGINLOCTOR_ERROR, e); + throw AddaxException.asAddaxException(RUNTIME_ERROR, e); } } @@ -194,15 +200,15 @@ public static void checkHbaseTable(Admin admin, TableName hTableName) throws IOException { if (!admin.tableExists(hTableName)) { - throw AddaxException.asAddaxException(Hbase11xReaderErrorCode.ILLEGAL_VALUE, "HBase源头表" + hTableName.toString() + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "HBase源头表" + hTableName.toString() + "不存在, 请检查您的配置 或者 联系 Hbase 管理员."); } if (!admin.isTableAvailable(hTableName)) { - throw AddaxException.asAddaxException(Hbase11xReaderErrorCode.ILLEGAL_VALUE, "HBase源头表" + hTableName.toString() + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "HBase源头表" + hTableName.toString() + " 不可用, 请检查您的配置 或者 联系 Hbase 管理员."); } if (admin.isTableDisabled(hTableName)) { - throw AddaxException.asAddaxException(Hbase11xReaderErrorCode.ILLEGAL_VALUE, "HBase源头表" + hTableName.toString() + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "HBase源头表" + hTableName.toString() + "is disabled, 请检查您的配置 或者 联系 Hbase 管理员."); } } @@ -336,7 +342,7 @@ public static HashMap> parseColumnOfMultiVersion if (!Hbase11xHelper.isRowkeyColumn(columnName)) { String[] cfAndQualifier = columnName.split(":"); if (cfAndQualifier.length != 2) { - throw AddaxException.asAddaxException(Hbase11xReaderErrorCode.ILLEGAL_VALUE, "Hbasereader 中，column 的列配置格式应该是：列族:列名. 您配置的列错误：" + columnName); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "Hbasereader 中，column 的列配置格式应该是：列族:列名. 您配置的列错误：" + columnName); } familyQualifier = StringUtils.join(cfAndQualifier[0].trim(), ":", cfAndQualifier[1].trim()); } @@ -360,14 +366,14 @@ public static List split(Configuration configuration) /* 如果用户配置了 startRowkey 和 endRowkey，需要确保：startRowkey <= endRowkey */ if (startRowkeyByte.length != 0 && endRowkeyByte.length != 0 && Bytes.compareTo(startRowkeyByte, endRowkeyByte) > 0) { - throw AddaxException.asAddaxException(Hbase11xReaderErrorCode.ILLEGAL_VALUE, "Hbasereader 中 startRowkey 不得大于 endRowkey."); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "Hbasereader 中 startRowkey 不得大于 endRowkey."); } RegionLocator regionLocator = Hbase11xHelper.getRegionLocator(configuration); List resultConfigurations; try { Pair regionRanges = regionLocator.getStartEndKeys(); if (null == regionRanges) { - throw AddaxException.asAddaxException(Hbase11xReaderErrorCode.SPLIT_ERROR, "获取源头 Hbase 表的 rowkey 范围失败."); + throw AddaxException.asAddaxException(EXECUTE_FAIL, "获取源头 Hbase 表的 rowkey 范围失败."); } resultConfigurations = Hbase11xHelper.doSplit(configuration, startRowkeyByte, endRowkeyByte, regionRanges); @@ -376,7 +382,7 @@ public static List split(Configuration configuration) return resultConfigurations; } catch (Exception e) { - throw AddaxException.asAddaxException(Hbase11xReaderErrorCode.SPLIT_ERROR, "切分源头 Hbase 表失败.", e); + throw AddaxException.asAddaxException(EXECUTE_FAIL, "切分源头 Hbase 表失败.", e); } finally { Hbase11xHelper.closeRegionLocator(regionLocator); @@ -481,15 +487,15 @@ private static String getStartKey(byte[] startRowkeyByte, byte[] regionStarKey) public static void validateParameter(Configuration originalConfig) { - originalConfig.getNecessaryValue(HBaseKey.HBASE_CONFIG, Hbase11xReaderErrorCode.REQUIRED_VALUE); - originalConfig.getNecessaryValue(HBaseKey.TABLE, Hbase11xReaderErrorCode.REQUIRED_VALUE); + originalConfig.getNecessaryValue(HBaseKey.HBASE_CONFIG, REQUIRED_VALUE); + originalConfig.getNecessaryValue(HBaseKey.TABLE, REQUIRED_VALUE); Hbase11xHelper.validateMode(originalConfig); //非必选参数处理 String encoding = originalConfig.getString(HBaseKey.ENCODING, HBaseConstant.DEFAULT_ENCODING); if (!Charset.isSupported(encoding)) { - throw AddaxException.asAddaxException(Hbase11xReaderErrorCode.ILLEGAL_VALUE, String.format("Hbasereader 不支持您所配置的编码:[%s]", encoding)); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, String.format("Hbasereader 不支持您所配置的编码:[%s]", encoding)); } originalConfig.set(HBaseKey.ENCODING, encoding); // 处理 range 的配置 @@ -518,10 +524,10 @@ public static void validateParameter(Configuration originalConfig) private static void validateMode(Configuration originalConfig) { - String mode = originalConfig.getNecessaryValue(HBaseKey.MODE, Hbase11xReaderErrorCode.REQUIRED_VALUE); + String mode = originalConfig.getNecessaryValue(HBaseKey.MODE, REQUIRED_VALUE); List column = originalConfig.getList(HBaseKey.COLUMN, Map.class); if (column == null || column.isEmpty()) { - throw AddaxException.asAddaxException(Hbase11xReaderErrorCode.REQUIRED_VALUE, "您配置的column为空,Hbase必须配置 column，其形式为：column:[{\"name\": \"cf0:column0\",\"type\": \"string\"},{\"name\": \"cf1:column1\",\"type\": \"long\"}]"); + throw AddaxException.asAddaxException(REQUIRED_VALUE, "您配置的column为空,Hbase必须配置 column，其形式为：column:[{\"name\": \"cf0:column0\",\"type\": \"string\"},{\"name\": \"cf1:column1\",\"type\": \"long\"}]"); } ModeType modeType = ModeType.getByTypeName(mode); switch (modeType) { @@ -541,7 +547,7 @@ private static void validateMode(Configuration originalConfig) break; } default: - throw AddaxException.asAddaxException(Hbase11xReaderErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, String.format("HbaseReader不支持该 mode 类型:%s", mode)); } } diff --git a/plugin/reader/hbase11xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase11xreader/Hbase11xReader.java b/plugin/reader/hbase11xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase11xreader/Hbase11xReader.java index 978bc2725..9c1557e82 100644 --- a/plugin/reader/hbase11xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase11xreader/Hbase11xReader.java +++ b/plugin/reader/hbase11xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase11xreader/Hbase11xReader.java @@ -30,6 +30,9 @@ import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.IO_ERROR; + /** * Hbase11xReader * Created by shf on 16/3/7. @@ -83,7 +86,7 @@ public void init() this.hbaseTaskProxy = new MultiVersionFixedColumnTask(taskConfig); break; default: - throw AddaxException.asAddaxException(Hbase11xReaderErrorCode.ILLEGAL_VALUE, "Hbasereader 不支持此类模式:" + modeType); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "Hbasereader 不支持此类模式:" + modeType); } } @@ -94,7 +97,7 @@ public void prepare() this.hbaseTaskProxy.prepare(); } catch (Exception e) { - throw AddaxException.asAddaxException(Hbase11xReaderErrorCode.PREPARE_READ_ERROR, e); + throw AddaxException.asAddaxException(IO_ERROR, e); } } diff --git a/plugin/reader/hbase11xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase11xreader/Hbase11xReaderErrorCode.java b/plugin/reader/hbase11xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase11xreader/Hbase11xReaderErrorCode.java deleted file mode 100644 index 65a76917a..000000000 --- a/plugin/reader/hbase11xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase11xreader/Hbase11xReaderErrorCode.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.reader.hbase11xreader; - -import com.wgzhao.addax.common.spi.ErrorCode; - -/** - * Created by shf on 16/3/8. - */ -public enum Hbase11xReaderErrorCode - implements ErrorCode -{ - REQUIRED_VALUE("Hbase11xReader-00", "您缺失了必须填写的参数值."), - ILLEGAL_VALUE("Hbase11xReader-01", "您填写的参数值不合法."), - PREPARE_READ_ERROR("HbaseReader-02", "准备读取 Hbase 时出错."), - SPLIT_ERROR("HbaseReader-03", "切分 Hbase 表时出错."), - GET_HBASE_CONNECTION_ERROR("HbaseReader-04", "获取Hbase连接时出错."), - GET_HBASE_TABLE_ERROR("HbaseReader-05", "初始化 Hbase 抽取表时出错."), - GET_HBASE_REGINLOCTOR_ERROR("HbaseReader-06", "获取 Hbase RegionLocator时出错."), - CLOSE_HBASE_CONNECTION_ERROR("HbaseReader-07", "关闭Hbase连接时出错."), - CLOSE_HBASE_TABLE_ERROR("HbaseReader-08", "关闭Hbase 抽取表时出错."), - CLOSE_HBASE_REGINLOCTOR_ERROR("HbaseReader-09", "关闭 Hbase RegionLocator时出错."), - CLOSE_HBASE_ADMIN_ERROR("HbaseReader-10", "关闭 Hbase admin时出错."); - - private final String code; - private final String description; - - private Hbase11xReaderErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s].", this.code, - this.description); - } -} diff --git a/plugin/reader/hbase11xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase11xreader/HbaseAbstractTask.java b/plugin/reader/hbase11xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase11xreader/HbaseAbstractTask.java index 352c87d46..503baa336 100644 --- a/plugin/reader/hbase11xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase11xreader/HbaseAbstractTask.java +++ b/plugin/reader/hbase11xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase11xreader/HbaseAbstractTask.java @@ -44,6 +44,8 @@ import java.io.UnsupportedEncodingException; import java.text.ParseException; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; + public abstract class HbaseAbstractTask { private final static Logger LOG = LoggerFactory.getLogger(HbaseAbstractTask.class); @@ -160,7 +162,7 @@ public Column convertBytesToAssignType(ColumnType columnType, byte[] byteArray, column = new DateColumn(isEmpty ? null : DateUtils.parseDate(dateValue, dateformat)); break; default: - throw AddaxException.asAddaxException(Hbase11xReaderErrorCode.ILLEGAL_VALUE, "Hbasereader 不支持您配置的列类型:" + columnType); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "Hbasereader 不支持您配置的列类型:" + columnType); } return column; } @@ -189,7 +191,7 @@ public Column convertValueToAssignType(ColumnType columnType, String constantVal column = new DateColumn(DateUtils.parseDate(constantValue, dateformat)); break; default: - throw AddaxException.asAddaxException(Hbase11xReaderErrorCode.ILLEGAL_VALUE, "Hbasereader 常量列不支持您配置的列类型:" + columnType); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "Hbasereader 常量列不支持您配置的列类型:" + columnType); } return column; } diff --git a/plugin/reader/hbase11xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase11xreader/ModeType.java b/plugin/reader/hbase11xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase11xreader/ModeType.java index 626cee9db..8a4e78fae 100644 --- a/plugin/reader/hbase11xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase11xreader/ModeType.java +++ b/plugin/reader/hbase11xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase11xreader/ModeType.java @@ -23,6 +23,8 @@ import java.util.Arrays; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; + public enum ModeType { NORMAL("normal"), @@ -42,7 +44,7 @@ public static ModeType getByTypeName(String modeName) return modeType; } } - throw AddaxException.asAddaxException(Hbase11xReaderErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, String.format("HbaseReader 不支持该 mode 类型:%s, 目前支持的 mode 类型是:%s", modeName, Arrays.asList(values()))); } diff --git a/plugin/reader/hbase11xsqlreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase11xsqlreader/HbaseSQLHelper.java b/plugin/reader/hbase11xsqlreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase11xsqlreader/HbaseSQLHelper.java index 4de66e4b4..81b1f34b7 100644 --- a/plugin/reader/hbase11xsqlreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase11xsqlreader/HbaseSQLHelper.java +++ b/plugin/reader/hbase11xsqlreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase11xsqlreader/HbaseSQLHelper.java @@ -42,6 +42,11 @@ import java.util.List; import java.util.Map; +import static com.wgzhao.addax.common.spi.ErrorCode.EXECUTE_FAIL; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.LOGIN_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; + public class HbaseSQLHelper { private static final Logger LOG = LoggerFactory.getLogger(HbaseSQLHelper.class); @@ -62,7 +67,7 @@ public Configuration parseConfig() if (hbaseCfg == null || hbaseCfg.isEmpty()) { // 集群配置必须存在且不为空 throw AddaxException.asAddaxException( - HbaseSQLReaderErrorCode.REQUIRED_VALUE, + REQUIRED_VALUE, String.format("%s must be configured with the following: %s and %s", HBaseKey.HBASE_CONFIG, HConstants.ZOOKEEPER_QUORUM, HConstants.ZOOKEEPER_ZNODE_PARENT)); } @@ -71,7 +76,7 @@ public Configuration parseConfig() List columns = jobConf.getList(Key.COLUMN, String.class); if (table == null && querySql == null) { throw AddaxException.asAddaxException( - HbaseSQLReaderErrorCode.REQUIRED_VALUE, + REQUIRED_VALUE, String.format("The %s and %s must have a configuration", Key.TABLE, Key.QUERY_SQL) ); } @@ -82,7 +87,7 @@ public Configuration parseConfig() // check columns if (columns == null) { throw AddaxException.asAddaxException( - HbaseSQLReaderErrorCode.ILLEGAL_VALUE, "The column configuration contains illegal chars, please check them"); + ILLEGAL_VALUE, "The column configuration contains illegal chars, please check them"); } String zkQuorum = hbaseCfg.getOrDefault(HConstants.ZOOKEEPER_QUORUM, "").toString(); @@ -90,11 +95,11 @@ public Configuration parseConfig() if (zkQuorum.isEmpty()) { throw AddaxException.asAddaxException( - HbaseSQLReaderErrorCode.ILLEGAL_VALUE, "The " + HConstants.ZOOKEEPER_QUORUM + " can not be set to empty"); + ILLEGAL_VALUE, "The " + HConstants.ZOOKEEPER_QUORUM + " can not be set to empty"); } if (!znode.startsWith("/")) { throw AddaxException.asAddaxException( - HbaseSQLReaderErrorCode.ILLEGAL_VALUE, "The " + HConstants.ZOOKEEPER_ZNODE_PARENT + " must be start with /" + ILLEGAL_VALUE, "The " + HConstants.ZOOKEEPER_ZNODE_PARENT + " must be start with /" ); } @@ -114,7 +119,7 @@ public Configuration parseConfig() String keytab = jobConf.getString(Key.KERBEROS_KEYTAB_FILE_PATH); if (principal == null || keytab == null) { throw AddaxException.asAddaxException( - HbaseSQLReaderErrorCode.REQUIRED_VALUE, + REQUIRED_VALUE, "To enable kerberos, you must both configure " + Key.KERBEROS_PRINCIPAL + " and " + Key.KERBEROS_KEYTAB_FILE_PATH ); } @@ -179,7 +184,7 @@ public List getPColumnNames(String fullTableName, String url) } catch (SQLException e) { throw AddaxException.asAddaxException( - HbaseSQLReaderErrorCode.GET_PHOENIX_COLUMN_ERROR, "Failed to get table's column description:\n" + e.getMessage(), e); + EXECUTE_FAIL, "Failed to get table's column description:\n" + e.getMessage(), e); } } @@ -194,7 +199,7 @@ private void kerberosAuthentication(String kerberosPrincipal, String kerberosKey String message = String.format("kerberos authentication failed, please make sure that kerberosKeytabFilePath[%s] and kerberosPrincipal[%s] are configure correctly", kerberosKeytabFilePath, kerberosPrincipal); LOG.error(message); - throw AddaxException.asAddaxException(HbaseSQLReaderErrorCode.KERBEROS_LOGIN_ERROR, e); + throw AddaxException.asAddaxException(LOGIN_ERROR, e); } } } diff --git a/plugin/reader/hbase11xsqlreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase11xsqlreader/HbaseSQLReader.java b/plugin/reader/hbase11xsqlreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase11xsqlreader/HbaseSQLReader.java index e65a78740..e1a15a494 100644 --- a/plugin/reader/hbase11xsqlreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase11xsqlreader/HbaseSQLReader.java +++ b/plugin/reader/hbase11xsqlreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase11xsqlreader/HbaseSQLReader.java @@ -50,6 +50,9 @@ import java.util.ArrayList; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.EXECUTE_FAIL; +import static com.wgzhao.addax.common.spi.ErrorCode.NOT_SUPPORT_TYPE; + public class HbaseSQLReader extends Reader { @@ -128,8 +131,7 @@ record = transportOneRecord(recordSender, resultSet, metaData, metaData.getColum recordSender.flush(); } catch (SQLException e) { - throw AddaxException.asAddaxException( - HbaseSQLReaderErrorCode.HBASE_CONNECTION_ERROR, + throw AddaxException.asAddaxException(EXECUTE_FAIL, e.getMessage() ); } @@ -187,7 +189,7 @@ private Record transportOneRecord(RecordSender recordSender, ResultSet resultSet break; default: throw AddaxException.asAddaxException( - HbaseSQLReaderErrorCode.PHOENIX_COLUMN_TYPE_CONVERT_ERROR, "The data type " + rmd.getColumnType(i) + " is unsupported yet"); + NOT_SUPPORT_TYPE, "The data type " + rmd.getColumnType(i) + " is unsupported yet"); } record.addColumn(column); } diff --git a/plugin/reader/hbase11xsqlreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase11xsqlreader/HbaseSQLReaderErrorCode.java b/plugin/reader/hbase11xsqlreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase11xsqlreader/HbaseSQLReaderErrorCode.java deleted file mode 100644 index 7ae84ba1b..000000000 --- a/plugin/reader/hbase11xsqlreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase11xsqlreader/HbaseSQLReaderErrorCode.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.reader.hbase11xsqlreader; - -import com.wgzhao.addax.common.spi.ErrorCode; - -public enum HbaseSQLReaderErrorCode - implements ErrorCode -{ - REQUIRED_VALUE("HBaseReader-00", "您缺失了必须填写的参数值."), - ILLEGAL_VALUE("HBaseReader-01", "您填写的参数值不合法."), - GET_PHOENIX_COLUMN_ERROR("HBaseReader-02", "获取phoenix表的列值错误"), - GET_PHOENIX_CONNECTION_ERROR("HBaseReader-03", "获取phoenix服务的 zkUrl 错误"), - GET_PHOENIX_SPLITS_ERROR("HBaseReader-04", "获取phoenix的split信息错误"), - PHOENIX_CREATE_READER_ERROR("HBaseReader-05", "获取phoenix的reader错误"), - PHOENIX_READER_INIT_ERROR("HBaseReader-06", "phoenix reader的初始化错误"), - PHOENIX_COLUMN_TYPE_CONVERT_ERROR("HBaseReader-07", "phoenix的列类型转换错误"), - // PHOENIX_RECORD_READ_ERROR("HBaseReader-08", "phoenix record 读取错误"), - PHOENIX_READER_CLOSE_ERROR("HBaseReader-09", "phoenix reader 的close错误"), - KERBEROS_LOGIN_ERROR("HBaseReader-10", "KERBEROS认证失败"), - HBASE_CONNECTION_ERROR("HBaseReader-11", "连接 HBase 失败"); - - private final String code; - private final String description; - - HbaseSQLReaderErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s].", this.code, this.description); - } -} diff --git a/plugin/reader/hbase20xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase20xreader/ColumnType.java b/plugin/reader/hbase20xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase20xreader/ColumnType.java index c1e2f6f2d..be7158722 100644 --- a/plugin/reader/hbase20xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase20xreader/ColumnType.java +++ b/plugin/reader/hbase20xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase20xreader/ColumnType.java @@ -24,6 +24,9 @@ import java.util.Arrays; +import static com.wgzhao.addax.common.spi.ErrorCode.NOT_SUPPORT_TYPE; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; + /** * 只对 normal 模式读取时有用，多版本读取时，不存在列类型的 */ @@ -49,7 +52,7 @@ public enum ColumnType public static ColumnType getByTypeName(String typeName) { if (StringUtils.isBlank(typeName)) { - throw AddaxException.asAddaxException(Hbase20xReaderErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(REQUIRED_VALUE, String.format("Hbasereader 不支持该类型:%s, 目前支持的类型是:%s", typeName, Arrays.asList(values()))); } for (ColumnType columnType : values()) { @@ -58,7 +61,7 @@ public static ColumnType getByTypeName(String typeName) } } - throw AddaxException.asAddaxException(Hbase20xReaderErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(NOT_SUPPORT_TYPE, String.format("Hbasereader 不支持该类型:%s, 目前支持的类型是:%s", typeName, Arrays.asList(values()))); } diff --git a/plugin/reader/hbase20xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase20xreader/Hbase20xHelper.java b/plugin/reader/hbase20xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase20xreader/Hbase20xHelper.java index 50263cc63..c68b0c45c 100644 --- a/plugin/reader/hbase20xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase20xreader/Hbase20xHelper.java +++ b/plugin/reader/hbase20xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase20xreader/Hbase20xHelper.java @@ -48,6 +48,11 @@ import java.util.List; import java.util.Map; +import static com.wgzhao.addax.common.spi.ErrorCode.EXECUTE_FAIL; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.IO_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; + public class Hbase20xHelper { @@ -60,26 +65,23 @@ public static org.apache.hadoop.hbase.client.Connection getHbaseConnection(Strin return H_CONNECTION; } if (StringUtils.isBlank(hbaseConfig)) { - throw AddaxException.asAddaxException(Hbase20xReaderErrorCode.REQUIRED_VALUE, "读 Hbase 时需要配置hbaseConfig，其内容为 Hbase 连接信息，请联系 Hbase PE 获取该信息."); + throw AddaxException.asAddaxException(REQUIRED_VALUE, "读 Hbase 时需要配置hbaseConfig，其内容为 Hbase 连接信息，请联系 Hbase PE 获取该信息."); } org.apache.hadoop.conf.Configuration hConfiguration = HBaseConfiguration.create(); - try { - Map hbaseConfigMap = JSON.parseObject(hbaseConfig, new TypeReference>() {}); - // 用户配置的 key-value 对 来表示 hbaseConfig - Validate.isTrue(hbaseConfigMap != null && hbaseConfigMap.size() != 0, "hbaseConfig不能为空Map结构!"); - for (Map.Entry entry : hbaseConfigMap.entrySet()) { - hConfiguration.set(entry.getKey(), entry.getValue()); - } - } - catch (Exception e) { - throw AddaxException.asAddaxException(Hbase20xReaderErrorCode.GET_HBASE_CONNECTION_ERROR, e); + + Map hbaseConfigMap = JSON.parseObject(hbaseConfig, new TypeReference>() {}); + // 用户配置的 key-value 对 来表示 hbaseConfig + Validate.isTrue(hbaseConfigMap != null && !hbaseConfigMap.isEmpty(), "hbaseConfig不能为空Map结构!"); + for (Map.Entry entry : hbaseConfigMap.entrySet()) { + hConfiguration.set(entry.getKey(), entry.getValue()); } + try { H_CONNECTION = ConnectionFactory.createConnection(hConfiguration); } - catch (Exception e) { + catch (IOException e) { Hbase20xHelper.closeConnection(H_CONNECTION); - throw AddaxException.asAddaxException(Hbase20xReaderErrorCode.GET_HBASE_CONNECTION_ERROR, e); + throw AddaxException.asAddaxException(IO_ERROR, e); } return H_CONNECTION; } @@ -97,10 +99,10 @@ public static Table getTable(Configuration configuration) Hbase20xHelper.checkHbaseTable(admin, hTableName); hTable = hConnection.getTable(hTableName); } - catch (Exception e) { + catch (IOException e) { Hbase20xHelper.closeAdmin(admin); Hbase20xHelper.closeConnection(hConnection); - throw AddaxException.asAddaxException(Hbase20xReaderErrorCode.GET_HBASE_TABLE_ERROR, e); + throw AddaxException.asAddaxException(IO_ERROR, e); } return hTable; } @@ -118,10 +120,10 @@ public static RegionLocator getRegionLocator(Configuration configuration) Hbase20xHelper.checkHbaseTable(admin, hTableName); regionLocator = hConnection.getRegionLocator(hTableName); } - catch (Exception e) { + catch (IOException e) { Hbase20xHelper.closeAdmin(admin); Hbase20xHelper.closeConnection(hConnection); - throw AddaxException.asAddaxException(Hbase20xReaderErrorCode.GET_HBASE_REGINLOCTOR_ERROR, e); + throw AddaxException.asAddaxException(IO_ERROR, e); } return regionLocator; } @@ -134,7 +136,7 @@ public synchronized static void closeConnection(Connection hConnection) } } catch (IOException e) { - throw AddaxException.asAddaxException(Hbase20xReaderErrorCode.CLOSE_HBASE_CONNECTION_ERROR, e); + throw AddaxException.asAddaxException(IO_ERROR, e); } } @@ -146,7 +148,7 @@ public static void closeAdmin(Admin admin) } } catch (IOException e) { - throw AddaxException.asAddaxException(Hbase20xReaderErrorCode.CLOSE_HBASE_ADMIN_ERROR, e); + throw AddaxException.asAddaxException(IO_ERROR, e); } } @@ -158,7 +160,7 @@ public static void closeTable(Table table) } } catch (IOException e) { - throw AddaxException.asAddaxException(Hbase20xReaderErrorCode.CLOSE_HBASE_TABLE_ERROR, e); + throw AddaxException.asAddaxException(IO_ERROR, e); } } @@ -177,7 +179,7 @@ public static void closeRegionLocator(RegionLocator regionLocator) } } catch (IOException e) { - throw AddaxException.asAddaxException(Hbase20xReaderErrorCode.CLOSE_HBASE_REGINLOCTOR_ERROR, e); + throw AddaxException.asAddaxException(IO_ERROR, e); } } @@ -185,15 +187,15 @@ public static void checkHbaseTable(Admin admin, TableName hTableName) throws IOException { if (!admin.tableExists(hTableName)) { - throw AddaxException.asAddaxException(Hbase20xReaderErrorCode.ILLEGAL_VALUE, "HBase源头表" + hTableName.toString() + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "HBase源头表" + hTableName.toString() + "不存在, 请检查您的配置 或者 联系 Hbase 管理员."); } if (!admin.isTableAvailable(hTableName)) { - throw AddaxException.asAddaxException(Hbase20xReaderErrorCode.ILLEGAL_VALUE, "HBase源头表" + hTableName.toString() + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "HBase源头表" + hTableName.toString() + " 不可用, 请检查您的配置 或者 联系 Hbase 管理员."); } if (admin.isTableDisabled(hTableName)) { - throw AddaxException.asAddaxException(Hbase20xReaderErrorCode.ILLEGAL_VALUE, "HBase源头表" + hTableName.toString() + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "HBase源头表" + hTableName.toString() + "is disabled, 请检查您的配置 或者 联系 Hbase 管理员."); } } @@ -288,7 +290,7 @@ public static List parseColumnOfNormalMode(List column) dateformat = HBaseConstant.DEFAULT_DATE_FORMAT; } Validate.isTrue(StringUtils.isNotBlank(columnName) - || StringUtils.isNotBlank(columnValue), + || StringUtils.isNotBlank(columnValue), "Hbasereader 在 normal 方式读取时则要么是 type + name + format 的组合，" + "要么是type + value + format 的组合. 而您的配置非这两种组合，请检查并修改."); @@ -301,7 +303,7 @@ public static List parseColumnOfNormalMode(List column) } else { Validate.isTrue(StringUtils.isNotBlank(columnName) - || StringUtils.isNotBlank(columnValue), + || StringUtils.isNotBlank(columnValue), "Hbasereader 在 normal 方式读取时，其列配置中，如果类型不是时间，" + "则要么是 type + name 的组合，要么是type + value 的组合. 而您的配置非这两种组合，请检查并修改."); oneColumnCell = new HbaseColumnCell.Builder(type) @@ -333,7 +335,7 @@ public static HashMap> parseColumnOfMultiversion if (!Hbase20xHelper.isRowkeyColumn(columnName)) { String[] cfAndQualifier = columnName.split(":"); if (cfAndQualifier.length != 2) { - throw AddaxException.asAddaxException(Hbase20xReaderErrorCode.ILLEGAL_VALUE, "Hbasereader 中，column 的列配置格式应该是：列族:列名. 您配置的列错误：" + columnName); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "Hbasereader 中，column 的列配置格式应该是：列族:列名. 您配置的列错误：" + columnName); } familyQualifier = StringUtils.join(cfAndQualifier[0].trim(), ":", cfAndQualifier[1].trim()); } @@ -357,14 +359,14 @@ public static List split(Configuration configuration) /* 如果用户配置了 startRowkey 和 endRowkey，需要确保：startRowkey <= endRowkey */ if (startRowkeyByte.length != 0 && endRowkeyByte.length != 0 && Bytes.compareTo(startRowkeyByte, endRowkeyByte) > 0) { - throw AddaxException.asAddaxException(Hbase20xReaderErrorCode.ILLEGAL_VALUE, "Hbasereader 中 startRowkey 不得大于 endRowkey."); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "Hbasereader 中 startRowkey 不得大于 endRowkey."); } RegionLocator regionLocator = Hbase20xHelper.getRegionLocator(configuration); List resultConfigurations; try { Pair regionRanges = regionLocator.getStartEndKeys(); if (null == regionRanges) { - throw AddaxException.asAddaxException(Hbase20xReaderErrorCode.SPLIT_ERROR, "获取源头 Hbase 表的 rowkey 范围失败."); + throw AddaxException.asAddaxException(EXECUTE_FAIL, "获取源头 Hbase 表的 rowkey 范围失败."); } resultConfigurations = Hbase20xHelper.doSplit(configuration, startRowkeyByte, endRowkeyByte, regionRanges); @@ -373,7 +375,7 @@ public static List split(Configuration configuration) return resultConfigurations; } catch (Exception e) { - throw AddaxException.asAddaxException(Hbase20xReaderErrorCode.SPLIT_ERROR, "切分源头 Hbase 表失败.", e); + throw AddaxException.asAddaxException(EXECUTE_FAIL, "切分源头 Hbase 表失败.", e); } finally { Hbase20xHelper.closeRegionLocator(regionLocator); @@ -478,15 +480,15 @@ private static String getStartKey(byte[] startRowkeyByte, byte[] regionStarKey) public static void validateParameter(Configuration originalConfig) { - originalConfig.getNecessaryValue(HBaseKey.HBASE_CONFIG, Hbase20xReaderErrorCode.REQUIRED_VALUE); - originalConfig.getNecessaryValue(HBaseKey.TABLE, Hbase20xReaderErrorCode.REQUIRED_VALUE); + originalConfig.getNecessaryValue(HBaseKey.HBASE_CONFIG, REQUIRED_VALUE); + originalConfig.getNecessaryValue(HBaseKey.TABLE, REQUIRED_VALUE); Hbase20xHelper.validateMode(originalConfig); //非必选参数处理 String encoding = originalConfig.getString(HBaseKey.ENCODING, HBaseConstant.DEFAULT_ENCODING); if (!Charset.isSupported(encoding)) { - throw AddaxException.asAddaxException(Hbase20xReaderErrorCode.ILLEGAL_VALUE, String.format("Hbasereader 不支持您所配置的编码:[%s]", encoding)); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, String.format("Hbasereader 不支持您所配置的编码:[%s]", encoding)); } originalConfig.set(HBaseKey.ENCODING, encoding); // 处理 range 的配置 @@ -515,10 +517,10 @@ public static void validateParameter(Configuration originalConfig) private static void validateMode(Configuration originalConfig) { - String mode = originalConfig.getNecessaryValue(HBaseKey.MODE, Hbase20xReaderErrorCode.REQUIRED_VALUE); + String mode = originalConfig.getNecessaryValue(HBaseKey.MODE, REQUIRED_VALUE); List column = originalConfig.getList(HBaseKey.COLUMN, Map.class); if (column == null || column.isEmpty()) { - throw AddaxException.asAddaxException(Hbase20xReaderErrorCode.REQUIRED_VALUE, + throw AddaxException.asAddaxException(REQUIRED_VALUE, "您配置的column为空,Hbase必须配置 column，其形式为：column:[{\"name\": \"cf0:column0\",\"type\": \"string\"},{\"name\": \"cf1:column1\",\"type\": \"long\"}]"); } ModeType modeType = ModeType.getByTypeName(mode); @@ -539,7 +541,7 @@ private static void validateMode(Configuration originalConfig) break; } default: - throw AddaxException.asAddaxException(Hbase20xReaderErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, String.format("HbaseReader不支持该 mode 类型:%s", mode)); } } diff --git a/plugin/reader/hbase20xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase20xreader/Hbase20xReader.java b/plugin/reader/hbase20xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase20xreader/Hbase20xReader.java index 4a52d7c51..d6b9d1ff7 100644 --- a/plugin/reader/hbase20xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase20xreader/Hbase20xReader.java +++ b/plugin/reader/hbase20xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase20xreader/Hbase20xReader.java @@ -30,6 +30,9 @@ import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; + /** * Hbase11xReader * Created by shf on 16/3/7. @@ -83,7 +86,7 @@ public void init() this.hbaseTaskProxy = new MultiVersionFixedColumnTask(taskConfig); break; default: - throw AddaxException.asAddaxException(Hbase20xReaderErrorCode.ILLEGAL_VALUE, "Hbasereader 不支持此类模式:" + modeType); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "Hbasereader 不支持此类模式:" + modeType); } } @@ -94,7 +97,7 @@ public void prepare() this.hbaseTaskProxy.prepare(); } catch (Exception e) { - throw AddaxException.asAddaxException(Hbase20xReaderErrorCode.PREPARE_READ_ERROR, e); + throw AddaxException.asAddaxException(RUNTIME_ERROR, e); } } diff --git a/plugin/reader/hbase20xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase20xreader/Hbase20xReaderErrorCode.java b/plugin/reader/hbase20xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase20xreader/Hbase20xReaderErrorCode.java deleted file mode 100644 index 3e9eeed40..000000000 --- a/plugin/reader/hbase20xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase20xreader/Hbase20xReaderErrorCode.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.reader.hbase20xreader; - -import com.wgzhao.addax.common.spi.ErrorCode; - -/** - * Created by shf on 16/3/8. - */ -public enum Hbase20xReaderErrorCode - implements ErrorCode -{ - REQUIRED_VALUE("Hbase20xReader-00", "您缺失了必须填写的参数值."), - ILLEGAL_VALUE("Hbase20xReader-01", "您填写的参数值不合法."), - PREPARE_READ_ERROR("HbaseReader-02", "准备读取 Hbase 时出错."), - SPLIT_ERROR("HbaseReader-03", "切分 Hbase 表时出错."), - GET_HBASE_CONNECTION_ERROR("HbaseReader-04", "获取Hbase连接时出错."), - GET_HBASE_TABLE_ERROR("HbaseReader-05", "初始化 Hbase 抽取表时出错."), - GET_HBASE_REGINLOCTOR_ERROR("HbaseReader-06", "获取 Hbase RegionLocator时出错."), - CLOSE_HBASE_CONNECTION_ERROR("HbaseReader-07", "关闭Hbase连接时出错."), - CLOSE_HBASE_TABLE_ERROR("HbaseReader-08", "关闭Hbase 抽取表时出错."), - CLOSE_HBASE_REGINLOCTOR_ERROR("HbaseReader-09", "关闭 Hbase RegionLocator时出错."), - CLOSE_HBASE_ADMIN_ERROR("HbaseReader-10", "关闭 Hbase admin时出错."); - - private final String code; - private final String description; - - Hbase20xReaderErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s].", this.code, - this.description); - } -} diff --git a/plugin/reader/hbase20xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase20xreader/HbaseAbstractTask.java b/plugin/reader/hbase20xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase20xreader/HbaseAbstractTask.java index 5acadd78b..01aae1052 100644 --- a/plugin/reader/hbase20xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase20xreader/HbaseAbstractTask.java +++ b/plugin/reader/hbase20xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase20xreader/HbaseAbstractTask.java @@ -42,6 +42,9 @@ import java.io.IOException; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.NOT_SUPPORT_TYPE; + public abstract class HbaseAbstractTask { private final static Logger LOG = LoggerFactory.getLogger(HbaseAbstractTask.class); @@ -157,7 +160,7 @@ public Column convertBytesToAssignType(ColumnType columnType, byte[] byteArray, column = new DateColumn(ArrayUtils.isEmpty(byteArray) ? null : DateUtils.parseDate(dateValue, dateformat)); break; default: - throw AddaxException.asAddaxException(Hbase20xReaderErrorCode.ILLEGAL_VALUE, "Hbasereader 不支持您配置的列类型:" + columnType); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "Hbasereader 不支持您配置的列类型:" + columnType); } return column; } @@ -186,7 +189,7 @@ public Column convertValueToAssignType(ColumnType columnType, String constantVal column = new DateColumn(DateUtils.parseDate(constantValue, dateformat)); break; default: - throw AddaxException.asAddaxException(Hbase20xReaderErrorCode.ILLEGAL_VALUE, "Hbasereader 常量列不支持您配置的列类型:" + columnType); + throw AddaxException.asAddaxException(NOT_SUPPORT_TYPE, "Hbasereader 常量列不支持您配置的列类型:" + columnType); } return column; } diff --git a/plugin/reader/hbase20xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase20xreader/ModeType.java b/plugin/reader/hbase20xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase20xreader/ModeType.java index 69098baa0..acc5410a7 100644 --- a/plugin/reader/hbase20xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase20xreader/ModeType.java +++ b/plugin/reader/hbase20xreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase20xreader/ModeType.java @@ -23,6 +23,8 @@ import java.util.Arrays; +import static com.wgzhao.addax.common.spi.ErrorCode.NOT_SUPPORT_TYPE; + public enum ModeType { NORMAL("normal"), @@ -42,7 +44,7 @@ public static ModeType getByTypeName(String modeName) return modeType; } } - throw AddaxException.asAddaxException(Hbase20xReaderErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(NOT_SUPPORT_TYPE, String.format("HbaseReader 不支持该 mode 类型:%s, 目前支持的 mode 类型是:%s", modeName, Arrays.asList(values()))); } diff --git a/plugin/reader/hbase20xsqlreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase20xsqlreader/HBase20SQLReaderHelper.java b/plugin/reader/hbase20xsqlreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase20xsqlreader/HBase20SQLReaderHelper.java index a054b637f..c009f9c29 100644 --- a/plugin/reader/hbase20xsqlreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase20xsqlreader/HBase20SQLReaderHelper.java +++ b/plugin/reader/hbase20xsqlreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase20xsqlreader/HBase20SQLReaderHelper.java @@ -23,7 +23,6 @@ import com.wgzhao.addax.common.base.HBaseKey; import com.wgzhao.addax.common.exception.AddaxException; import com.wgzhao.addax.common.util.Configuration; -import com.wgzhao.addax.rdbms.util.DBUtilErrorCode; import com.wgzhao.addax.rdbms.util.RdbmsRangeSplitWrap; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.ImmutablePair; @@ -42,6 +41,13 @@ import java.util.ArrayList; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.CONNECT_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.EXECUTE_FAIL; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.NOT_SUPPORT_TYPE; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; + public class HBase20SQLReaderHelper { private static final Logger LOG = LoggerFactory.getLogger(HBase20SQLReaderHelper.class); @@ -96,7 +102,7 @@ private static boolean isPKTypeValid(ResultSetMetaData rsMetaData) } } catch (Exception e) { - throw AddaxException.asAddaxException(DBUtilErrorCode.ILLEGAL_SPLIT_PK, + throw AddaxException.asAddaxException(EXECUTE_FAIL, "Addax 获取切分主键(splitPk)字段类型失败. 该错误通常是系统底层异常导致."); } return ret; @@ -122,7 +128,7 @@ public void validateParameter() { // query server地址必须配置 String queryServerAddress = configuration.getNecessaryValue(HBaseKey.QUERY_SERVER_ADDRESS, - HBase20xSQLReaderErrorCode.REQUIRED_VALUE); + REQUIRED_VALUE); String serialization = configuration.getString(HBaseKey.SERIALIZATION_NAME, HBaseConstant.DEFAULT_SERIALIZATION); connection = getConnection(queryServerAddress, serialization); @@ -132,7 +138,7 @@ public void validateParameter() LOG.info("Split according to splitKey or split points."); String schema = configuration.getString(HBaseKey.SCHEMA, null); - String tableName = configuration.getNecessaryValue(HBaseKey.TABLE, HBase20xSQLReaderErrorCode.REQUIRED_VALUE); + String tableName = configuration.getNecessaryValue(HBaseKey.TABLE, REQUIRED_VALUE); if (schema != null && !schema.isEmpty()) { fullTableName = "\"" + schema + "\".\"" + tableName + "\""; } @@ -163,7 +169,7 @@ public Connection getConnection(String queryServerAddress, String serialization) conn.setAutoCommit(false); } catch (Throwable e) { - throw AddaxException.asAddaxException(HBase20xSQLReaderErrorCode.GET_QUERY_SERVER_CONNECTION_ERROR, + throw AddaxException.asAddaxException(CONNECT_ERROR, "无法连接QueryServer，配置不正确或服务未启动，请检查配置和服务状态或者联系HBase管理员.", e); } LOG.debug("Connected to QueryServer successfully."); @@ -206,7 +212,7 @@ public void checkTable(String schema, String tableName) for (String columnName : columnNames) { if (!allColumnName.contains(columnName)) { // 用户配置的列名在元数据中不存在 - throw AddaxException.asAddaxException(HBase20xSQLReaderErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "您配置的列" + columnName + "在表" + tableName + "的元数据中不存在，请检查您的配置或者联系HBase管理员."); } } @@ -217,12 +223,12 @@ public void checkTable(String schema, String tableName) } if (splitKey != null && !primaryColumnNames.contains(splitKey)) { // 切分列必须是主键列，否则会严重影响读取性能 - throw AddaxException.asAddaxException(HBase20xSQLReaderErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "您配置的切分列" + splitKey + "不是表" + tableName + "的主键，请检查您的配置或者联系HBase管理员."); } } catch (SQLException e) { - throw AddaxException.asAddaxException(HBase20xSQLReaderErrorCode.GET_PHOENIX_TABLE_ERROR, + throw AddaxException.asAddaxException(EXECUTE_FAIL, "获取表" + tableName + "信息失败，请检查您的集群和表状态或者联系HBase管理员.", e); } finally { @@ -244,7 +250,7 @@ public void closeJdbc(Connection connection, Statement statement, ResultSet resu } } catch (SQLException e) { - LOG.warn("数据库连接关闭异常. {}", HBase20xSQLReaderErrorCode.CLOSE_PHOENIX_CONNECTION_ERROR, e); + LOG.warn("数据库连接关闭异常. {}", CONNECT_ERROR, e); } } @@ -278,7 +284,7 @@ public List doSplit(int adviceNumber) LOG.info("Split according to the min and max value of splitColumn..."); Pair minMaxPK = getPkRange(configuration); if (null == minMaxPK) { - throw AddaxException.asAddaxException(HBase20xSQLReaderErrorCode.ILLEGAL_SPLIT_PK, + throw AddaxException.asAddaxException(CONFIG_ERROR, "根据切分主键切分表失败. 仅支持切分主键为一个,并且类型为整数或者字符串类型. " + "请尝试使用其他的切分主键或者联系 HBase管理员 进行处理."); } @@ -304,7 +310,7 @@ else if (isLongType) { adviceNumber, splitKey); } else { - throw AddaxException.asAddaxException(HBase20xSQLReaderErrorCode.ILLEGAL_SPLIT_PK, + throw AddaxException.asAddaxException(NOT_SUPPORT_TYPE, "您配置的切分主键(splitPk) 类型不支持. 仅支持切分主键为一个,并且类型为整数或者字符串类型. " + "请尝试使用其他的切分主键或者联系HBase管理员进行处理."); } @@ -377,7 +383,7 @@ private List buildSplitRange() case Types.BINARY: case Types.VARBINARY: case Types.ARRAY: - throw AddaxException.asAddaxException(HBase20xSQLReaderErrorCode.ILLEGAL_SPLIT_PK, + throw AddaxException.asAddaxException(NOT_SUPPORT_TYPE, "切分列类型为" + rsMetaData.getColumnTypeName(1) + "，暂不支持该类型字段作为切分列。"); default: break; @@ -400,7 +406,7 @@ else if (i == splitPoints.size()) { return splitConditions; } catch (SQLException e) { - throw AddaxException.asAddaxException(HBase20xSQLReaderErrorCode.GET_TABLE_COLUMN_TYPE_ERROR, + throw AddaxException.asAddaxException(EXECUTE_FAIL, "获取切分列类型失败，请检查服务或给定表和切分列是否正常，或者联系HBase管理员进行处理。", e); } finally { @@ -443,19 +449,19 @@ else if (isLongType(rsMetaData.getColumnType(1))) { } } else { - throw AddaxException.asAddaxException(HBase20xSQLReaderErrorCode.ILLEGAL_SPLIT_PK, + throw AddaxException.asAddaxException(CONFIG_ERROR, "您配置的切分主键(splitPk)有误. 因为您配置的切分主键(splitPk) 类型不支持. " + "仅支持切分主键为一个,并且类型为整数或者字符串类型. 请尝试使用其他的切分主键或者联系HBASE管理员进行处理."); } } else { - throw AddaxException.asAddaxException(HBase20xSQLReaderErrorCode.ILLEGAL_SPLIT_PK, + throw AddaxException.asAddaxException(CONFIG_ERROR, "您配置的切分主键(splitPk)有误. 因为您配置的切分主键(splitPk) 类型不支持. " + "仅支持切分主键为一个,并且类型为整数或者字符串类型. 请尝试使用其他的切分主键或者联系HBASE管理员进行处理."); } } catch (SQLException e) { - throw AddaxException.asAddaxException(HBase20xSQLReaderErrorCode.ILLEGAL_SPLIT_PK, e); + throw AddaxException.asAddaxException(CONFIG_ERROR, e); } finally { closeJdbc(null, statement, resultSet); diff --git a/plugin/reader/hbase20xsqlreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase20xsqlreader/HBase20xSQLReaderErrorCode.java b/plugin/reader/hbase20xsqlreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase20xsqlreader/HBase20xSQLReaderErrorCode.java deleted file mode 100644 index 98b7976fb..000000000 --- a/plugin/reader/hbase20xsqlreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase20xsqlreader/HBase20xSQLReaderErrorCode.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.reader.hbase20xsqlreader; - -import com.wgzhao.addax.common.spi.ErrorCode; - -public enum HBase20xSQLReaderErrorCode - implements ErrorCode -{ - REQUIRED_VALUE("Hbasewriter-00", "您缺失了必须填写的参数值."), - ILLEGAL_VALUE("Hbasewriter-01", "您填写的参数值不合法."), - GET_QUERY_SERVER_CONNECTION_ERROR("Hbasewriter-02", "获取QueryServer连接时出错."), - GET_PHOENIX_TABLE_ERROR("Hbasewriter-03", "获取 Phoenix table时出错."), - GET_TABLE_COLUMN_TYPE_ERROR("Hbasewriter-05", "获取表列类型时出错."), - CLOSE_PHOENIX_CONNECTION_ERROR("Hbasewriter-06", "关闭JDBC连接时时出错."), - ILLEGAL_SPLIT_PK("Hbasewriter-07", "非法splitKey配置."), - PHOENIX_COLUMN_TYPE_CONVERT_ERROR("Hbasewriter-08", "phoenix的列类型转换错误."), - QUERY_DATA_ERROR("Hbasewriter-09", "truncate hbase表时发生异常."), - ; - - private final String code; - private final String description; - - HBase20xSQLReaderErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s].", this.code, this.description); - } -} diff --git a/plugin/reader/hbase20xsqlreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase20xsqlreader/HBase20xSQLReaderTask.java b/plugin/reader/hbase20xsqlreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase20xsqlreader/HBase20xSQLReaderTask.java index 328a6b170..5af0ff85d 100644 --- a/plugin/reader/hbase20xsqlreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase20xsqlreader/HBase20xSQLReaderTask.java +++ b/plugin/reader/hbase20xsqlreader/src/main/java/com/wgzhao/addax/plugin/reader/hbase20xsqlreader/HBase20xSQLReaderTask.java @@ -47,6 +47,9 @@ import java.sql.Timestamp; import java.sql.Types; +import static com.wgzhao.addax.common.spi.ErrorCode.EXECUTE_FAIL; +import static com.wgzhao.addax.common.spi.ErrorCode.NOT_SUPPORT_TYPE; + public class HBase20xSQLReaderTask { private static final Logger LOG = LoggerFactory.getLogger(HBase20xSQLReaderTask.class); @@ -101,7 +104,7 @@ public void readRecord(RecordSender recordSender) } catch (SQLException e) { throw AddaxException.asAddaxException( - HBase20xSQLReaderErrorCode.QUERY_DATA_ERROR, "查询Phoenix数据出现异常，请检查服务状态或与HBase管理员联系！", e); + EXECUTE_FAIL, "查询Phoenix数据出现异常，请检查服务状态或与HBase管理员联系！", e); } finally { helper.closeJdbc(conn, statement, resultSet); @@ -155,7 +158,7 @@ private Column convertPhoenixValueToAddaxColumn(int sqlType, Object value) break; default: throw AddaxException.asAddaxException( - HBase20xSQLReaderErrorCode.PHOENIX_COLUMN_TYPE_CONVERT_ERROR, "遇到不可识别的phoenix类型，" + "sqlType :" + sqlType); + NOT_SUPPORT_TYPE, "遇到不可识别的phoenix类型，" + "sqlType :" + sqlType); } return column; } diff --git a/plugin/reader/hdfsreader/src/main/java/com/wgzhao/addax/plugin/reader/hdfsreader/DFSUtil.java b/plugin/reader/hdfsreader/src/main/java/com/wgzhao/addax/plugin/reader/hdfsreader/DFSUtil.java index 626495bee..7367bb9c0 100644 --- a/plugin/reader/hdfsreader/src/main/java/com/wgzhao/addax/plugin/reader/hdfsreader/DFSUtil.java +++ b/plugin/reader/hdfsreader/src/main/java/com/wgzhao/addax/plugin/reader/hdfsreader/DFSUtil.java @@ -40,7 +40,6 @@ import com.wgzhao.addax.common.plugin.RecordSender; import com.wgzhao.addax.common.plugin.TaskPluginCollector; import com.wgzhao.addax.common.util.Configuration; -import com.wgzhao.addax.storage.reader.StorageReaderErrorCode; import com.wgzhao.addax.storage.reader.StorageReaderUtil; import org.apache.avro.Conversions; import org.apache.avro.generic.GenericData; @@ -98,6 +97,11 @@ import static com.wgzhao.addax.common.base.Key.COLUMN; import static com.wgzhao.addax.common.base.Key.NULL_FORMAT; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.EXECUTE_FAIL; +import static com.wgzhao.addax.common.spi.ErrorCode.IO_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.LOGIN_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.NOT_SUPPORT_TYPE; /** * Created by mingya.wmy on 2015/8/12. @@ -162,7 +166,7 @@ private void kerberosAuthentication(String kerberosPrincipal, String kerberosKey catch (Exception e) { String message = String.format("kerberos认证失败,请确定kerberosKeytabFilePath[%s]和kerberosPrincipal[%s]填写正确", kerberosKeytabFilePath, kerberosPrincipal); - throw AddaxException.asAddaxException(HdfsReaderErrorCode.KERBEROS_LOGIN_ERROR, message, e); + throw AddaxException.asAddaxException(LOGIN_ERROR, message, e); } } } @@ -225,7 +229,7 @@ else if (f.isDirectory()) { } catch (IOException e) { LOG.error("IO exception occurred while reading file(s) under [{}].", hdfsPath); - throw AddaxException.asAddaxException(HdfsReaderErrorCode.PATH_CONFIG_ERROR, e); + throw AddaxException.asAddaxException(CONFIG_ERROR, e); } } @@ -272,7 +276,7 @@ private void addSourceFileByType(String filePath) , filePath, this.specifiedFileType); LOG.error(message); throw AddaxException.asAddaxException( - HdfsReaderErrorCode.FILE_TYPE_UNSUPPORTED, message); + NOT_SUPPORT_TYPE, message); } } @@ -289,7 +293,7 @@ public InputStream getInputStream(String filepath) } catch (IOException e) { String message = String.format("IO exception occurred while reading the file [%s].", filepath); - throw AddaxException.asAddaxException(HdfsReaderErrorCode.READ_FILE_ERROR, message, e); + throw AddaxException.asAddaxException(IO_ERROR, message, e); } } @@ -314,7 +318,7 @@ public void sequenceFileStartRead(String sourceSequenceFilePath, Configuration r catch (Exception e) { String message = String.format("Exception occurred while reading the file [%s].", sourceSequenceFilePath); LOG.error(message); - throw AddaxException.asAddaxException(HdfsReaderErrorCode.READ_SEQUENCE_FILE_ERROR, message, e); + throw AddaxException.asAddaxException(EXECUTE_FAIL, message, e); } } @@ -351,7 +355,7 @@ public void rcFileStartRead(String sourceRcFilePath, Configuration readerSliceCo catch (IOException e) { String message = String.format("IO exception occurred while reading the file [%s].", sourceRcFilePath); LOG.error(message); - throw AddaxException.asAddaxException(HdfsReaderErrorCode.READ_RCFILE_ERROR, message, e); + throw AddaxException.asAddaxException(IO_ERROR, message, e); } finally { try { @@ -396,7 +400,7 @@ public void orcFileStartRead(String sourceOrcFilePath, Configuration readerSlice catch (Exception e) { String message = String.format("Exception occurred while reading the file [%s].", sourceOrcFilePath); LOG.error(message); - throw AddaxException.asAddaxException(HdfsReaderErrorCode.READ_FILE_ERROR, message); + throw AddaxException.asAddaxException(IO_ERROR, message); } } @@ -513,7 +517,7 @@ public void parquetFileStartRead(String sourceParquetFilePath, Configuration rea catch (IOException e) { String message = String.format("IO exception occurred while reading the parquet-file [%s]", sourceParquetFilePath); LOG.error(message); - throw AddaxException.asAddaxException(HdfsReaderErrorCode.READ_FILE_ERROR, message); + throw AddaxException.asAddaxException(IO_ERROR, message); } } @@ -653,7 +657,7 @@ private TypeDescription getOrcSchema(String filePath) return reader.getSchema(); } catch (IOException e) { - throw AddaxException.asAddaxException(HdfsReaderErrorCode.READ_FILE_ERROR, "IO exception occurred when reading orc file"); + throw AddaxException.asAddaxException(IO_ERROR, "IO exception occurred when reading orc file"); } } @@ -685,7 +689,7 @@ else if (StringUtils.equalsIgnoreCase(specifiedFileType, HdfsConstant.CSV) String message = String.format("Can not get the file format for [%s]，it only supports [%s].", filepath, HdfsConstant.SUPPORT_FILE_TYPE); LOG.error(message); - throw AddaxException.asAddaxException(HdfsReaderErrorCode.READ_FILE_ERROR, message, e); + throw AddaxException.asAddaxException(EXECUTE_FAIL, message, e); } return false; } diff --git a/plugin/reader/hdfsreader/src/main/java/com/wgzhao/addax/plugin/reader/hdfsreader/HdfsReader.java b/plugin/reader/hdfsreader/src/main/java/com/wgzhao/addax/plugin/reader/hdfsreader/HdfsReader.java index f98fb7d35..5fe25fa45 100644 --- a/plugin/reader/hdfsreader/src/main/java/com/wgzhao/addax/plugin/reader/hdfsreader/HdfsReader.java +++ b/plugin/reader/hdfsreader/src/main/java/com/wgzhao/addax/plugin/reader/hdfsreader/HdfsReader.java @@ -42,6 +42,11 @@ import static com.wgzhao.addax.common.base.Key.INDEX; import static com.wgzhao.addax.common.base.Key.TYPE; import static com.wgzhao.addax.common.base.Key.VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.EXECUTE_FAIL; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.NOT_SUPPORT_TYPE; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; public class HdfsReader extends Reader @@ -82,30 +87,30 @@ public void init() public void validate() { - readerOriginConfig.getNecessaryValue(Key.DEFAULT_FS, HdfsReaderErrorCode.DEFAULT_FS_NOT_FIND_ERROR); + readerOriginConfig.getNecessaryValue(Key.DEFAULT_FS, CONFIG_ERROR); // path check - String pathInString = readerOriginConfig.getNecessaryValue(Key.PATH, HdfsReaderErrorCode.REQUIRED_VALUE); + String pathInString = readerOriginConfig.getNecessaryValue(Key.PATH, REQUIRED_VALUE); if (!pathInString.startsWith("[") && !pathInString.endsWith("]")) { path = Collections.singletonList(pathInString); } else { path = readerOriginConfig.getList(Key.PATH, String.class); if (null == path || path.isEmpty()) { - throw AddaxException.asAddaxException(HdfsReaderErrorCode.REQUIRED_VALUE, "The item path is required."); + throw AddaxException.asAddaxException(REQUIRED_VALUE, "The item path is required."); } for (String eachPath : path) { if (!eachPath.startsWith("/")) { String message = String.format("The item path [%s] should be a absolute path.", eachPath); LOG.error(message); - throw AddaxException.asAddaxException(HdfsReaderErrorCode.ILLEGAL_VALUE, message); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, message); } } } - specifiedFileType = readerOriginConfig.getNecessaryValue(Key.FILE_TYPE, HdfsReaderErrorCode.REQUIRED_VALUE).toUpperCase(); + specifiedFileType = readerOriginConfig.getNecessaryValue(Key.FILE_TYPE, REQUIRED_VALUE).toUpperCase(); if (!HdfsConstant.SUPPORT_FILE_TYPE.contains(specifiedFileType)) { - throw AddaxException.asAddaxException(HdfsReaderErrorCode.FILE_TYPE_ERROR, + throw AddaxException.asAddaxException(NOT_SUPPORT_TYPE, "The file type only supports " + HdfsConstant.SUPPORT_FILE_TYPE + " but not " + specifiedFileType); } @@ -116,18 +121,18 @@ public void validate() } catch (UnsupportedCharsetException uce) { throw AddaxException.asAddaxException( - HdfsReaderErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, "The encoding [" + encoding + "] is unsupported.", uce); } catch (Exception e) { throw AddaxException.asAddaxException( - HdfsReaderErrorCode.ILLEGAL_VALUE, "Exception occurred", e); + ILLEGAL_VALUE, "Exception occurred", e); } //check Kerberos boolean haveKerberos = readerOriginConfig.getBool(Key.HAVE_KERBEROS, false); if (haveKerberos) { - readerOriginConfig.getNecessaryValue(Key.KERBEROS_KEYTAB_FILE_PATH, HdfsReaderErrorCode.REQUIRED_VALUE); - readerOriginConfig.getNecessaryValue(Key.KERBEROS_PRINCIPAL, HdfsReaderErrorCode.REQUIRED_VALUE); + readerOriginConfig.getNecessaryValue(Key.KERBEROS_KEYTAB_FILE_PATH, REQUIRED_VALUE); + readerOriginConfig.getNecessaryValue(Key.KERBEROS_PRINCIPAL, REQUIRED_VALUE); } // validate the Columns @@ -155,23 +160,23 @@ private void validateColumns() List columns = readerOriginConfig.getListConfiguration(COLUMN); if (null == columns || columns.isEmpty()) { - throw AddaxException.asAddaxException(HdfsReaderErrorCode.CONFIG_INVALID_EXCEPTION, + throw AddaxException.asAddaxException(CONFIG_ERROR, "The item columns is required."); } for (Configuration eachColumnConf : columns) { - eachColumnConf.getNecessaryValue(TYPE, HdfsReaderErrorCode.REQUIRED_VALUE); + eachColumnConf.getNecessaryValue(TYPE, REQUIRED_VALUE); Integer columnIndex = eachColumnConf.getInt(INDEX); String columnValue = eachColumnConf.getString(VALUE); if (null == columnIndex && null == columnValue) { throw AddaxException.asAddaxException( - HdfsReaderErrorCode.NO_INDEX_VALUE, + CONFIG_ERROR, "The index or value must have one, both of them are null."); } if (null != columnIndex && null != columnValue) { - throw AddaxException.asAddaxException(HdfsReaderErrorCode.MIXED_INDEX_VALUE, + throw AddaxException.asAddaxException(CONFIG_ERROR, "The index and value must have one, can not have both."); } } @@ -195,7 +200,7 @@ public List split(int adviceNumber) // warn:每个slice拖且仅拖一个文件, int splitNumber = sourceFiles.size(); if (0 == splitNumber) { - throw AddaxException.asAddaxException(HdfsReaderErrorCode.EMPTY_DIR_EXCEPTION, + throw AddaxException.asAddaxException(EXECUTE_FAIL, "Can not find any file in path : [" + readerOriginConfig.getString(Key.PATH) + "]"); } @@ -239,7 +244,7 @@ public void init() this.taskConfig = getPluginJobConf(); this.sourceFiles = taskConfig.getList(HdfsConstant.SOURCE_FILES, String.class); - this.specifiedFileType = taskConfig.getNecessaryValue(Key.FILE_TYPE, HdfsReaderErrorCode.REQUIRED_VALUE); + this.specifiedFileType = taskConfig.getNecessaryValue(Key.FILE_TYPE, REQUIRED_VALUE); this.dfsUtil = new DFSUtil(taskConfig); } @@ -277,7 +282,7 @@ else if (specifiedFileType.equalsIgnoreCase(HdfsConstant.PARQUET)) { dfsUtil.parquetFileStartRead(sourceFile, taskConfig, recordSender, getTaskPluginCollector()); } else { - throw AddaxException.asAddaxException(HdfsReaderErrorCode.FILE_TYPE_UNSUPPORTED, + throw AddaxException.asAddaxException(NOT_SUPPORT_TYPE, "The specifiedFileType: [" + specifiedFileType + "] is unsupported. " + "HdfsReader only support TEXT, CSV, ORC, SEQUENCE, RC, PARQUET now."); } diff --git a/plugin/reader/hdfsreader/src/main/java/com/wgzhao/addax/plugin/reader/hdfsreader/HdfsReaderErrorCode.java b/plugin/reader/hdfsreader/src/main/java/com/wgzhao/addax/plugin/reader/hdfsreader/HdfsReaderErrorCode.java deleted file mode 100644 index 7c8199fe9..000000000 --- a/plugin/reader/hdfsreader/src/main/java/com/wgzhao/addax/plugin/reader/hdfsreader/HdfsReaderErrorCode.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.reader.hdfsreader; - -import com.wgzhao.addax.common.spi.ErrorCode; - -public enum HdfsReaderErrorCode - implements ErrorCode -{ - BAD_CONFIG_VALUE("HdfsReader-00", "您配置的值不合法."), - // PATH_NOT_FIND_ERROR("HdfsReader-01", "您未配置path值"), - DEFAULT_FS_NOT_FIND_ERROR("HdfsReader-02", "您未配置defaultFS值"), - ILLEGAL_VALUE("HdfsReader-03", "值错误"), - CONFIG_INVALID_EXCEPTION("HdfsReader-04", "参数配置错误"), - REQUIRED_VALUE("HdfsReader-05", "您缺失了必须填写的参数值."), - NO_INDEX_VALUE("HdfsReader-06", "没有 index"), - MIXED_INDEX_VALUE("HdfsReader-07", "index 和 value 混合"), - EMPTY_DIR_EXCEPTION("HdfsReader-08", "您尝试读取的文件目录为空."), - PATH_CONFIG_ERROR("HdfsReader-09", "您配置的path格式有误"), - READ_FILE_ERROR("HdfsReader-10", "读取文件出错"), - // MALFORMED_ORC_ERROR("HdfsReader-10", "ORC FILE格式异常"), - FILE_TYPE_ERROR("HdfsReader-11", "文件类型配置错误"), - FILE_TYPE_UNSUPPORTED("HdfsReader-12", "文件类型目前不支持"), - KERBEROS_LOGIN_ERROR("HdfsReader-13", "KERBEROS认证失败"), - READ_SEQUENCE_FILE_ERROR("HdfsReader-14", "读取SequenceFile文件出错"), - READ_RCFILE_ERROR("HdfsReader-15", "读取RCFile文件出错"); - - private final String code; - private final String description; - - HdfsReaderErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s]. ", this.code, - this.description); - } -} \ No newline at end of file diff --git a/plugin/reader/httpreader/src/main/java/com/wgzhao/addax/plugin/reader/httpreader/HttpReader.java b/plugin/reader/httpreader/src/main/java/com/wgzhao/addax/plugin/reader/httpreader/HttpReader.java index 92b555fb7..05edc679a 100644 --- a/plugin/reader/httpreader/src/main/java/com/wgzhao/addax/plugin/reader/httpreader/HttpReader.java +++ b/plugin/reader/httpreader/src/main/java/com/wgzhao/addax/plugin/reader/httpreader/HttpReader.java @@ -79,6 +79,9 @@ import java.util.List; import java.util.Map; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; + public class HttpReader extends Reader { @@ -234,8 +237,7 @@ else if (object instanceof JSONObject) { List columns = readerSliceConfig.getList(HttpKey.COLUMN, String.class); if (columns == null || columns.isEmpty()) { - throw AddaxException.asAddaxException( - HttpReaderErrorCode.REQUIRED_VALUE, + throw AddaxException.asAddaxException(REQUIRED_VALUE, "The parameter [" + HttpKey.COLUMN + "] is not set." ); } @@ -266,7 +268,7 @@ record = recordSender.createRecord(); catch (URISyntaxException | IOException e) { throw AddaxException.asAddaxException( - HttpReaderErrorCode.ILLEGAL_VALUE, e.getMessage() + ILLEGAL_VALUE, e.getMessage() ); } } @@ -298,7 +300,7 @@ else if ("post".equalsIgnoreCase(method)) { } else { throw AddaxException.asAddaxException( - HttpReaderErrorCode.ILLEGAL_VALUE, "不支持的请求模式: " + method + ILLEGAL_VALUE, "不支持的请求模式: " + method ); } return response; diff --git a/plugin/reader/httpreader/src/main/java/com/wgzhao/addax/plugin/reader/httpreader/HttpReaderErrorCode.java b/plugin/reader/httpreader/src/main/java/com/wgzhao/addax/plugin/reader/httpreader/HttpReaderErrorCode.java deleted file mode 100644 index b936da232..000000000 --- a/plugin/reader/httpreader/src/main/java/com/wgzhao/addax/plugin/reader/httpreader/HttpReaderErrorCode.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.reader.httpreader; - -import com.wgzhao.addax.common.spi.ErrorCode; - -public enum HttpReaderErrorCode - implements ErrorCode -{ - REQUIRED_VALUE("HttpReader-00", "您缺失了必须填写的参数值."), - - ILLEGAL_VALUE("HttpReader-01", "您填写的参数值不合法."), - - FILE_NOT_EXISTS("HttpReader-04", "您配置的没有权限读取."), - -// OPEN_FILE_WITH_CHARSET_ERROR("HttpReader-05", "您配置的文件编码和实际文件编码不符合."), - - // READ_FILE_IO_ERROR("HttpReader-07", "您配置的文件在读取时出现IO异常."), -// SECURITY_NOT_ENOUGH("HttpReader-08", "您缺少权限执行相应的文件操作."), -// CONFIG_INVALID_EXCEPTION("HttpReader-09", "您的参数配置错误."), -// RUNTIME_EXCEPTION("HttpReader-10", "出现运行时异常, 请联系我们"), - - FAIL_LOGIN("HttpReader-12", "登录失败,无法与http服务器建立连接."), - - FAIL_DISCONNECT("HttpReader-13", "关闭http连接失败,无法与http服务器断开连接."), - - COMMAND_FTP_IO_EXCEPTION("HttpReader-14", "与http服务器连接异常."), - NOT_SUPPORT("HttpReader-15", "暂不支持该方式"), - ; - - private final String code; - private final String description; - - HttpReaderErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } -} diff --git a/plugin/reader/influxdb2reader/src/main/java/com/wgzhao/addax/plugin/reader/influxdb2reader/InfluxDB2Reader.java b/plugin/reader/influxdb2reader/src/main/java/com/wgzhao/addax/plugin/reader/influxdb2reader/InfluxDB2Reader.java index 96a92f55f..5d07b30db 100644 --- a/plugin/reader/influxdb2reader/src/main/java/com/wgzhao/addax/plugin/reader/influxdb2reader/InfluxDB2Reader.java +++ b/plugin/reader/influxdb2reader/src/main/java/com/wgzhao/addax/plugin/reader/influxdb2reader/InfluxDB2Reader.java @@ -46,6 +46,7 @@ import static com.wgzhao.addax.common.base.Key.ENDPOINT; import static com.wgzhao.addax.common.base.Key.QUERY_SQL; import static com.wgzhao.addax.common.base.Key.TABLE; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; public class InfluxDB2Reader extends Reader @@ -72,13 +73,13 @@ public void init() @Override public void prepare() { - this.token = originalConfig.getNecessaryValue(InfluxDB2Key.TOKEN, InfluxDB2ReaderErrorCode.REQUIRED_VALUE); - originalConfig.getNecessaryValue(InfluxDB2Key.RANGE, InfluxDB2ReaderErrorCode.REQUIRED_VALUE); + this.token = originalConfig.getNecessaryValue(InfluxDB2Key.TOKEN, REQUIRED_VALUE); + originalConfig.getNecessaryValue(InfluxDB2Key.RANGE, REQUIRED_VALUE); this.range = originalConfig.getList(InfluxDB2Key.RANGE, String.class); Configuration connConf = Configuration.from(originalConfig.getList(CONNECTION, Object.class).get(0).toString()); - this.endpoint = connConf.getNecessaryValue(InfluxDB2Key.ENDPOINT, InfluxDB2ReaderErrorCode.REQUIRED_VALUE); - this.bucket = connConf.getNecessaryValue(InfluxDB2Key.BUCKET, InfluxDB2ReaderErrorCode.REQUIRED_VALUE); - this.org = connConf.getNecessaryValue(InfluxDB2Key.ORG, InfluxDB2ReaderErrorCode.REQUIRED_VALUE); + this.endpoint = connConf.getNecessaryValue(InfluxDB2Key.ENDPOINT, REQUIRED_VALUE); + this.bucket = connConf.getNecessaryValue(InfluxDB2Key.BUCKET, REQUIRED_VALUE); + this.org = connConf.getNecessaryValue(InfluxDB2Key.ORG, REQUIRED_VALUE); this.tables = connConf.getList(TABLE, String.class); this.columns = originalConfig.getList(COLUMN, String.class); @@ -132,7 +133,7 @@ public Configuration dealColumns() fluxColumns.add(map); } else { - throw AddaxException.asAddaxException(InfluxDB2ReaderErrorCode.MISSING_COLUMN, + throw AddaxException.asAddaxException(REQUIRED_VALUE, "The column '" + col + "' you specified doest not exists"); } } diff --git a/plugin/reader/influxdb2reader/src/main/java/com/wgzhao/addax/plugin/reader/influxdb2reader/InfluxDB2ReaderErrorCode.java b/plugin/reader/influxdb2reader/src/main/java/com/wgzhao/addax/plugin/reader/influxdb2reader/InfluxDB2ReaderErrorCode.java deleted file mode 100644 index 466c5b436..000000000 --- a/plugin/reader/influxdb2reader/src/main/java/com/wgzhao/addax/plugin/reader/influxdb2reader/InfluxDB2ReaderErrorCode.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.reader.influxdb2reader; - -import com.wgzhao.addax.common.spi.ErrorCode; - -public enum InfluxDB2ReaderErrorCode - implements ErrorCode -{ - REQUIRED_VALUE("InfluxDB2Reader-00", "Missing mandatory configuration items"), - MISSING_COLUMN("InfluxDB2Reader-01", "Column does not exist"); - - private final String code; - private final String description; - - InfluxDB2ReaderErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s]. ", this.code, this.description); - } -} diff --git a/plugin/reader/influxdbreader/src/main/java/com/wgzhao/addax/plugin/reader/influxdbreader/InfluxDBReader.java b/plugin/reader/influxdbreader/src/main/java/com/wgzhao/addax/plugin/reader/influxdbreader/InfluxDBReader.java index 0c2019c09..17c733623 100644 --- a/plugin/reader/influxdbreader/src/main/java/com/wgzhao/addax/plugin/reader/influxdbreader/InfluxDBReader.java +++ b/plugin/reader/influxdbreader/src/main/java/com/wgzhao/addax/plugin/reader/influxdbreader/InfluxDBReader.java @@ -28,6 +28,8 @@ import java.util.ArrayList; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; + public class InfluxDBReader extends Reader { @@ -48,19 +50,19 @@ public void init() public void preCheck() { init(); - originalConfig.getNecessaryValue(InfluxDBKey.ENDPOINT, InfluxDBReaderErrorCode.REQUIRED_VALUE); + originalConfig.getNecessaryValue(InfluxDBKey.ENDPOINT, REQUIRED_VALUE); List columns = originalConfig.getList(InfluxDBKey.COLUMN, String.class); String querySql = originalConfig.getNecessaryValue(InfluxDBKey.QUERY_SQL, null); String database = originalConfig.getString(InfluxDBKey.DATABASE, null); if (StringUtils.isAllBlank(querySql,database)) { throw AddaxException.asAddaxException( - InfluxDBReaderErrorCode.REQUIRED_VALUE, + REQUIRED_VALUE, "One of database or querySql must be specified" ); } if (columns == null || columns.isEmpty()) { throw AddaxException.asAddaxException( - InfluxDBReaderErrorCode.REQUIRED_VALUE, + REQUIRED_VALUE, "The parameter [" + InfluxDBKey.COLUMN + "] is not set."); } } diff --git a/plugin/reader/influxdbreader/src/main/java/com/wgzhao/addax/plugin/reader/influxdbreader/InfluxDBReaderErrorCode.java b/plugin/reader/influxdbreader/src/main/java/com/wgzhao/addax/plugin/reader/influxdbreader/InfluxDBReaderErrorCode.java deleted file mode 100644 index 4d9ba4307..000000000 --- a/plugin/reader/influxdbreader/src/main/java/com/wgzhao/addax/plugin/reader/influxdbreader/InfluxDBReaderErrorCode.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.reader.influxdbreader; - -import com.wgzhao.addax.common.spi.ErrorCode; - -public enum InfluxDBReaderErrorCode - implements ErrorCode -{ - REQUIRED_VALUE("InfluxDBReader-00", "缺失必要的值"), - ILLEGAL_VALUE("InfluxDBReader-01", "值非法"); - - private final String code; - private final String description; - - InfluxDBReaderErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s]. ", this.code, - this.description); - } -} diff --git a/plugin/reader/influxdbreader/src/main/java/com/wgzhao/addax/plugin/reader/influxdbreader/InfluxDBReaderTask.java b/plugin/reader/influxdbreader/src/main/java/com/wgzhao/addax/plugin/reader/influxdbreader/InfluxDBReaderTask.java index a8b3d5783..e907fa6fc 100644 --- a/plugin/reader/influxdbreader/src/main/java/com/wgzhao/addax/plugin/reader/influxdbreader/InfluxDBReaderTask.java +++ b/plugin/reader/influxdbreader/src/main/java/com/wgzhao/addax/plugin/reader/influxdbreader/InfluxDBReaderTask.java @@ -39,6 +39,8 @@ import java.time.ZoneOffset; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; + public class InfluxDBReaderTask { private static final Logger LOG = LoggerFactory.getLogger(InfluxDBReaderTask.class); @@ -101,12 +103,12 @@ public void startRead(RecordSender recordSender, TaskPluginCollector taskPluginC } catch (Exception e) { throw AddaxException.asAddaxException( - InfluxDBReaderErrorCode.ILLEGAL_VALUE, "Failed to get data point！", e); + ILLEGAL_VALUE, "Failed to get data point！", e); } if (StringUtils.isBlank(result)) { throw AddaxException.asAddaxException( - InfluxDBReaderErrorCode.ILLEGAL_VALUE, "Get nothing!", null); + ILLEGAL_VALUE, "Get nothing!", null); } try { JSONObject jsonObject = JSONObject.parseObject(result); @@ -134,12 +136,12 @@ public void startRead(RecordSender recordSender, TaskPluginCollector taskPluginC } else if (resultsMap.containsKey("error")) { throw AddaxException.asAddaxException( - InfluxDBReaderErrorCode.ILLEGAL_VALUE, "Error occurred in data sets！", null); + ILLEGAL_VALUE, "Error occurred in data sets！", null); } } catch (Exception e) { throw AddaxException.asAddaxException( - InfluxDBReaderErrorCode.ILLEGAL_VALUE, "Failed to send data", e); + ILLEGAL_VALUE, "Failed to send data", e); } } diff --git a/plugin/reader/jsonfilereader/src/main/java/com/wgzhao/addax/plugin/reader/jsonfilereader/JsonReader.java b/plugin/reader/jsonfilereader/src/main/java/com/wgzhao/addax/plugin/reader/jsonfilereader/JsonReader.java index 200c2156e..79d5daf89 100644 --- a/plugin/reader/jsonfilereader/src/main/java/com/wgzhao/addax/plugin/reader/jsonfilereader/JsonReader.java +++ b/plugin/reader/jsonfilereader/src/main/java/com/wgzhao/addax/plugin/reader/jsonfilereader/JsonReader.java @@ -58,6 +58,12 @@ import java.util.ArrayList; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.ENCODING_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.IO_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.NOT_SUPPORT_TYPE; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; + /** * Created by jin.zhang on 18-05-30. */ @@ -87,10 +93,10 @@ private void validateParameter() { // Compatible with the old version, path is a string before String pathInString = this.originConfig.getNecessaryValue(Key.PATH, - JsonReaderErrorCode.REQUIRED_VALUE); + REQUIRED_VALUE); if (StringUtils.isBlank(pathInString)) { throw AddaxException.asAddaxException( - JsonReaderErrorCode.REQUIRED_VALUE, + REQUIRED_VALUE, "您需要指定待读取的源目录或文件"); } if (!pathInString.startsWith("[") && !pathInString.endsWith("]")) { @@ -101,7 +107,7 @@ private void validateParameter() path = this.originConfig.getList(Key.PATH, String.class); if (null == path || path.isEmpty()) { throw AddaxException.asAddaxException( - JsonReaderErrorCode.REQUIRED_VALUE, + REQUIRED_VALUE, "您需要指定待读取的源目录或文件"); } } @@ -118,12 +124,12 @@ private void validateParameter() } catch (UnsupportedCharsetException uce) { throw AddaxException.asAddaxException( - JsonReaderErrorCode.ILLEGAL_VALUE, + NOT_SUPPORT_TYPE, String.format("不支持您配置的编码格式 : [%s]", encoding), uce); } catch (Exception e) { throw AddaxException.asAddaxException( - JsonReaderErrorCode.CONFIG_INVALID_EXCEPTION, + ENCODING_ERROR, String.format("编码配置异常, 请联系我们: %s", e.getMessage()), e); } @@ -135,16 +141,16 @@ private void validateParameter() if (null != columns && !columns.isEmpty()) { for (Configuration eachColumnConf : columns) { - eachColumnConf.getNecessaryValue(Key.TYPE, JsonReaderErrorCode.REQUIRED_VALUE); + eachColumnConf.getNecessaryValue(Key.TYPE, REQUIRED_VALUE); String columnIndex = eachColumnConf.getString(Key.INDEX); String columnValue = eachColumnConf.getString(Key.VALUE); if (null == columnIndex && null == columnValue) { - throw AddaxException.asAddaxException(JsonReaderErrorCode.NO_INDEX_VALUE, "由于您配置了type, 则至少需要配置 index 或 value"); + throw AddaxException.asAddaxException(CONFIG_ERROR, "由于您配置了type, 则至少需要配置 index 或 value"); } if (null != columnIndex && null != columnValue) { - throw AddaxException.asAddaxException(JsonReaderErrorCode.MIXED_INDEX_VALUE, "您混合配置了index, value, 每一列同时仅能选择其中一种"); + throw AddaxException.asAddaxException(CONFIG_ERROR, "您混合配置了index, value, 每一列同时仅能选择其中一种"); } } } @@ -183,7 +189,7 @@ public List split(int adviceNumber) int splitNumber = this.sourceFiles.size(); if (0 == splitNumber) { throw AddaxException.asAddaxException( - JsonReaderErrorCode.EMPTY_DIR_EXCEPTION, + CONFIG_ERROR, String.format("NOT find any file in your path: %s", originConfig.getString(Key.PATH))); } @@ -305,7 +311,7 @@ private Column getColumn(String type, String columnValue, String columnFormat) default: String errorMessage = String.format("The type %s is unsupported", type); LOG.error(errorMessage); - throw AddaxException.asAddaxException(JsonReaderErrorCode.NOT_SUPPORT_TYPE, errorMessage); + throw AddaxException.asAddaxException(NOT_SUPPORT_TYPE, errorMessage); } return columnGenerated; } @@ -353,7 +359,7 @@ public void startRead(RecordSender recordSender) // warn: sock 文件无法read,能影响所有文件的传输,需要用户自己保证 String message = String.format("The file %s not found", fileName); LOG.error(message); - throw AddaxException.asAddaxException(JsonReaderErrorCode.OPEN_FILE_ERROR, message); + throw AddaxException.asAddaxException(CONFIG_ERROR, message); } try { if (compressType != null) { @@ -388,7 +394,7 @@ public void startRead(RecordSender recordSender) // warn: 有可能本地无法读取文件 String message = String.format("Failed to open file %s", fileName); LOG.error(message); - throw AddaxException.asAddaxException(JsonReaderErrorCode.READ_FILE_IO_ERROR, message); + throw AddaxException.asAddaxException(IO_ERROR, message); } finally { IOUtils.closeQuietly(reader, null); diff --git a/plugin/reader/jsonfilereader/src/main/java/com/wgzhao/addax/plugin/reader/jsonfilereader/JsonReaderErrorCode.java b/plugin/reader/jsonfilereader/src/main/java/com/wgzhao/addax/plugin/reader/jsonfilereader/JsonReaderErrorCode.java deleted file mode 100644 index 4c0f7941c..000000000 --- a/plugin/reader/jsonfilereader/src/main/java/com/wgzhao/addax/plugin/reader/jsonfilereader/JsonReaderErrorCode.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.reader.jsonfilereader; - -import com.wgzhao.addax.common.spi.ErrorCode; - -public enum JsonReaderErrorCode - implements ErrorCode -{ - REQUIRED_VALUE("JsonFilereader-00", "Missing required value"), - ILLEGAL_VALUE("JsonFilereader-01", "Illegal value"), - MIXED_INDEX_VALUE("JsonFilereader-02", "Both configure index and value."), - NO_INDEX_VALUE("JsonFilereader-03", "You specify columns, but not configure index and value "), - FILE_NOT_EXISTS("JsonFilereader-04", "Directory not exists."), - OPEN_FILE_ERROR("JsonFilereader-06", "Failed to open the file."), - READ_FILE_IO_ERROR("JsonFilereader-07", "IOException occurred when open file"), - SECURITY_NOT_ENOUGH("JsonFilereader-08", "Permission denied"), - CONFIG_INVALID_EXCEPTION("JsonFilereader-09", "incorrect configure."), - NOT_SUPPORT_TYPE("JsonFilereader-10", "The type is unsupported."), - EMPTY_DIR_EXCEPTION("JsonFilereader-11", "Empty directory"), - ; - - private final String code; - private final String description; - - JsonReaderErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s].", this.code, this.description); - } -} - - - diff --git a/plugin/reader/kafkareader/src/main/java/com/wgzhao/addax/plugin/reader/kafkareader/KafkaReader.java b/plugin/reader/kafkareader/src/main/java/com/wgzhao/addax/plugin/reader/kafkareader/KafkaReader.java index 77511398c..473f924fb 100644 --- a/plugin/reader/kafkareader/src/main/java/com/wgzhao/addax/plugin/reader/kafkareader/KafkaReader.java +++ b/plugin/reader/kafkareader/src/main/java/com/wgzhao/addax/plugin/reader/kafkareader/KafkaReader.java @@ -26,6 +26,9 @@ import java.util.Map; import java.util.Properties; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; + public class KafkaReader extends Reader { @@ -38,8 +41,8 @@ public static class Job public void init() { this.conf = getPluginJobConf(); - conf.getNecessaryValue(KafkaKey.BROKER_LIST, KafkaReaderErrorCode.REQUIRED_VALUE); - conf.getNecessaryValue(KafkaKey.TOPIC, KafkaReaderErrorCode.REQUIRED_VALUE); + conf.getNecessaryValue(KafkaKey.BROKER_LIST, REQUIRED_VALUE); + conf.getNecessaryValue(KafkaKey.TOPIC, REQUIRED_VALUE); } @Override @@ -118,7 +121,7 @@ public void startRead(RecordSender recordSender) for (String col : columns) { if (!jsonObject.containsKey(col)) { if (this.missKeyValue == null) { - throw AddaxException.asAddaxException(KafkaReaderErrorCode.NOT_MATCHED_COLUMNS, + throw AddaxException.asAddaxException(CONFIG_ERROR, "The column " + col + " not exists"); } record.addColumn(new StringColumn(this.missKeyValue)); diff --git a/plugin/reader/kafkareader/src/main/java/com/wgzhao/addax/plugin/reader/kafkareader/KafkaReaderErrorCode.java b/plugin/reader/kafkareader/src/main/java/com/wgzhao/addax/plugin/reader/kafkareader/KafkaReaderErrorCode.java deleted file mode 100644 index 027e79abc..000000000 --- a/plugin/reader/kafkareader/src/main/java/com/wgzhao/addax/plugin/reader/kafkareader/KafkaReaderErrorCode.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.reader.kafkareader; - -import com.wgzhao.addax.common.spi.ErrorCode; - -public enum KafkaReaderErrorCode - implements ErrorCode -{ - REQUIRED_VALUE("KafkaReader-00", "You are missing a required parameter value."), - ILLEGAL_VALUE("KafkaReader-01", "You fill in the parameter values are not legitimate."), - GET_KAFKA_CONNECTION_ERROR("KafkaReader-02", "Error getting KAFKA connection."), - GET_KAFKA_TABLE_ERROR("KafkaReader-03", "Error getting KAFKA table."), - CLOSE_KAFKA_CONNECTION_ERROR("KafkaReader-04", "Error closing KAFKA connection."), - CLOSE_KAFKA_SESSION_ERROR("KafkaReader-06", "Error closing KAFKA table connection."), - PUT_KAFKA_ERROR("KafkaReader-07", "IO exception occurred when writing to KAFKA."), - DELETE_KAFKA_ERROR("KafkaReader-08", "An exception occurred while delete KAFKA table."), - CREATE_KAFKA_TABLE_ERROR("KafkaReader-09", "Error creating KAFKA table."), - PARAMETER_NUM_ERROR("KafkaReader-10", "The number of parameters does not match."), - TABLE_NOT_EXISTS("KafkaReader-11", "The table you specified does not exists yet"), - COLUMN_NOT_EXISTS("KafkaReader-12", "the column doest not exists"), - NOT_MATCHED_COLUMNS("KafkaReader-13", "the number of columns does not match the record"); - - private final String code; - private final String description; - - KafkaReaderErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return code; - } - - @Override - public String getDescription() - { - return description; - } -} diff --git a/plugin/reader/kudureader/src/main/java/com/wgzhao/addax/plugin/reader/kudureader/KuduReader.java b/plugin/reader/kudureader/src/main/java/com/wgzhao/addax/plugin/reader/kudureader/KuduReader.java index 9ad2e1df0..c7f541aea 100644 --- a/plugin/reader/kudureader/src/main/java/com/wgzhao/addax/plugin/reader/kudureader/KuduReader.java +++ b/plugin/reader/kudureader/src/main/java/com/wgzhao/addax/plugin/reader/kudureader/KuduReader.java @@ -54,6 +54,9 @@ import static com.wgzhao.addax.common.base.Key.COLUMN; import static com.wgzhao.addax.common.base.Key.WHERE; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.NOT_SUPPORT_TYPE; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; /** * Kudu reader plugin @@ -140,12 +143,12 @@ public void prepare() result.add(conf); } else { - throw AddaxException.asAddaxException(KuduReaderErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(NOT_SUPPORT_TYPE, "operator '" + matcher.group(2) + "' is unsupported"); } } else { - throw AddaxException.asAddaxException(KuduReaderErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "Illegal where clause: " + w); } } @@ -210,7 +213,7 @@ public void startRead(RecordSender recordSender) } catch (KuduException ex) { throw AddaxException.asAddaxException( - KuduReaderErrorCode.UNKNOWN_EXCEPTION, + RUNTIME_ERROR, ex.getMessage() ); } @@ -243,7 +246,7 @@ public void startRead(RecordSender recordSender) for (String column : columns) { if (!schema.hasColumn(column)) { throw AddaxException.asAddaxException( - KuduReaderErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, "column '" + column + "' does not exists in the table '" + tableName + "'" ); } @@ -270,7 +273,7 @@ public void startRead(RecordSender recordSender) } catch (KuduException ex) { throw AddaxException.asAddaxException( - KuduReaderErrorCode.UNKNOWN_EXCEPTION, + RUNTIME_ERROR, ex.getMessage() ); } @@ -345,7 +348,7 @@ public void startRead(RecordSender recordSender) } catch (KuduException ex) { throw AddaxException.asAddaxException( - KuduReaderErrorCode.UNKNOWN_EXCEPTION, + RUNTIME_ERROR, ex.getMessage() ); } @@ -422,7 +425,7 @@ private List processWhere(List where, Schema schem op = KUDU_OPERATORS.get(conf.getString("op")); if (!schema.hasColumn(field)) { throw AddaxException.asAddaxException( - KuduReaderErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, "column '" + field + "' in where clause does not exists in the table '" + tableName + "'" ); } @@ -477,7 +480,7 @@ public void destroy() } catch (KuduException ex) { throw AddaxException.asAddaxException( - KuduReaderErrorCode.UNKNOWN_EXCEPTION, + RUNTIME_ERROR, ex.getMessage() ); } diff --git a/plugin/reader/kudureader/src/main/java/com/wgzhao/addax/plugin/reader/kudureader/KuduReaderErrorCode.java b/plugin/reader/kudureader/src/main/java/com/wgzhao/addax/plugin/reader/kudureader/KuduReaderErrorCode.java deleted file mode 100644 index 998283c00..000000000 --- a/plugin/reader/kudureader/src/main/java/com/wgzhao/addax/plugin/reader/kudureader/KuduReaderErrorCode.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.reader.kudureader; - -import com.wgzhao.addax.common.spi.ErrorCode; - -/** - * Created by roy on 2019/12/12 1543. - */ -public enum KuduReaderErrorCode implements ErrorCode { - - ILLEGAL_VALUE("Illegal parameter value","参数不合法"), - ILLEGAL_ADDRESS("Illegal address","不合法的Kudu Master Addresses"), - UNKNOWN_EXCEPTION("Unknown exception","未知异常"); - - private final String code; - - private final String description; - - KuduReaderErrorCode(String code, String description) { - this.code = code; - this.description = description; - } - - @Override - public String getCode() { - return code; - } - - @Override - public String getDescription() { - return description; - } -} - diff --git a/plugin/reader/mongodbreader/src/main/java/com/wgzhao/addax/plugin/reader/mongodbreader/MongoDBReader.java b/plugin/reader/mongodbreader/src/main/java/com/wgzhao/addax/plugin/reader/mongodbreader/MongoDBReader.java index 7373e9c34..c3630b3db 100644 --- a/plugin/reader/mongodbreader/src/main/java/com/wgzhao/addax/plugin/reader/mongodbreader/MongoDBReader.java +++ b/plugin/reader/mongodbreader/src/main/java/com/wgzhao/addax/plugin/reader/mongodbreader/MongoDBReader.java @@ -53,6 +53,8 @@ import static com.wgzhao.addax.common.base.Key.FETCH_SIZE; import static com.wgzhao.addax.common.base.Key.PASSWORD; import static com.wgzhao.addax.common.base.Key.USERNAME; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; public class MongoDBReader extends Reader @@ -82,7 +84,7 @@ public void init() { this.originalConfig = getPluginJobConf(); // check required configuration - String userName = originalConfig.getNecessaryValue(USERNAME, MongoDBReaderErrorCode.REQUIRED_VALUE); + String userName = originalConfig.getNecessaryValue(USERNAME, REQUIRED_VALUE); String password = originalConfig.getString(PASSWORD); if (password != null && password.startsWith(Constant.ENC_PASSWORD_PREFIX)) { // encrypted password, need to decrypt @@ -90,12 +92,12 @@ public void init() originalConfig.set(Key.PASSWORD, password); } Configuration connConf = Configuration.from(originalConfig.getList(CONNECTION, Object.class).get(0).toString()); - String database = connConf.getNecessaryValue(DATABASE, MongoDBReaderErrorCode.REQUIRED_VALUE); + String database = connConf.getNecessaryValue(DATABASE, REQUIRED_VALUE); String authDb = connConf.getString(KeyConstant.MONGO_AUTH_DB, database); List addressList = connConf.getList(KeyConstant.MONGO_ADDRESS, Object.class); List columns = originalConfig.getList(COLUMN, String.class); if (columns == null || (columns.size() == 1 && "*".equals(columns.get(0)))) { - throw AddaxException.asAddaxException(MongoDBReaderErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "The configuration column must be required and DOES NOT support \"*\" yet"); } if (notNullAndEmpty((userName)) && notNullAndEmpty((password))) { @@ -142,8 +144,8 @@ public void startRead(RecordSender recordSender) if (lowerBound == null || upperBound == null || mongoClient == null || database == null || collection == null || mongodbColumnMeta == null) { - throw AddaxException.asAddaxException(MongoDBReaderErrorCode.ILLEGAL_VALUE, - MongoDBReaderErrorCode.ILLEGAL_VALUE.getDescription()); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, + ILLEGAL_VALUE.getDescription()); } MongoDatabase db = mongoClient.getDatabase(database); MongoCollection col = db.getCollection(this.collection); diff --git a/plugin/reader/mongodbreader/src/main/java/com/wgzhao/addax/plugin/reader/mongodbreader/MongoDBReaderErrorCode.java b/plugin/reader/mongodbreader/src/main/java/com/wgzhao/addax/plugin/reader/mongodbreader/MongoDBReaderErrorCode.java deleted file mode 100644 index 0bfa2dc53..000000000 --- a/plugin/reader/mongodbreader/src/main/java/com/wgzhao/addax/plugin/reader/mongodbreader/MongoDBReaderErrorCode.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.reader.mongodbreader; - -import com.wgzhao.addax.common.spi.ErrorCode; - -/** - * Created by jianying.wcj on 2015/3/19 0019. - */ -public enum MongoDBReaderErrorCode - implements ErrorCode -{ - - ILLEGAL_VALUE("ILLEGAL_PARAMETER_VALUE", "参数不合法"), - ILLEGAL_ADDRESS("ILLEGAL_ADDRESS", "不合法的Mongo地址"), - UNKNOWN_EXCEPTION("UNKNOWN_EXCEPTION", "未知异常"), - REQUIRED_VALUE("REQUIRED_VALUE_EXCEPTION","Missing required parameters"); - - private final String code; - - private final String description; - - MongoDBReaderErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return code; - } - - @Override - public String getDescription() - { - return description; - } -} - diff --git a/plugin/reader/mongodbreader/src/main/java/com/wgzhao/addax/plugin/reader/mongodbreader/util/CollectionSplitUtil.java b/plugin/reader/mongodbreader/src/main/java/com/wgzhao/addax/plugin/reader/mongodbreader/util/CollectionSplitUtil.java index bfbcdfc16..d55cb4cd6 100644 --- a/plugin/reader/mongodbreader/src/main/java/com/wgzhao/addax/plugin/reader/mongodbreader/util/CollectionSplitUtil.java +++ b/plugin/reader/mongodbreader/src/main/java/com/wgzhao/addax/plugin/reader/mongodbreader/util/CollectionSplitUtil.java @@ -27,7 +27,6 @@ import com.wgzhao.addax.common.exception.AddaxException; import com.wgzhao.addax.common.util.Configuration; import com.wgzhao.addax.plugin.reader.mongodbreader.KeyConstant; -import com.wgzhao.addax.plugin.reader.mongodbreader.MongoDBReaderErrorCode; import org.bson.Document; import org.bson.types.ObjectId; @@ -36,6 +35,7 @@ import static com.wgzhao.addax.common.base.Key.CONNECTION; import static com.wgzhao.addax.common.base.Key.DATABASE; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; public class CollectionSplitUtil { @@ -53,8 +53,8 @@ public static List doSplit(Configuration originalSliceConfig, int String collName = connConf.getString(KeyConstant.MONGO_COLLECTION_NAME); if (null == dbName || dbName.isEmpty() || null == collName || collName.isEmpty() || mongoClient == null) { - throw AddaxException.asAddaxException(MongoDBReaderErrorCode.ILLEGAL_VALUE, - MongoDBReaderErrorCode.ILLEGAL_VALUE.getDescription()); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, + ILLEGAL_VALUE.getDescription()); } boolean isObjectId = isPrimaryIdObjectId(mongoClient, dbName, collName); diff --git a/plugin/reader/mongodbreader/src/main/java/com/wgzhao/addax/plugin/reader/mongodbreader/util/MongoUtil.java b/plugin/reader/mongodbreader/src/main/java/com/wgzhao/addax/plugin/reader/mongodbreader/util/MongoUtil.java index 4035d0458..1c754ffdc 100644 --- a/plugin/reader/mongodbreader/src/main/java/com/wgzhao/addax/plugin/reader/mongodbreader/util/MongoUtil.java +++ b/plugin/reader/mongodbreader/src/main/java/com/wgzhao/addax/plugin/reader/mongodbreader/util/MongoUtil.java @@ -23,7 +23,6 @@ import com.mongodb.client.MongoClient; import com.mongodb.client.MongoClients; import com.wgzhao.addax.common.exception.AddaxException; -import com.wgzhao.addax.plugin.reader.mongodbreader.MongoDBReaderErrorCode; import com.mongodb.MongoCredential; import com.mongodb.ServerAddress; @@ -31,6 +30,9 @@ import java.util.ArrayList; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; + /** * Created by jianying.wcj on 2015/3/17 0017. * Modified by mingyan.zc on 2016/6/13. @@ -49,7 +51,7 @@ public static MongoClient initCredentialMongoClient(List addressList, St { if (!isHostPortPattern(addressList)) { - throw AddaxException.asAddaxException(MongoDBReaderErrorCode.ILLEGAL_VALUE, "不合法参数"); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "不合法参数"); } try { MongoCredential credential = null; @@ -62,7 +64,7 @@ public static MongoClient initCredentialMongoClient(List addressList, St builder.hosts(parseServerAddress(addressList)); } catch (UnknownHostException e) { - throw AddaxException.asAddaxException(MongoDBReaderErrorCode.ILLEGAL_ADDRESS, "不合法的地址"); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "不合法的地址"); } }); if (credential != null) { @@ -72,10 +74,10 @@ public static MongoClient initCredentialMongoClient(List addressList, St } catch (NumberFormatException e) { - throw AddaxException.asAddaxException(MongoDBReaderErrorCode.ILLEGAL_VALUE, "不合法参数"); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "不合法参数"); } catch (Exception e) { - throw AddaxException.asAddaxException(MongoDBReaderErrorCode.UNKNOWN_EXCEPTION, "未知异常"); + throw AddaxException.asAddaxException(RUNTIME_ERROR, "未知异常"); } } diff --git a/plugin/reader/mysqlreader/src/main/java/com/wgzhao/addax/plugin/reader/mysqlreader/MysqlReaderErrorCode.java b/plugin/reader/mysqlreader/src/main/java/com/wgzhao/addax/plugin/reader/mysqlreader/MysqlReaderErrorCode.java deleted file mode 100644 index f619bc4a3..000000000 --- a/plugin/reader/mysqlreader/src/main/java/com/wgzhao/addax/plugin/reader/mysqlreader/MysqlReaderErrorCode.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.reader.mysqlreader; - -import com.wgzhao.addax.common.spi.ErrorCode; - -public enum MysqlReaderErrorCode - implements ErrorCode -{ - ; - - private final String code; - private final String description; - - MysqlReaderErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s]. ", this.code, - this.description); - } -} diff --git a/plugin/reader/oraclereader/src/main/java/com/wgzhao/addax/plugin/reader/oraclereader/OracleReader.java b/plugin/reader/oraclereader/src/main/java/com/wgzhao/addax/plugin/reader/oraclereader/OracleReader.java index fa648c9a3..bd0cabd31 100644 --- a/plugin/reader/oraclereader/src/main/java/com/wgzhao/addax/plugin/reader/oraclereader/OracleReader.java +++ b/plugin/reader/oraclereader/src/main/java/com/wgzhao/addax/plugin/reader/oraclereader/OracleReader.java @@ -30,7 +30,6 @@ import com.wgzhao.addax.common.util.Configuration; import com.wgzhao.addax.rdbms.reader.CommonRdbmsReader; import com.wgzhao.addax.rdbms.reader.util.HintUtil; -import com.wgzhao.addax.rdbms.util.DBUtilErrorCode; import com.wgzhao.addax.rdbms.util.DataBaseType; import oracle.spatial.geometry.JGeometry; import org.apache.commons.lang3.StringUtils; @@ -45,6 +44,8 @@ import static com.wgzhao.addax.common.base.Constant.DEFAULT_FETCH_SIZE; import static com.wgzhao.addax.common.base.Key.FETCH_SIZE; import static com.wgzhao.addax.common.base.Key.IS_TABLE_MODE; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; public class OracleReader extends Reader @@ -100,7 +101,7 @@ private void dealFetchSize(Configuration originalConfig) { int fetchSize = originalConfig.getInt(FETCH_SIZE, DEFAULT_FETCH_SIZE); if (fetchSize < 1) { - throw AddaxException.asAddaxException(DBUtilErrorCode.REQUIRED_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "The value of fetchSize [" + fetchSize + "] in OracleReader can not be less than 1."); } originalConfig.set(FETCH_SIZE, fetchSize); @@ -112,7 +113,7 @@ private void dealHint(Configuration originalConfig) if (StringUtils.isNotBlank(hint)) { boolean isTableMode = originalConfig.getBool(IS_TABLE_MODE); if (!isTableMode) { - throw AddaxException.asAddaxException(OracleReaderErrorCode.HINT_ERROR, + throw AddaxException.asAddaxException(CONFIG_ERROR, "Only querySql mode can configure HINT, please set isTableMode to false."); } HintUtil.initHintConf(DATABASE_TYPE, originalConfig); diff --git a/plugin/reader/oraclereader/src/main/java/com/wgzhao/addax/plugin/reader/oraclereader/OracleReaderErrorCode.java b/plugin/reader/oraclereader/src/main/java/com/wgzhao/addax/plugin/reader/oraclereader/OracleReaderErrorCode.java deleted file mode 100644 index a25f24825..000000000 --- a/plugin/reader/oraclereader/src/main/java/com/wgzhao/addax/plugin/reader/oraclereader/OracleReaderErrorCode.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.reader.oraclereader; - -import com.wgzhao.addax.common.spi.ErrorCode; - -public enum OracleReaderErrorCode - implements ErrorCode -{ - HINT_ERROR("Oraclereader-00", "您的 Hint 配置出错."), - - ; - - private final String code; - private final String description; - - OracleReaderErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s]. ", this.code, - this.description); - } -} diff --git a/plugin/reader/postgresqlreader/src/main/java/com/wgzhao/addax/plugin/reader/postgresqlreader/PostgresqlReader.java b/plugin/reader/postgresqlreader/src/main/java/com/wgzhao/addax/plugin/reader/postgresqlreader/PostgresqlReader.java index 39a6026b4..c237bc3ad 100644 --- a/plugin/reader/postgresqlreader/src/main/java/com/wgzhao/addax/plugin/reader/postgresqlreader/PostgresqlReader.java +++ b/plugin/reader/postgresqlreader/src/main/java/com/wgzhao/addax/plugin/reader/postgresqlreader/PostgresqlReader.java @@ -26,7 +26,6 @@ import com.wgzhao.addax.common.spi.Reader; import com.wgzhao.addax.common.util.Configuration; import com.wgzhao.addax.rdbms.reader.CommonRdbmsReader; -import com.wgzhao.addax.rdbms.util.DBUtilErrorCode; import com.wgzhao.addax.rdbms.util.DataBaseType; import java.io.UnsupportedEncodingException; @@ -38,6 +37,7 @@ import static com.wgzhao.addax.common.base.Constant.DEFAULT_FETCH_SIZE; import static com.wgzhao.addax.common.base.Key.FETCH_SIZE; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; public class PostgresqlReader extends Reader @@ -57,7 +57,7 @@ public void init() this.originalConfig = super.getPluginJobConf(); int fetchSize = this.originalConfig.getInt(FETCH_SIZE, DEFAULT_FETCH_SIZE); if (fetchSize < 1) { - throw AddaxException.asAddaxException(DBUtilErrorCode.REQUIRED_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, String.format("您配置的fetchSize有误，fetchSize : [%d] 设置值不能小于 1.", fetchSize)); } this.originalConfig.set(FETCH_SIZE, fetchSize); diff --git a/plugin/reader/rdbmsreader/src/main/java/com/wgzhao/addax/plugin/reader/rdbmsreader/RdbmsReader.java b/plugin/reader/rdbmsreader/src/main/java/com/wgzhao/addax/plugin/reader/rdbmsreader/RdbmsReader.java index e4049db93..9eb0eb3e8 100644 --- a/plugin/reader/rdbmsreader/src/main/java/com/wgzhao/addax/plugin/reader/rdbmsreader/RdbmsReader.java +++ b/plugin/reader/rdbmsreader/src/main/java/com/wgzhao/addax/plugin/reader/rdbmsreader/RdbmsReader.java @@ -25,7 +25,6 @@ import com.wgzhao.addax.common.spi.Reader; import com.wgzhao.addax.common.util.Configuration; import com.wgzhao.addax.rdbms.reader.CommonRdbmsReader; -import com.wgzhao.addax.rdbms.util.DBUtilErrorCode; import com.wgzhao.addax.rdbms.util.DataBaseType; import org.apache.commons.lang3.StringUtils; @@ -36,6 +35,7 @@ import static com.wgzhao.addax.common.base.Key.CONNECTION; import static com.wgzhao.addax.common.base.Key.FETCH_SIZE; import static com.wgzhao.addax.common.base.Key.JDBC_DRIVER; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; public class RdbmsReader extends Reader @@ -56,13 +56,13 @@ public void init() int fetchSize = this.originalConfig.getInt(FETCH_SIZE, DEFAULT_FETCH_SIZE); if (fetchSize < 1) { throw AddaxException.asAddaxException( - DBUtilErrorCode.REQUIRED_VALUE, + REQUIRED_VALUE, String.format("您配置的fetchSize有误，fetchSize : [%d] 设置值不能小于 1.", fetchSize)); } this.originalConfig.set(FETCH_SIZE, fetchSize); Configuration connection = this.originalConfig.getListConfiguration(CONNECTION).get(0); if (connection == null) { - throw AddaxException.asAddaxException(DBUtilErrorCode.REQUIRED_VALUE, "config 'connection' is required and must not be " + + throw AddaxException.asAddaxException(REQUIRED_VALUE, "config 'connection' is required and must not be " + "empty"); } String jdbcDriver = this.originalConfig.getString(JDBC_DRIVER, null); diff --git a/plugin/reader/redisreader/src/main/java/com/wgzhao/addax/plugin/reader/redisreader/RedisErrorCode.java b/plugin/reader/redisreader/src/main/java/com/wgzhao/addax/plugin/reader/redisreader/RedisErrorCode.java deleted file mode 100644 index a14377590..000000000 --- a/plugin/reader/redisreader/src/main/java/com/wgzhao/addax/plugin/reader/redisreader/RedisErrorCode.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.wgzhao.addax.plugin.reader.redisreader; - -import com.wgzhao.addax.common.spi.ErrorCode; - -public enum RedisErrorCode - implements ErrorCode -{ - REQUIRED_VALUE("RedisReader-00","The required item is missing"), - ILLEGAL_VALUE("RedisReader-02","The value is illegal"); - - private final String code; - - private final String describe; - - RedisErrorCode(String code, String describe) - { - this.code = code; - this.describe = describe; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.describe; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Describe:[%s]", this.code, this.describe); - } -} diff --git a/plugin/reader/redisreader/src/main/java/com/wgzhao/addax/plugin/reader/redisreader/RedisReader.java b/plugin/reader/redisreader/src/main/java/com/wgzhao/addax/plugin/reader/redisreader/RedisReader.java index c4c091c67..1694b8ae9 100644 --- a/plugin/reader/redisreader/src/main/java/com/wgzhao/addax/plugin/reader/redisreader/RedisReader.java +++ b/plugin/reader/redisreader/src/main/java/com/wgzhao/addax/plugin/reader/redisreader/RedisReader.java @@ -82,6 +82,8 @@ import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_ZSET; import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_ZSET_2; import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_ZSET_ZIPLIST; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; public class RedisReader extends Reader @@ -105,15 +107,15 @@ private void validateParam() Configuration conConf = Configuration.from(connection.toString()); String uri = conConf.getString(RedisKey.URI); if (uri == null || uri.isEmpty()) { - throw AddaxException.asAddaxException(RedisErrorCode.REQUIRED_VALUE, "uri is null or empty"); + throw AddaxException.asAddaxException(REQUIRED_VALUE, "uri is null or empty"); } if (!(uri.startsWith("tcp") || uri.startsWith("file") || uri.startsWith("http") || uri.startsWith("https"))) { - throw AddaxException.asAddaxException(RedisErrorCode.ILLEGAL_VALUE, "uri is not start with tcp, file, http or https"); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "uri is not start with tcp, file, http or https"); } String mode = conConf.getString(RedisKey.MODE, "standalone"); if ("sentinel".equalsIgnoreCase(mode)) { // required other items - conConf.getNecessaryValue(RedisKey.MASTER_NAME, RedisErrorCode.REQUIRED_VALUE); + conConf.getNecessaryValue(RedisKey.MASTER_NAME, REQUIRED_VALUE); } } } diff --git a/plugin/reader/s3reader/src/main/java/com/wgzhao/addax/plugin/reader/s3reader/S3Reader.java b/plugin/reader/s3reader/src/main/java/com/wgzhao/addax/plugin/reader/s3reader/S3Reader.java index ded14dc1b..5cce13c05 100644 --- a/plugin/reader/s3reader/src/main/java/com/wgzhao/addax/plugin/reader/s3reader/S3Reader.java +++ b/plugin/reader/s3reader/src/main/java/com/wgzhao/addax/plugin/reader/s3reader/S3Reader.java @@ -7,14 +7,10 @@ import com.wgzhao.addax.common.util.Configuration; import com.wgzhao.addax.storage.reader.StorageReaderUtil; import org.apache.commons.io.Charsets; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.GetObjectRequest; -import software.amazon.awssdk.services.s3.model.ListObjectsRequest; -import software.amazon.awssdk.services.s3.model.ListObjectsResponse; import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; import software.amazon.awssdk.services.s3.model.NoSuchKeyException; @@ -26,7 +22,10 @@ import java.util.List; import java.util.regex.Pattern; -import static com.wgzhao.addax.plugin.reader.s3reader.S3ReaderErrorCode.REQUIRED_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; public class S3Reader extends Reader @@ -63,11 +62,11 @@ private void validate() Charsets.toCharset(encoding); } catch (UnsupportedCharsetException uce) { - throw AddaxException.asAddaxException(S3ReaderErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, String.format("unsupported encoding : [%s]", encoding), uce); } catch (Exception e) { - throw AddaxException.asAddaxException(S3ReaderErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, String.format("Runtime Error : %s", e.getMessage()), e); } @@ -83,7 +82,7 @@ private void validate() if (null == columns || columns.isEmpty()) { throw AddaxException.asAddaxException( - S3ReaderErrorCode.CONFIG_INVALID_EXCEPTION, + REQUIRED_VALUE, "The item column is required"); } @@ -94,13 +93,13 @@ private void validate() if (null == columnIndex && null == columnValue) { throw AddaxException.asAddaxException( - S3ReaderErrorCode.NO_INDEX_VALUE, + CONFIG_ERROR, "You configured type, also configured index or value"); } if (null != columnIndex && null != columnValue) { throw AddaxException.asAddaxException( - S3ReaderErrorCode.MIXED_INDEX_VALUE, + CONFIG_ERROR, "You configured both index and value"); } } @@ -126,7 +125,7 @@ public List split(int adviceNumber) List objects = parseOriginObjects(readerOriginConfig.getList(S3Key.OBJECT, String.class)); if (objects.isEmpty()) { throw AddaxException.asAddaxException( - S3ReaderErrorCode.EMPTY_BUCKET_EXCEPTION, + RUNTIME_ERROR, String.format( "The object %s in bucket %s is not found", this.readerOriginConfig.get(S3Key.OBJECT), diff --git a/plugin/reader/s3reader/src/main/java/com/wgzhao/addax/plugin/reader/s3reader/S3ReaderErrorCode.java b/plugin/reader/s3reader/src/main/java/com/wgzhao/addax/plugin/reader/s3reader/S3ReaderErrorCode.java deleted file mode 100644 index f69c487e2..000000000 --- a/plugin/reader/s3reader/src/main/java/com/wgzhao/addax/plugin/reader/s3reader/S3ReaderErrorCode.java +++ /dev/null @@ -1,45 +0,0 @@ -package com.wgzhao.addax.plugin.reader.s3reader; - -import com.wgzhao.addax.common.spi.ErrorCode; - -public enum S3ReaderErrorCode - implements ErrorCode -{ - S3_EXCEPTION("S3FileReader-01", "Exception occurred when reading configure"), - CONFIG_INVALID_EXCEPTION("S3FileReader-02", "Invalid configure"), - NOT_SUPPORT_TYPE("S3Reader-03", "Non-supported type"), - SECURITY_EXCEPTION("S3Reader-05", "Permission denied"), - ILLEGAL_VALUE("S3Reader-06", "Illegal value"), - REQUIRED_VALUE("S3Reader-07", "Missing required value"), - NO_INDEX_VALUE("S3Reader-08", "Missing index"), - MIXED_INDEX_VALUE("S3Reader-09", "Mix index and value"), - EMPTY_BUCKET_EXCEPTION("S3Reader-10", "Empty bucket"), - OBJECT_NOT_EXIST("S3Reader-11", "Object does not exists"); - - private final String code; - private final String description; - - S3ReaderErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s].", this.code, this.description); - } -} diff --git a/plugin/reader/s3reader/src/main/java/com/wgzhao/addax/plugin/reader/s3reader/S3Util.java b/plugin/reader/s3reader/src/main/java/com/wgzhao/addax/plugin/reader/s3reader/S3Util.java index 633f75841..461c07c08 100644 --- a/plugin/reader/s3reader/src/main/java/com/wgzhao/addax/plugin/reader/s3reader/S3Util.java +++ b/plugin/reader/s3reader/src/main/java/com/wgzhao/addax/plugin/reader/s3reader/S3Util.java @@ -9,6 +9,8 @@ import java.net.URI; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; + public class S3Util { public static S3Client initS3Client(Configuration conf) { @@ -26,7 +28,7 @@ public static S3Client initS3Client(Configuration conf) { .build(); } catch (IllegalArgumentException e) { throw AddaxException.asAddaxException( - S3ReaderErrorCode.ILLEGAL_VALUE, e.getMessage()); + ILLEGAL_VALUE, e.getMessage()); } } } diff --git a/plugin/reader/sqlitereader/src/main/java/com/wgzhao/addax/plugin/reader/sqlitereader/SqliteReaderErrorCode.java b/plugin/reader/sqlitereader/src/main/java/com/wgzhao/addax/plugin/reader/sqlitereader/SqliteReaderErrorCode.java deleted file mode 100644 index 060aceb59..000000000 --- a/plugin/reader/sqlitereader/src/main/java/com/wgzhao/addax/plugin/reader/sqlitereader/SqliteReaderErrorCode.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.reader.sqlitereader; - -import com.wgzhao.addax.common.spi.ErrorCode; - -public enum SqliteReaderErrorCode - implements ErrorCode -{ - ; - private final String code; - private final String description; - - SqliteReaderErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s]. ", this.code, this.description); - } -} diff --git a/plugin/reader/sqlserverreader/src/main/java/com/wgzhao/addax/plugin/reader/sqlserverreader/SqlServerReader.java b/plugin/reader/sqlserverreader/src/main/java/com/wgzhao/addax/plugin/reader/sqlserverreader/SqlServerReader.java index 641340903..f1192424e 100644 --- a/plugin/reader/sqlserverreader/src/main/java/com/wgzhao/addax/plugin/reader/sqlserverreader/SqlServerReader.java +++ b/plugin/reader/sqlserverreader/src/main/java/com/wgzhao/addax/plugin/reader/sqlserverreader/SqlServerReader.java @@ -27,7 +27,6 @@ import com.wgzhao.addax.common.spi.Reader; import com.wgzhao.addax.common.util.Configuration; import com.wgzhao.addax.rdbms.reader.CommonRdbmsReader; -import com.wgzhao.addax.rdbms.util.DBUtilErrorCode; import com.wgzhao.addax.rdbms.util.DataBaseType; import java.io.UnsupportedEncodingException; @@ -39,6 +38,7 @@ import static com.wgzhao.addax.common.base.Key.FETCH_SIZE; import static com.wgzhao.addax.common.base.Constant.DEFAULT_FETCH_SIZE; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; public class SqlServerReader extends Reader @@ -60,7 +60,7 @@ public void init() int fetchSize = this.originalConfig.getInt(FETCH_SIZE, DEFAULT_FETCH_SIZE); if (fetchSize < 1) { throw AddaxException - .asAddaxException(DBUtilErrorCode.REQUIRED_VALUE, + .asAddaxException(ILLEGAL_VALUE, String.format("您配置的fetchSize有误，fetchSize : [%d] 设置值不能小于 1.", fetchSize)); } this.originalConfig.set(FETCH_SIZE, fetchSize); diff --git a/plugin/reader/streamreader/src/main/java/com/wgzhao/addax/plugin/reader/streamreader/StreamReader.java b/plugin/reader/streamreader/src/main/java/com/wgzhao/addax/plugin/reader/streamreader/StreamReader.java index 575cf616a..55cd74f32 100644 --- a/plugin/reader/streamreader/src/main/java/com/wgzhao/addax/plugin/reader/streamreader/StreamReader.java +++ b/plugin/reader/streamreader/src/main/java/com/wgzhao/addax/plugin/reader/streamreader/StreamReader.java @@ -52,6 +52,10 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.NOT_SUPPORT_TYPE; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; + public class StreamReader extends Reader { @@ -91,11 +95,11 @@ public void init() Long sliceRecordCount = this.originalConfig .getLong(Key.SLICE_RECORD_COUNT); if (null == sliceRecordCount) { - throw AddaxException.asAddaxException(StreamReaderErrorCode.REQUIRED_VALUE, + throw AddaxException.asAddaxException(REQUIRED_VALUE, "The item sliceRecordCount is required."); } else if (sliceRecordCount < 1) { - throw AddaxException.asAddaxException(StreamReaderErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "The value of item sliceRecordCount must be greater than 0."); } } @@ -105,7 +109,7 @@ private void dealColumn(Configuration originalConfig) List columns = originalConfig.getList(Key.COLUMN, JSONObject.class); if (null == columns || columns.isEmpty()) { - throw AddaxException.asAddaxException(StreamReaderErrorCode.REQUIRED_VALUE, + throw AddaxException.asAddaxException(REQUIRED_VALUE, "The item column is required."); } @@ -116,7 +120,7 @@ private void dealColumn(Configuration originalConfig) this.parseMixupFunctions(eachColumnConfig); } catch (Exception e) { - throw AddaxException.asAddaxException(StreamReaderErrorCode.NOT_SUPPORT_TYPE, + throw AddaxException.asAddaxException(NOT_SUPPORT_TYPE, String.format("Failed to parse mixup functions [%s]", e.getMessage()), e); } @@ -135,7 +139,7 @@ private void dealColumn(Configuration originalConfig) } if (!Type.isTypeIllegal(typeName)) { throw AddaxException.asAddaxException( - StreamReaderErrorCode.NOT_SUPPORT_TYPE, + NOT_SUPPORT_TYPE, String.format("The [%s] is unsupported.", typeName)); } } @@ -175,7 +179,7 @@ private void parseMixupFunctions(Configuration eachColumnConfig) String columnRandom = eachColumnConfig.getString(StreamConstant.RANDOM); String columnIncr = eachColumnConfig.getString(StreamConstant.INCR); if (StringUtils.isBlank(columnRandom) && StringUtils.isBlank(columnIncr)) { - eachColumnConfig.getNecessaryValue(Key.VALUE, StreamReaderErrorCode.REQUIRED_VALUE); + eachColumnConfig.getNecessaryValue(Key.VALUE, REQUIRED_VALUE); } if (StringUtils.isNotBlank(columnIncr)) { // 类型判断 @@ -194,7 +198,7 @@ private void parseMixupFunctions(Configuration eachColumnConfig) } catch (NumberFormatException e) { throw AddaxException.asAddaxException( - StreamReaderErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, "The value of must be numeric, value [" + columnValue + "] is not valid." ); } @@ -209,7 +213,7 @@ else if (fields.length == 2) { Integer.parseInt(fields[1]); } catch (NumberFormatException e) { throw AddaxException.asAddaxException( - StreamReaderErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, "The second field must be numeric, value [" + fields[1] + "] is not valid" ); } @@ -225,7 +229,7 @@ else if (fields.length == 2) { } else { throw AddaxException.asAddaxException( - StreamReaderErrorCode.NOT_SUPPORT_TYPE, + NOT_SUPPORT_TYPE, "The increment sequence must be long or date, value [" + dType + "] is not valid." ); } @@ -247,7 +251,7 @@ else if (fields.length == 2) { String[] split = columnRandom.split(","); if (split.length < 2) { throw AddaxException.asAddaxException( - StreamReaderErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, String.format("Illegal random value [%s], supported random value like 'minVal, MaxVal[,scale]'", columnRandom)); } @@ -257,7 +261,7 @@ else if (fields.length == 2) { long param2Int; if (StringUtils.isBlank(param1) && StringUtils.isBlank(param2)) { throw AddaxException.asAddaxException( - StreamReaderErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, "The random function's params can not be empty."); } @@ -273,7 +277,7 @@ else if (fields.length == 2) { } catch (ParseException e) { throw AddaxException.asAddaxException( - StreamReaderErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, String.format("The random function's params [%s,%s] does not match the dateFormat[%s].", dateFormat, param1, param2), e); } @@ -284,13 +288,13 @@ else if (fields.length == 2) { } if (param1Int < 0 || param2Int < 0) { throw AddaxException.asAddaxException( - StreamReaderErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, String.format("The random function's params [%s,%s] can not be negative.", param1, param2)); } if (!Type.BOOL.name().equalsIgnoreCase(typeName) && param1Int > param2Int) { throw AddaxException.asAddaxException( - StreamReaderErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, String.format("The random function's params [%s,%s] is not valid, the first param must be less than the second one.", param1, param2)); } @@ -328,7 +332,7 @@ private void validateDateIncrUnit(String unit) } if (!isOK) { throw AddaxException.asAddaxException( - StreamReaderErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, unit + " is NOT valid interval unit，for more details, please refer to the documentation"); } } @@ -498,7 +502,7 @@ else if (isIncr) { currVal = sdf.parse(fields[0]); } catch (java.text.ParseException e) { throw AddaxException.asAddaxException( - StreamReaderErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, String.format("can not parse date value [%s] with date format [%s]", fields[0], datePattern) ); } @@ -507,7 +511,7 @@ else if (isIncr) { return new DateColumn((Date)currVal); } else { throw AddaxException.asAddaxException( - StreamReaderErrorCode.NOT_SUPPORT_TYPE, + NOT_SUPPORT_TYPE, columnType + " can not support for increment" ); } @@ -584,7 +588,7 @@ private Record buildOneRecord(RecordSender recordSender, } } catch (Exception e) { - throw AddaxException.asAddaxException(StreamReaderErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "Failed to build record.", e); } return record; diff --git a/plugin/reader/streamreader/src/main/java/com/wgzhao/addax/plugin/reader/streamreader/StreamReaderErrorCode.java b/plugin/reader/streamreader/src/main/java/com/wgzhao/addax/plugin/reader/streamreader/StreamReaderErrorCode.java deleted file mode 100644 index 43ecc4af0..000000000 --- a/plugin/reader/streamreader/src/main/java/com/wgzhao/addax/plugin/reader/streamreader/StreamReaderErrorCode.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.reader.streamreader; - -import com.wgzhao.addax.common.spi.ErrorCode; - -public enum StreamReaderErrorCode - implements ErrorCode -{ - REQUIRED_VALUE("StreamReader-00", "缺失必要的值"), - ILLEGAL_VALUE("StreamReader-01", "值非法"), - NOT_SUPPORT_TYPE("StreamReader-02", "不支持的column类型"), - ; - - private final String code; - private final String description; - - StreamReaderErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s]. ", this.code, - this.description); - } -} diff --git a/plugin/reader/tdenginereader/src/main/java/com/wgzhao/addax/plugin/reader/tdenginereader/TDengineReader.java b/plugin/reader/tdenginereader/src/main/java/com/wgzhao/addax/plugin/reader/tdenginereader/TDengineReader.java index 6ce48b0d5..430d0dec1 100644 --- a/plugin/reader/tdenginereader/src/main/java/com/wgzhao/addax/plugin/reader/tdenginereader/TDengineReader.java +++ b/plugin/reader/tdenginereader/src/main/java/com/wgzhao/addax/plugin/reader/tdenginereader/TDengineReader.java @@ -37,6 +37,8 @@ import static com.wgzhao.addax.common.base.Constant.DEFAULT_DATE_FORMAT; import static com.wgzhao.addax.common.base.Constant.DEFAULT_FETCH_SIZE; import static com.wgzhao.addax.common.base.Key.FETCH_SIZE; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; public class TDengineReader extends Reader @@ -69,32 +71,32 @@ public void preCheck() // check beginDateTime String beginDatetime = this.originalConfig.getString(TDKey.BEGIN_DATETIME); if (StringUtils.isBlank(beginDatetime)) { - throw AddaxException.asAddaxException(TDengineReaderErrorCode.REQUIRED_VALUE, "The parameter [" + TDKey.BEGIN_DATETIME + "] is not set."); + throw AddaxException.asAddaxException(REQUIRED_VALUE, "The parameter [" + TDKey.BEGIN_DATETIME + "] is not set."); } long start; try { start = format.parse(beginDatetime).getTime(); } catch (ParseException e) { - throw AddaxException.asAddaxException(TDengineReaderErrorCode.ILLEGAL_VALUE, "The parameter [" + TDKey.BEGIN_DATETIME + + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "The parameter [" + TDKey.BEGIN_DATETIME + "] needs to conform to the [" + DEFAULT_DATE_FORMAT + "] format."); } // check endDateTime String endDatetime = this.originalConfig.getString(TDKey.END_DATETIME); if (StringUtils.isBlank(endDatetime)) { - throw AddaxException.asAddaxException(TDengineReaderErrorCode.REQUIRED_VALUE, "The parameter [" + TDKey.END_DATETIME + "] is not set."); + throw AddaxException.asAddaxException(REQUIRED_VALUE, "The parameter [" + TDKey.END_DATETIME + "] is not set."); } long end; try { end = format.parse(endDatetime).getTime(); } catch (ParseException e) { - throw AddaxException.asAddaxException(TDengineReaderErrorCode.ILLEGAL_VALUE, "The parameter [" + TDKey.END_DATETIME + "] " + + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "The parameter [" + TDKey.END_DATETIME + "] " + "needs to conform to the [" + DEFAULT_DATE_FORMAT + "] format."); } if (start >= end) { - throw AddaxException.asAddaxException(TDengineReaderErrorCode.ILLEGAL_VALUE, "The parameter [" + TDKey.BEGIN_DATETIME + + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "The parameter [" + TDKey.BEGIN_DATETIME + "] should be less than the parameter [" + TDKey.END_DATETIME + "]."); } @@ -102,14 +104,14 @@ public void preCheck() String splitInterval = this.originalConfig.getString(TDKey.SPLIT_INTERVAL); Long split; if (StringUtils.isBlank(splitInterval)) { - throw AddaxException.asAddaxException(TDengineReaderErrorCode.REQUIRED_VALUE, "The parameter [" + TDKey.SPLIT_INTERVAL + + throw AddaxException.asAddaxException(REQUIRED_VALUE, "The parameter [" + TDKey.SPLIT_INTERVAL + "] is not set."); } try { split = parseSplitInterval(splitInterval); } catch (Exception e) { - throw AddaxException.asAddaxException(TDengineReaderErrorCode.ILLEGAL_VALUE, "The parameter [" + TDKey.SPLIT_INTERVAL + + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "The parameter [" + TDKey.SPLIT_INTERVAL + "] should be like: \"123d|h|m|s\", error: " + e.getMessage()); } diff --git a/plugin/reader/tdenginereader/src/main/java/com/wgzhao/addax/plugin/reader/tdenginereader/TDengineReaderErrorCode.java b/plugin/reader/tdenginereader/src/main/java/com/wgzhao/addax/plugin/reader/tdenginereader/TDengineReaderErrorCode.java deleted file mode 100644 index e07f517fb..000000000 --- a/plugin/reader/tdenginereader/src/main/java/com/wgzhao/addax/plugin/reader/tdenginereader/TDengineReaderErrorCode.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.reader.tdenginereader; - -import com.wgzhao.addax.common.spi.ErrorCode; - -public enum TDengineReaderErrorCode - implements ErrorCode -{ - REQUIRED_VALUE("TDengineReader-00", "Missing required value"), - ILLEGAL_VALUE("TDengineReader-01", "Illegal value"), - CONNECTION_FAILED("TDengineReader-02", "Connection failure"); - - private final String code; - private final String description; - - TDengineReaderErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s]. ", this.code, this.description); - } -} diff --git a/plugin/reader/txtfilereader/src/main/java/com/wgzhao/addax/plugin/reader/txtfilereader/TxtFileReader.java b/plugin/reader/txtfilereader/src/main/java/com/wgzhao/addax/plugin/reader/txtfilereader/TxtFileReader.java index 25a306733..594be2fad 100644 --- a/plugin/reader/txtfilereader/src/main/java/com/wgzhao/addax/plugin/reader/txtfilereader/TxtFileReader.java +++ b/plugin/reader/txtfilereader/src/main/java/com/wgzhao/addax/plugin/reader/txtfilereader/TxtFileReader.java @@ -40,6 +40,11 @@ import java.util.Arrays; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.IO_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; + /** * Created by haiwei.luo on 14-9-20. */ @@ -67,9 +72,9 @@ public void prepare() { LOG.debug("prepare() begin..."); // Compatible with the old version, path is a string before - String pathInString = this.originConfig.getNecessaryValue(Key.PATH, TxtFileReaderErrorCode.REQUIRED_VALUE); + String pathInString = this.originConfig.getNecessaryValue(Key.PATH, REQUIRED_VALUE); if (StringUtils.isBlank(pathInString)) { - throw AddaxException.asAddaxException(TxtFileReaderErrorCode.REQUIRED_VALUE, "the path is required"); + throw AddaxException.asAddaxException(REQUIRED_VALUE, "the path is required"); } List path; if (!pathInString.startsWith("[") && !pathInString.endsWith("]")) { @@ -79,7 +84,7 @@ public void prepare() else { path = this.originConfig.getList(Key.PATH, String.class); if (null == path || path.isEmpty()) { - throw AddaxException.asAddaxException(TxtFileReaderErrorCode.REQUIRED_VALUE, "the path is required"); + throw AddaxException.asAddaxException(REQUIRED_VALUE, "the path is required"); } } @@ -123,7 +128,7 @@ public List split(int adviceNumber) int splitNumber = this.sourceFiles.size(); if (0 == splitNumber) { throw AddaxException.asAddaxException( - TxtFileReaderErrorCode.EMPTY_DIR_EXCEPTION, String + CONFIG_ERROR, String .format("未能找到待读取的文件,请确认您的配置项path: %s", this.originConfig.getString(Key.PATH))); } @@ -146,7 +151,7 @@ private int getIndexByName(String name, String[] allNames) } } throw AddaxException.asAddaxException( - TxtFileReaderErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, "The name '" + name + "' DOES NOT exists in file header: " + Arrays.toString(allNames) ); } @@ -224,7 +229,7 @@ public void startRead(RecordSender recordSender) } catch (FileNotFoundException e) { throw AddaxException.asAddaxException( - TxtFileReaderErrorCode.OPEN_FILE_ERROR, + IO_ERROR, "Open file '" + fileName + "' failure" ); } diff --git a/plugin/reader/txtfilereader/src/main/java/com/wgzhao/addax/plugin/reader/txtfilereader/TxtFileReaderErrorCode.java b/plugin/reader/txtfilereader/src/main/java/com/wgzhao/addax/plugin/reader/txtfilereader/TxtFileReaderErrorCode.java deleted file mode 100644 index d736662cc..000000000 --- a/plugin/reader/txtfilereader/src/main/java/com/wgzhao/addax/plugin/reader/txtfilereader/TxtFileReaderErrorCode.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.reader.txtfilereader; - -import com.wgzhao.addax.common.spi.ErrorCode; - -/** - * Created by haiwei.luo on 14-9-20. - */ -public enum TxtFileReaderErrorCode - implements ErrorCode -{ - REQUIRED_VALUE("TxtFileReader-00", "您缺失了必须填写的参数值."), - ILLEGAL_VALUE("TxtFileReader-01", "您填写的参数值不合法."), - MIXED_INDEX_VALUE("TxtFileReader-02", "您的列信息配置同时包含了index,value."), - NO_INDEX_VALUE("TxtFileReader-03", "您明确的配置列信息,但未填写相应的index,value."), - FILE_NOT_EXISTS("TxtFileReader-04", "您配置的目录文件路径不存在."), -// OPEN_FILE_WITH_CHARSET_ERROR("TxtFileReader-05", "您配置的文件编码和实际文件编码不符合."), - OPEN_FILE_ERROR("TxtFileReader-06", "您配置的文件在打开时异常,建议您检查源目录是否有隐藏文件,管道文件等特殊文件."), -// READ_FILE_IO_ERROR("TxtFileReader-07", "您配置的文件在读取时出现IO异常."), - SECURITY_NOT_ENOUGH("TxtFileReader-08", "您缺少权限执行相应的文件操作."), - CONFIG_INVALID_EXCEPTION("TxtFileReader-09", "您的参数配置错误."), -// RUNTIME_EXCEPTION("TxtFileReader-10", "出现运行时异常, 请联系我们"), - EMPTY_DIR_EXCEPTION("TxtFileReader-11", "您尝试读取的文件目录为空."), - NOT_SUPPORT_TYPE("TxtFileReader-12", "您配置的列类型暂不支持."), - OPEN_FILE_WITH_CHARSET_ERROR("TxtFileReader-13", "您配置的编码和实际存储编码不符合."), - // OPEN_FILE_ERROR("UnstructuredStorageReader-08", "您配置的源在打开时异常,建议您检查源源是否有隐藏实体,管道文件等特殊文件."), - READ_FILE_IO_ERROR("TxtFileReader-14", "您配置的文件在读取时出现IO异常."), - RUNTIME_EXCEPTION("TxtFileReader-15", "出现运行时异常, 请联系我们") - ; - - private final String code; - private final String description; - - TxtFileReaderErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s].", this.code, - this.description); - } -} diff --git a/plugin/writer/cassandrawriter/src/main/java/com/wgzhao/addax/plugin/writer/cassandrawriter/CassandraWriter.java b/plugin/writer/cassandrawriter/src/main/java/com/wgzhao/addax/plugin/writer/cassandrawriter/CassandraWriter.java index f90c21870..2b9c94924 100644 --- a/plugin/writer/cassandrawriter/src/main/java/com/wgzhao/addax/plugin/writer/cassandrawriter/CassandraWriter.java +++ b/plugin/writer/cassandrawriter/src/main/java/com/wgzhao/addax/plugin/writer/cassandrawriter/CassandraWriter.java @@ -48,6 +48,8 @@ import java.util.concurrent.TimeUnit; import static com.datastax.driver.core.querybuilder.QueryBuilder.timestamp; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.EXECUTE_FAIL; /** * Created by mazhenlin on 2019/8/19. @@ -109,7 +111,7 @@ public void startWrite(RecordReceiver lineReceiver) // 源头读取字段列数与目的表字段写入列数不相等，直接报错 throw AddaxException .asAddaxException( - CassandraWriterErrorCode.CONF_ERROR, + CONFIG_ERROR, String.format( "列配置信息有错误. 因为您配置的任务中，源头读取字段数:%s 与 目的表要写入的字段数:%s 不相等. 请检查您的配置并作出修改.", record.getColumnNumber(), @@ -180,7 +182,7 @@ public void startWrite(RecordReceiver lineReceiver) } catch (Exception e) { throw AddaxException.asAddaxException( - CassandraWriterErrorCode.WRITE_DATA_ERROR, e); + EXECUTE_FAIL, e); } } @@ -229,7 +231,7 @@ public void init() if (writeTimeCol != -1) { throw AddaxException .asAddaxException( - CassandraWriterErrorCode.CONF_ERROR, + CONFIG_ERROR, "列配置信息有错误. 只能有一个时间戳列(writetime())"); } writeTimeCol = columnTypes.size(); @@ -240,7 +242,7 @@ public void init() if (col == null) { throw AddaxException .asAddaxException( - CassandraWriterErrorCode.CONF_ERROR, + CONFIG_ERROR, String.format( "列配置信息有错误. 表中未找到列名 '%s' .", colunmnName)); diff --git a/plugin/writer/cassandrawriter/src/main/java/com/wgzhao/addax/plugin/writer/cassandrawriter/CassandraWriterErrorCode.java b/plugin/writer/cassandrawriter/src/main/java/com/wgzhao/addax/plugin/writer/cassandrawriter/CassandraWriterErrorCode.java deleted file mode 100644 index 387fc165f..000000000 --- a/plugin/writer/cassandrawriter/src/main/java/com/wgzhao/addax/plugin/writer/cassandrawriter/CassandraWriterErrorCode.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.writer.cassandrawriter; - -import com.wgzhao.addax.common.spi.ErrorCode; - -/** - * Created by mazhenlin on 2019/8/19. - */ -public enum CassandraWriterErrorCode - implements ErrorCode -{ - CONF_ERROR("CassandraWriter-00", "配置错误."), - WRITE_DATA_ERROR("CassandraWriter-01", "写入数据时失败."), - ; - - private final String code; - private final String description; - - CassandraWriterErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s].", this.code, this.description); - } -} diff --git a/plugin/writer/cassandrawriter/src/main/java/com/wgzhao/addax/plugin/writer/cassandrawriter/CassandraWriterHelper.java b/plugin/writer/cassandrawriter/src/main/java/com/wgzhao/addax/plugin/writer/cassandrawriter/CassandraWriterHelper.java index 9df473d20..d7586f8f4 100644 --- a/plugin/writer/cassandrawriter/src/main/java/com/wgzhao/addax/plugin/writer/cassandrawriter/CassandraWriterHelper.java +++ b/plugin/writer/cassandrawriter/src/main/java/com/wgzhao/addax/plugin/writer/cassandrawriter/CassandraWriterHelper.java @@ -49,6 +49,9 @@ import java.util.Set; import java.util.UUID; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.NOT_SUPPORT_TYPE; + /** * Created by mazhenlin on 2019/8/21. */ @@ -147,7 +150,7 @@ public static Object parseFromString(String s, DataType sqlType) return parseFromJson(jsonObject, sqlType); default: - throw AddaxException.asAddaxException(CassandraWriterErrorCode.CONF_ERROR, + throw AddaxException.asAddaxException(CONFIG_ERROR, "不支持您配置的列类型:" + sqlType + ", 请检查您的配置 或者 联系 管理员."); } // end switch } @@ -355,7 +358,7 @@ public static void setupColumn(BoundStatement ps, int pos, DataType sqlType, Col break; default: - throw AddaxException.asAddaxException(CassandraWriterErrorCode.CONF_ERROR, + throw AddaxException.asAddaxException(NOT_SUPPORT_TYPE, "不支持您配置的列类型:" + sqlType + ", 请检查您的配置 或者 联系 管理员."); } // end switch } diff --git a/plugin/writer/cassandrawriter/src/main/java/com/wgzhao/addax/plugin/writer/cassandrawriter/LocalStrings.properties b/plugin/writer/cassandrawriter/src/main/java/com/wgzhao/addax/plugin/writer/cassandrawriter/LocalStrings.properties deleted file mode 100644 index d68d24403..000000000 --- a/plugin/writer/cassandrawriter/src/main/java/com/wgzhao/addax/plugin/writer/cassandrawriter/LocalStrings.properties +++ /dev/null @@ -1,21 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -errorcode.config_invalid_exception=\u914D\u7F6E\u9519\u8BEF. -errorcode.write_failed_exception=\u5199\u5165\u6570\u636E\u65F6\u5931\u8D25 \ No newline at end of file diff --git a/plugin/writer/cassandrawriter/src/main/java/com/wgzhao/addax/plugin/writer/cassandrawriter/LocalStrings_en_US.properties b/plugin/writer/cassandrawriter/src/main/java/com/wgzhao/addax/plugin/writer/cassandrawriter/LocalStrings_en_US.properties deleted file mode 100644 index 9550cd09f..000000000 --- a/plugin/writer/cassandrawriter/src/main/java/com/wgzhao/addax/plugin/writer/cassandrawriter/LocalStrings_en_US.properties +++ /dev/null @@ -1,21 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -errorcode.config_invalid_exception=Error in parameter configuration. -errorcode.write_failed_exception=\u5199\u5165\u6570\u636E\u65F6\u5931\u8D25 \ No newline at end of file diff --git a/plugin/writer/cassandrawriter/src/main/java/com/wgzhao/addax/plugin/writer/cassandrawriter/LocalStrings_ja_JP.properties b/plugin/writer/cassandrawriter/src/main/java/com/wgzhao/addax/plugin/writer/cassandrawriter/LocalStrings_ja_JP.properties deleted file mode 100644 index d68d24403..000000000 --- a/plugin/writer/cassandrawriter/src/main/java/com/wgzhao/addax/plugin/writer/cassandrawriter/LocalStrings_ja_JP.properties +++ /dev/null @@ -1,21 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -errorcode.config_invalid_exception=\u914D\u7F6E\u9519\u8BEF. -errorcode.write_failed_exception=\u5199\u5165\u6570\u636E\u65F6\u5931\u8D25 \ No newline at end of file diff --git a/plugin/writer/cassandrawriter/src/main/java/com/wgzhao/addax/plugin/writer/cassandrawriter/LocalStrings_zh_CN.properties b/plugin/writer/cassandrawriter/src/main/java/com/wgzhao/addax/plugin/writer/cassandrawriter/LocalStrings_zh_CN.properties deleted file mode 100644 index d68d24403..000000000 --- a/plugin/writer/cassandrawriter/src/main/java/com/wgzhao/addax/plugin/writer/cassandrawriter/LocalStrings_zh_CN.properties +++ /dev/null @@ -1,21 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -errorcode.config_invalid_exception=\u914D\u7F6E\u9519\u8BEF. -errorcode.write_failed_exception=\u5199\u5165\u6570\u636E\u65F6\u5931\u8D25 \ No newline at end of file diff --git a/plugin/writer/cassandrawriter/src/main/java/com/wgzhao/addax/plugin/writer/cassandrawriter/LocalStrings_zh_HK.properties b/plugin/writer/cassandrawriter/src/main/java/com/wgzhao/addax/plugin/writer/cassandrawriter/LocalStrings_zh_HK.properties deleted file mode 100644 index d68d24403..000000000 --- a/plugin/writer/cassandrawriter/src/main/java/com/wgzhao/addax/plugin/writer/cassandrawriter/LocalStrings_zh_HK.properties +++ /dev/null @@ -1,21 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -errorcode.config_invalid_exception=\u914D\u7F6E\u9519\u8BEF. -errorcode.write_failed_exception=\u5199\u5165\u6570\u636E\u65F6\u5931\u8D25 \ No newline at end of file diff --git a/plugin/writer/cassandrawriter/src/main/java/com/wgzhao/addax/plugin/writer/cassandrawriter/LocalStrings_zh_TW.properties b/plugin/writer/cassandrawriter/src/main/java/com/wgzhao/addax/plugin/writer/cassandrawriter/LocalStrings_zh_TW.properties deleted file mode 100644 index d68d24403..000000000 --- a/plugin/writer/cassandrawriter/src/main/java/com/wgzhao/addax/plugin/writer/cassandrawriter/LocalStrings_zh_TW.properties +++ /dev/null @@ -1,21 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -errorcode.config_invalid_exception=\u914D\u7F6E\u9519\u8BEF. -errorcode.write_failed_exception=\u5199\u5165\u6570\u636E\u65F6\u5931\u8D25 \ No newline at end of file diff --git a/plugin/writer/dbfwriter/src/main/java/com/wgzhao/addax/plugin/writer/dbfwriter/DbfWriter.java b/plugin/writer/dbfwriter/src/main/java/com/wgzhao/addax/plugin/writer/dbfwriter/DbfWriter.java index 2ff55b506..acf9b598d 100644 --- a/plugin/writer/dbfwriter/src/main/java/com/wgzhao/addax/plugin/writer/dbfwriter/DbfWriter.java +++ b/plugin/writer/dbfwriter/src/main/java/com/wgzhao/addax/plugin/writer/dbfwriter/DbfWriter.java @@ -49,6 +49,14 @@ import java.util.Objects; import java.util.Set; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.EXECUTE_FAIL; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.IO_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.PERMISSION_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; + /** * Created by haiwei.luo on 14-9-17. */ @@ -76,29 +84,29 @@ public void init() { } private void validateParameter() { - this.writerSliceConfig.getNecessaryValue(Key.FILE_NAME, DbfWriterErrorCode.REQUIRED_VALUE); + this.writerSliceConfig.getNecessaryValue(Key.FILE_NAME, REQUIRED_VALUE); - String path = this.writerSliceConfig.getNecessaryValue(Key.PATH, DbfWriterErrorCode.REQUIRED_VALUE); + String path = this.writerSliceConfig.getNecessaryValue(Key.PATH, REQUIRED_VALUE); try { // warn: 这里用户需要配一个目录 File dir = new File(path); if (dir.isFile()) { throw AddaxException.asAddaxException( - DbfWriterErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, String.format("The path [%s] you configured exists ,but it is file not directory.", path)); } if (!dir.exists()) { boolean createdOk = dir.mkdirs(); if (!createdOk) { throw AddaxException.asAddaxException( - DbfWriterErrorCode.CONFIG_INVALID_EXCEPTION, + CONFIG_ERROR, String.format("Failed to create directory [%s].", path)); } } } catch (SecurityException se) { throw AddaxException.asAddaxException( - DbfWriterErrorCode.SECURITY_NOT_ENOUGH, + PERMISSION_ERROR, String.format("Permission denied while creating directory [%s].", path), se); } } @@ -129,19 +137,19 @@ public void prepare() { } catch (NullPointerException npe) { throw AddaxException .asAddaxException( - DbfWriterErrorCode.WRITE_FILE_ERROR, + RUNTIME_ERROR, String.format("NPE occurred whiling cleanup [%s].", path), npe); } catch (IllegalArgumentException iae) { throw AddaxException.asAddaxException( - DbfWriterErrorCode.SECURITY_NOT_ENOUGH, + PERMISSION_ERROR, String.format("IllegalArgumentException occurred cleanup [%s].", path)); } catch (SecurityException se) { throw AddaxException.asAddaxException( - DbfWriterErrorCode.SECURITY_NOT_ENOUGH, + PERMISSION_ERROR, String.format("Permission denied for cleaning up [%s]", path)); } catch (IOException e) { throw AddaxException.asAddaxException( - DbfWriterErrorCode.WRITE_FILE_ERROR, + IO_ERROR, String.format("IO exception occurred while cleanup [%s]", path), e); } } else if ("append".equals(writeMode)) { @@ -155,7 +163,7 @@ public void prepare() { if (dir.exists()) { if (dir.isFile()) { throw AddaxException.asAddaxException( - DbfWriterErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, String.format("The path [%s] exists, but it is file not directory.", path)); } // fileName is not null @@ -169,25 +177,25 @@ public void prepare() { } LOG.error("The following files are conflict: [{}]", StringUtils.join(allFiles, ",")); throw AddaxException.asAddaxException( - DbfWriterErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, String.format("The path [%s] is not empty.", path)); } } else { boolean createdOk = dir.mkdirs(); if (!createdOk) { throw AddaxException.asAddaxException( - DbfWriterErrorCode.CONFIG_INVALID_EXCEPTION, + EXECUTE_FAIL, String.format("Failed to create directory [%s].", path)); } } } catch (SecurityException se) { throw AddaxException.asAddaxException( - DbfWriterErrorCode.SECURITY_NOT_ENOUGH, + PERMISSION_ERROR, String.format("Permission denied for creating directory [%s]", path)); } } else { throw AddaxException.asAddaxException( - DbfWriterErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, String.format("The item writeMode only supports truncate, append and nonConflict the [%s] is not supported.", writeMode)); } @@ -197,14 +205,14 @@ public void prepare() { if ("numeric".equalsIgnoreCase(column.getString(Key.TYPE)) && (column.getString(Key.LENGTH, null) == null || column.getString(Key.SCALE, null) == null)) { throw AddaxException.asAddaxException( - DbfWriterErrorCode.CONFIG_INVALID_EXCEPTION, + CONFIG_ERROR, String.format("The numeric type both require configured item %s and %s.", Key.LENGTH, Key.SCALE) ); } if ("char".equalsIgnoreCase(column.getString(Key.TYPE)) && column.getString(Key.LENGTH, null) == null) { throw AddaxException.asAddaxException( - DbfWriterErrorCode.CONFIG_INVALID_EXCEPTION, + CONFIG_ERROR, String.format("The char type require configured item %s.", Key.LENGTH) ); } @@ -241,7 +249,7 @@ public List split(int mandatoryNumber) { allFiles = new HashSet<>(Arrays.asList(Objects.requireNonNull(dir.list()))); } catch (SecurityException se) { throw AddaxException.asAddaxException( - DbfWriterErrorCode.SECURITY_NOT_ENOUGH, + PERMISSION_ERROR, String.format("Permission denied for viewing directory [%s].", path)); } @@ -371,7 +379,7 @@ public void startWrite(RecordReceiver lineReceiver) { writer.close(); } catch (SecurityException se) { throw AddaxException.asAddaxException( - DbfWriterErrorCode.SECURITY_NOT_ENOUGH, + PERMISSION_ERROR, String.format("Permission denied for create directory [%s].", this.fileName)); } LOG.info("Writing finished"); diff --git a/plugin/writer/dbfwriter/src/main/java/com/wgzhao/addax/plugin/writer/dbfwriter/DbfWriterErrorCode.java b/plugin/writer/dbfwriter/src/main/java/com/wgzhao/addax/plugin/writer/dbfwriter/DbfWriterErrorCode.java deleted file mode 100644 index cc60496b8..000000000 --- a/plugin/writer/dbfwriter/src/main/java/com/wgzhao/addax/plugin/writer/dbfwriter/DbfWriterErrorCode.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.writer.dbfwriter; - -import com.wgzhao.addax.common.spi.ErrorCode; - -/** - * Created by haiwei.luo on 14-9-17. - */ -public enum DbfWriterErrorCode - implements ErrorCode -{ - - CONFIG_INVALID_EXCEPTION("DbfFileWriter-00", "您的参数配置错误."), - REQUIRED_VALUE("DbfFileWriter-01", "您缺失了必须填写的参数值."), - ILLEGAL_VALUE("DbfFileWriter-02", "您填写的参数值不合法."), - WRITE_FILE_ERROR("DbfFileWriter-03", "您配置的目标文件在写入时异常."), -// Write_FILE_IO_ERROR("DbfFileWriter-04", "您配置的文件在写入时出现IO异常."), - SECURITY_NOT_ENOUGH("DbfFileWriter-05", "您缺少权限执行相应的文件写入操作."); - - private final String code; - private final String description; - - DbfWriterErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s].", this.code, - this.description); - } -} diff --git a/plugin/writer/doriswriter/src/main/java/com/wgzhao/addax/plugin/writer/doriswriter/DorisKey.java b/plugin/writer/doriswriter/src/main/java/com/wgzhao/addax/plugin/writer/doriswriter/DorisKey.java index 28f69f94b..cdd74292c 100644 --- a/plugin/writer/doriswriter/src/main/java/com/wgzhao/addax/plugin/writer/doriswriter/DorisKey.java +++ b/plugin/writer/doriswriter/src/main/java/com/wgzhao/addax/plugin/writer/doriswriter/DorisKey.java @@ -24,7 +24,6 @@ import com.wgzhao.addax.common.exception.AddaxException; import com.wgzhao.addax.common.util.Configuration; import com.wgzhao.addax.rdbms.util.DBUtil; -import com.wgzhao.addax.rdbms.util.DBUtilErrorCode; import com.wgzhao.addax.rdbms.util.DataBaseType; import com.wgzhao.addax.rdbms.util.RdbmsException; @@ -34,9 +33,10 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import static com.wgzhao.addax.common.base.Constant.DEFAULT_BATCH_SIZE; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; public class DorisKey extends Key @@ -59,7 +59,6 @@ public enum StreamLoadFormat private static final String DEFAULT_LABEL_PREFIX = "addax_doris_writer_"; - private final Configuration options; private List infoSchemaColumns; @@ -72,8 +71,8 @@ public DorisKey(Configuration options) { this.options = options; Configuration conn = Configuration.from(options.getList(CONNECTION).get(0).toString()); - this.database = conn.getNecessaryValue(DATABASE, DBUtilErrorCode.REQUIRED_VALUE); - this.jdbcUrl = conn.getNecessaryValue(JDBC_URL, DBUtilErrorCode.REQUIRED_VALUE); + this.database = conn.getNecessaryValue(DATABASE, REQUIRED_VALUE); + this.jdbcUrl = conn.getNecessaryValue(JDBC_URL, REQUIRED_VALUE); this.table = conn.getList(TABLE, String.class).get(0); infoSchemaColumns = options.getList(COLUMN, String.class); @@ -209,7 +208,7 @@ private void validateStreamLoadUrl() List urlList = getLoadUrlList(); for (String host : urlList) { if (host.split(":").length < 2) { - throw AddaxException.asAddaxException(DBUtilErrorCode.CONF_ERROR, + throw AddaxException.asAddaxException(CONFIG_ERROR, "The format of loadUrl is not correct, please enter:[`fe_ip:fe_http_ip;fe_ip:fe_http_ip`]."); } } diff --git a/plugin/writer/doriswriter/src/main/java/com/wgzhao/addax/plugin/writer/doriswriter/DorisWriter.java b/plugin/writer/doriswriter/src/main/java/com/wgzhao/addax/plugin/writer/doriswriter/DorisWriter.java index 057822787..81ae6413a 100644 --- a/plugin/writer/doriswriter/src/main/java/com/wgzhao/addax/plugin/writer/doriswriter/DorisWriter.java +++ b/plugin/writer/doriswriter/src/main/java/com/wgzhao/addax/plugin/writer/doriswriter/DorisWriter.java @@ -26,7 +26,6 @@ import com.wgzhao.addax.common.spi.Writer; import com.wgzhao.addax.common.util.Configuration; import com.wgzhao.addax.rdbms.util.DBUtil; -import com.wgzhao.addax.rdbms.util.DBUtilErrorCode; import com.wgzhao.addax.rdbms.util.DataBaseType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,6 +34,8 @@ import java.util.ArrayList; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.EXECUTE_FAIL; /** * doris data writer @@ -143,7 +144,7 @@ public void startWrite(RecordReceiver recordReceiver) if (record.getColumnNumber() != options.getColumns().size()) { throw AddaxException .asAddaxException( - DBUtilErrorCode.CONF_ERROR, + CONFIG_ERROR, String.format( "There is an error in the column configuration information. " + "This is because you have configured a task where the number of fields to be read from the source:%s " + @@ -156,7 +157,7 @@ public void startWrite(RecordReceiver recordReceiver) } } catch (Exception e) { - throw AddaxException.asAddaxException(DBUtilErrorCode.WRITE_DATA_ERROR, e); + throw AddaxException.asAddaxException(EXECUTE_FAIL, e); } } @@ -167,7 +168,7 @@ public void post() writerManager.close(); } catch (Exception e) { - throw AddaxException.asAddaxException(DBUtilErrorCode.WRITE_DATA_ERROR, e); + throw AddaxException.asAddaxException(EXECUTE_FAIL, e); } } diff --git a/plugin/writer/elasticsearchwriter/src/main/java/com/wgzhao/addax/plugin/writer/elasticsearchwriter/ESKey.java b/plugin/writer/elasticsearchwriter/src/main/java/com/wgzhao/addax/plugin/writer/elasticsearchwriter/ESKey.java index 4f4e6077b..80c03eafd 100644 --- a/plugin/writer/elasticsearchwriter/src/main/java/com/wgzhao/addax/plugin/writer/elasticsearchwriter/ESKey.java +++ b/plugin/writer/elasticsearchwriter/src/main/java/com/wgzhao/addax/plugin/writer/elasticsearchwriter/ESKey.java @@ -25,6 +25,8 @@ import java.util.HashMap; import java.util.Map; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; + public final class ESKey { // ---------------------------------------- @@ -56,7 +58,7 @@ else if ("update".equals(actionType)) { public static String getEndpoint(Configuration conf) { - return conf.getNecessaryValue("endpoint", ESWriterErrorCode.BAD_CONFIG_VALUE); + return conf.getNecessaryValue("endpoint", REQUIRED_VALUE); } public static String getAccessID(Configuration conf) @@ -106,7 +108,7 @@ public static boolean isMultiThread(Configuration conf) public static String getIndexName(Configuration conf) { - return conf.getNecessaryValue("index", ESWriterErrorCode.BAD_CONFIG_VALUE); + return conf.getNecessaryValue("index", REQUIRED_VALUE); } public static String getTypeName(Configuration conf) diff --git a/plugin/writer/elasticsearchwriter/src/main/java/com/wgzhao/addax/plugin/writer/elasticsearchwriter/ESWriter.java b/plugin/writer/elasticsearchwriter/src/main/java/com/wgzhao/addax/plugin/writer/elasticsearchwriter/ESWriter.java index bb0b377c5..098f759c7 100644 --- a/plugin/writer/elasticsearchwriter/src/main/java/com/wgzhao/addax/plugin/writer/elasticsearchwriter/ESWriter.java +++ b/plugin/writer/elasticsearchwriter/src/main/java/com/wgzhao/addax/plugin/writer/elasticsearchwriter/ESWriter.java @@ -47,6 +47,11 @@ import java.util.Map; import java.util.concurrent.Callable; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.EXECUTE_FAIL; +import static com.wgzhao.addax.common.spi.ErrorCode.NOT_SUPPORT_TYPE; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; + public class ESWriter extends Writer { @@ -101,7 +106,7 @@ public void prepare() } } catch (Exception ex) { - throw AddaxException.asAddaxException(ESWriterErrorCode.ES_MAPPINGS, ex.toString()); + throw AddaxException.asAddaxException(EXECUTE_FAIL, ex.toString()); } esClient.closeJestClient(); } @@ -119,11 +124,11 @@ private String genMappings(String typeName) String colName = jo.getString("name"); String colTypeStr = jo.getString("type"); if (colTypeStr == null) { - throw AddaxException.asAddaxException(ESWriterErrorCode.BAD_CONFIG_VALUE, col.toString() + " column must have type"); + throw AddaxException.asAddaxException(CONFIG_ERROR, col.toString() + " column must have type"); } ESFieldType colType = ESFieldType.getESFieldType(colTypeStr); if (colType == null) { - throw AddaxException.asAddaxException(ESWriterErrorCode.BAD_CONFIG_VALUE, col.toString() + " unsupported type"); + throw AddaxException.asAddaxException(NOT_SUPPORT_TYPE, col.toString() + " unsupported type"); } ESColumn columnItem = new ESColumn(); @@ -199,7 +204,7 @@ private String genMappings(String typeName) mappings = JSON.toJSONString(rootMappings); if (mappings == null || "".equals(mappings)) { - throw AddaxException.asAddaxException(ESWriterErrorCode.BAD_CONFIG_VALUE, "must have mappings"); + throw AddaxException.asAddaxException(REQUIRED_VALUE, "must have mappings"); } return mappings; @@ -233,7 +238,7 @@ public void post() esClient.alias(ESKey.getIndexName(conf), alias, ESKey.isNeedCleanAlias(conf)); } catch (IOException e) { - throw AddaxException.asAddaxException(ESWriterErrorCode.ES_ALIAS_MODIFY, e); + throw AddaxException.asAddaxException(EXECUTE_FAIL, e); } } } @@ -451,12 +456,12 @@ public Integer call() for (BulkResult.BulkResultItem item : failedItems) { if (item.status != 400) { // 400 BAD_REQUEST 如果非数据异常,请求异常,则不允许忽略 - throw AddaxException.asAddaxException(ESWriterErrorCode.ES_INDEX_INSERT, String.format("status:[%d], error: %s", item.status, item.error)); + throw AddaxException.asAddaxException(EXECUTE_FAIL, String.format("status:[%d], error: %s", item.status, item.error)); } else { // 如果用户选择不忽略解析错误,则抛异常,默认为忽略 if (!ESKey.isIgnoreParseError(conf)) { - throw AddaxException.asAddaxException(ESWriterErrorCode.ES_INDEX_INSERT, String.format("status:[%d], error: %s, config not ignoreParseError so throw this error", item.status, item.error)); + throw AddaxException.asAddaxException(EXECUTE_FAIL, String.format("status:[%d], error: %s, config not ignoreParseError so throw this error", item.status, item.error)); } } } @@ -476,7 +481,7 @@ public Integer call() //TOO_MANY_REQUESTS log.warn("server response too many requests, so auto reduce speed"); } - throw AddaxException.asAddaxException(ESWriterErrorCode.ES_INDEX_INSERT, jestResult.getErrorMessage()); + throw AddaxException.asAddaxException(EXECUTE_FAIL, jestResult.getErrorMessage()); } } }, trySize, 60000L, true); @@ -486,7 +491,7 @@ public Integer call() log.warn(String.format("重试[%d]次写入失败，忽略该错误，继续写入!", trySize)); } else { - throw AddaxException.asAddaxException(ESWriterErrorCode.ES_INDEX_INSERT, e); + throw AddaxException.asAddaxException(EXECUTE_FAIL, e); } } return 0; diff --git a/plugin/writer/elasticsearchwriter/src/main/java/com/wgzhao/addax/plugin/writer/elasticsearchwriter/ESWriterErrorCode.java b/plugin/writer/elasticsearchwriter/src/main/java/com/wgzhao/addax/plugin/writer/elasticsearchwriter/ESWriterErrorCode.java deleted file mode 100644 index c2ea8221f..000000000 --- a/plugin/writer/elasticsearchwriter/src/main/java/com/wgzhao/addax/plugin/writer/elasticsearchwriter/ESWriterErrorCode.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.writer.elasticsearchwriter; - -import com.wgzhao.addax.common.spi.ErrorCode; - -public enum ESWriterErrorCode - implements ErrorCode -{ - BAD_CONFIG_VALUE("ESWriter-00", "您配置的值不合法."), -// ES_INDEX_DELETE("ESWriter-01", "删除index错误."), -// ES_INDEX_CREATE("ESWriter-02", "创建index错误."), - ES_MAPPINGS("ESWriter-03", "mappings错误."), - ES_INDEX_INSERT("ESWriter-04", "插入数据错误."), - ES_ALIAS_MODIFY("ESWriter-05", "别名修改错误."), - ; - - private final String code; - private final String description; - - ESWriterErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s]. ", this.code, - this.description); - } -} \ No newline at end of file diff --git a/plugin/writer/excelwriter/src/main/java/com/wgzhao/addax/plugin/writer/excelwriter/ExcelWriter.java b/plugin/writer/excelwriter/src/main/java/com/wgzhao/addax/plugin/writer/excelwriter/ExcelWriter.java index 84859a7fa..12aeb830c 100644 --- a/plugin/writer/excelwriter/src/main/java/com/wgzhao/addax/plugin/writer/excelwriter/ExcelWriter.java +++ b/plugin/writer/excelwriter/src/main/java/com/wgzhao/addax/plugin/writer/excelwriter/ExcelWriter.java @@ -23,6 +23,13 @@ import static com.wgzhao.addax.common.base.Key.FILE_NAME; import static com.wgzhao.addax.common.base.Key.HEADER; import static com.wgzhao.addax.common.base.Key.PATH; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.EXECUTE_FAIL; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.IO_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.NOT_SUPPORT_TYPE; +import static com.wgzhao.addax.common.spi.ErrorCode.PERMISSION_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; public class ExcelWriter extends Writer @@ -41,11 +48,11 @@ public void init() private void validateParameter() { - this.conf.getNecessaryValue(PATH, ExcelWriterErrorCode.REQUIRED_VALUE); - String path = this.conf.getNecessaryValue(PATH, ExcelWriterErrorCode.REQUIRED_VALUE); - String fileName = this.conf.getNecessaryValue(FILE_NAME, ExcelWriterErrorCode.REQUIRED_VALUE); + this.conf.getNecessaryValue(PATH, REQUIRED_VALUE); + String path = this.conf.getNecessaryValue(PATH, REQUIRED_VALUE); + String fileName = this.conf.getNecessaryValue(FILE_NAME, REQUIRED_VALUE); if (fileName.endsWith(".xls")){ - throw AddaxException.asAddaxException(ExcelWriterErrorCode.FILE_FORMAT_ERROR, "Only support new excel format file(.xlsx)"); + throw AddaxException.asAddaxException(NOT_SUPPORT_TYPE, "Only support new excel format file(.xlsx)"); } if (fileName.split("\\.").length == 1) { // no suffix ? @@ -55,18 +62,18 @@ private void validateParameter() // warn: 这里用户需要配一个目录 File dir = new File(path); if (dir.isFile()) { - throw AddaxException.asAddaxException(ExcelWriterErrorCode.ILLEGAL_VALUE, path + " is normal file instead of directory"); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, path + " is normal file instead of directory"); } if (!dir.exists()) { boolean createdOk = dir.mkdirs(); if (!createdOk) { - throw AddaxException.asAddaxException(ExcelWriterErrorCode.CONFIG_INVALID_EXCEPTION, + throw AddaxException.asAddaxException(EXECUTE_FAIL, "can not create directory '" + dir + "' failure"); } } } catch (SecurityException se) { - throw AddaxException.asAddaxException(ExcelWriterErrorCode.SECURITY_NOT_ENOUGH, + throw AddaxException.asAddaxException(PERMISSION_ERROR, "Create directory '" + path + "' failure: permission deny: ", se); } } @@ -172,10 +179,10 @@ public void startWrite(RecordReceiver lineReceiver) workbook.close(); } catch (FileNotFoundException e) { - throw AddaxException.asAddaxException(ExcelWriterErrorCode.WRITE_FILE_ERROR, "No such file: " + filePath); + throw AddaxException.asAddaxException(CONFIG_ERROR, "No such file: " + filePath); } catch (IOException e) { - throw AddaxException.asAddaxException(ExcelWriterErrorCode.WRITE_FILE_IO_ERROR, "IOException occurred while writing to " + filePath); + throw AddaxException.asAddaxException(IO_ERROR, "IOException occurred while writing to " + filePath); } } } diff --git a/plugin/writer/excelwriter/src/main/java/com/wgzhao/addax/plugin/writer/excelwriter/ExcelWriterErrorCode.java b/plugin/writer/excelwriter/src/main/java/com/wgzhao/addax/plugin/writer/excelwriter/ExcelWriterErrorCode.java deleted file mode 100644 index c8f046a99..000000000 --- a/plugin/writer/excelwriter/src/main/java/com/wgzhao/addax/plugin/writer/excelwriter/ExcelWriterErrorCode.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.writer.excelwriter; - -import com.wgzhao.addax.common.spi.ErrorCode; - -/** - * Created by haiwei.luo on 14-9-17. - */ -public enum ExcelWriterErrorCode - implements ErrorCode -{ - - CONFIG_INVALID_EXCEPTION("ExcelWriter-00", "您的参数配置错误."), - REQUIRED_VALUE("ExcelWriter-01", "您缺失了必须填写的参数值."), - ILLEGAL_VALUE("ExcelWriter-02", "您填写的参数值不合法."), - WRITE_FILE_ERROR("ExcelWriter-03", "您配置的目标文件在写入时异常."), - WRITE_FILE_IO_ERROR("ExcelWriter-04", "您配置的文件在写入时出现IO异常."), - SECURITY_NOT_ENOUGH("ExcelWriter-05", "您缺少权限执行相应的文件写入操作."), - PATH_NOT_VALID("ExcelWriter-06", "配置的路径无效"), - FILE_FORMAT_ERROR("ExcelWriter-07", "不支持的文件格式"), - PATH_NOT_DIR("ExcelWriter-08", "您配置的路径不是文件夹."); - - private final String code; - private final String description; - - ExcelWriterErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s].", this.code, - this.description); - } -} diff --git a/plugin/writer/ftpwriter/src/main/java/com/wgzhao/addax/plugin/writer/ftpwriter/FtpWriter.java b/plugin/writer/ftpwriter/src/main/java/com/wgzhao/addax/plugin/writer/ftpwriter/FtpWriter.java index 150269253..f9f920db2 100644 --- a/plugin/writer/ftpwriter/src/main/java/com/wgzhao/addax/plugin/writer/ftpwriter/FtpWriter.java +++ b/plugin/writer/ftpwriter/src/main/java/com/wgzhao/addax/plugin/writer/ftpwriter/FtpWriter.java @@ -47,6 +47,10 @@ import static com.wgzhao.addax.common.base.Key.FILE_NAME; import static com.wgzhao.addax.common.base.Key.SUFFIX; import static com.wgzhao.addax.common.base.Key.WRITE_MODE; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.IO_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.LOGIN_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; public class FtpWriter extends Writer @@ -93,28 +97,28 @@ public void init() protocol, username, host, port, e.getMessage()); LOG.error(message); throw AddaxException.asAddaxException( - FtpWriterErrorCode.FAIL_LOGIN, message, e); + LOGIN_ERROR, message, e); } } private void validateParameter() { - this.writerSliceConfig.getNecessaryValue(FILE_NAME, FtpWriterErrorCode.REQUIRED_VALUE); - String path = this.writerSliceConfig.getNecessaryValue(FtpKey.PATH, FtpWriterErrorCode.REQUIRED_VALUE); + this.writerSliceConfig.getNecessaryValue(FILE_NAME, REQUIRED_VALUE); + String path = this.writerSliceConfig.getNecessaryValue(FtpKey.PATH, REQUIRED_VALUE); if (!path.startsWith("/")) { String message = String.format("The item path [%s] should be configured as absolute path.", path); LOG.error(message); - throw AddaxException.asAddaxException(FtpWriterErrorCode.ILLEGAL_VALUE, message); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, message); } - this.host = this.writerSliceConfig.getNecessaryValue(FtpKey.HOST, FtpWriterErrorCode.REQUIRED_VALUE); - this.username = this.writerSliceConfig.getNecessaryValue(FtpKey.USERNAME, FtpWriterErrorCode.REQUIRED_VALUE); + this.host = this.writerSliceConfig.getNecessaryValue(FtpKey.HOST, REQUIRED_VALUE); + this.username = this.writerSliceConfig.getNecessaryValue(FtpKey.USERNAME, REQUIRED_VALUE); this.password = this.writerSliceConfig.getString(FtpKey.PASSWORD, null); this.timeout = this.writerSliceConfig.getInt(FtpKey.TIMEOUT, DEFAULT_TIMEOUT); this.protocol = this.writerSliceConfig.getString(FtpKey.PROTOCOL, "ftp"); if (!("ftp".equalsIgnoreCase(protocol) || "sftp".equalsIgnoreCase(protocol))) { - throw AddaxException.asAddaxException(FtpWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, protocol + " is unsupported, supported protocol are ftp and sftp"); } this.writerSliceConfig.set(FtpKey.PROTOCOL, protocol); @@ -137,7 +141,7 @@ private void validateParameter() else { String msg = "You have configured to use the key, but neither the configured key file nor the default file(" + DEFAULT_PRIVATE_KEY + " exists"; - throw AddaxException.asAddaxException(FtpWriterErrorCode.ILLEGAL_VALUE, msg); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, msg); } } } @@ -150,7 +154,7 @@ else if ("ftp".equalsIgnoreCase(protocol)) { } else { throw AddaxException.asAddaxException( - FtpWriterErrorCode.ILLEGAL_VALUE, protocol + " is unsupported, supported protocol are ftp and sftp"); + ILLEGAL_VALUE, protocol + " is unsupported, supported protocol are ftp and sftp"); } this.writerSliceConfig.set(FtpKey.PORT, this.port); } @@ -189,14 +193,14 @@ else if ("nonConflict".equals(writeMode)) { LOG.info("The directory [{}] includes the following files with prefix [{}]: [{}].", path, fileName, StringUtils.join(allFilesInDir.iterator(), ", ")); throw AddaxException.asAddaxException( - FtpWriterErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, String.format("您配置的path: [%s] 目录不为空, 下面存在其他文件或文件夹.", path)); } } else { throw AddaxException .asAddaxException( - FtpWriterErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, String.format("仅支持 truncate, append, nonConflict 三种模式, 不支持您配置的 writeMode 模式 : [%s]", writeMode)); } @@ -302,7 +306,7 @@ else if ("ftp".equalsIgnoreCase(protocol)) { host, username, port, e.getMessage()); LOG.error(message); throw AddaxException.asAddaxException( - FtpWriterErrorCode.FAIL_LOGIN, message, e); + LOGIN_ERROR, message, e); } } @@ -332,7 +336,7 @@ public void startWrite(RecordReceiver lineReceiver) } catch (Exception e) { throw AddaxException.asAddaxException( - FtpWriterErrorCode.WRITE_FILE_IO_ERROR, + IO_ERROR, String.format("无法创建待写文件 : [%s]", this.fileName), e); } finally { diff --git a/plugin/writer/ftpwriter/src/main/java/com/wgzhao/addax/plugin/writer/ftpwriter/FtpWriterErrorCode.java b/plugin/writer/ftpwriter/src/main/java/com/wgzhao/addax/plugin/writer/ftpwriter/FtpWriterErrorCode.java deleted file mode 100644 index 4b78e1c8a..000000000 --- a/plugin/writer/ftpwriter/src/main/java/com/wgzhao/addax/plugin/writer/ftpwriter/FtpWriterErrorCode.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.writer.ftpwriter; - -import com.wgzhao.addax.common.spi.ErrorCode; - -public enum FtpWriterErrorCode - implements ErrorCode -{ - - REQUIRED_VALUE("FtpWriter-00", "您缺失了必须填写的参数值."), - ILLEGAL_VALUE("FtpWriter-01", "您填写的参数值不合法."), - OPEN_FILE_ERROR("FtpWriter-06", "您配置的文件在打开时异常."), - WRITE_FILE_IO_ERROR("FtpWriter-07", "您配置的文件在读取时出现IO异常."), - FAIL_LOGIN("FtpWriter-12", "登录失败,无法与ftp服务器建立连接."), - FAIL_DISCONNECT("FtpWriter-13", "关闭ftp连接失败,无法与ftp服务器断开连接."), - COMMAND_FTP_IO_EXCEPTION("FtpWriter-14", "与ftp服务器连接异常."); - - private final String code; - private final String description; - - FtpWriterErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return code; - } - - @Override - public String getDescription() - { - return description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s].", code, description); - } -} diff --git a/plugin/writer/ftpwriter/src/main/java/com/wgzhao/addax/plugin/writer/ftpwriter/util/SftpHelperImpl.java b/plugin/writer/ftpwriter/src/main/java/com/wgzhao/addax/plugin/writer/ftpwriter/util/SftpHelperImpl.java index e7523c691..4c255bf4f 100644 --- a/plugin/writer/ftpwriter/src/main/java/com/wgzhao/addax/plugin/writer/ftpwriter/util/SftpHelperImpl.java +++ b/plugin/writer/ftpwriter/src/main/java/com/wgzhao/addax/plugin/writer/ftpwriter/util/SftpHelperImpl.java @@ -29,7 +29,6 @@ import com.jcraft.jsch.SftpATTRS; import com.jcraft.jsch.SftpException; import com.wgzhao.addax.common.exception.AddaxException; -import com.wgzhao.addax.plugin.writer.ftpwriter.FtpWriterErrorCode; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -42,6 +41,10 @@ import java.util.Set; import java.util.Vector; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.IO_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.LOGIN_ERROR; + public class SftpHelperImpl implements IFtpHelper { @@ -64,13 +67,13 @@ public void loginFtpServer(String host, int port, String username, String passwo } } catch (JSchException e) { - throw AddaxException.asAddaxException(FtpWriterErrorCode.ILLEGAL_VALUE, "Failed to use private key", e); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "Failed to use private key", e); } } try { this.session = jsch.getSession(username, host, port); if (this.session == null) { - throw AddaxException.asAddaxException(FtpWriterErrorCode.FAIL_LOGIN, + throw AddaxException.asAddaxException(LOGIN_ERROR, String.format("Failed to connect %s:%s via sftp protocol", host, port)); } @@ -88,7 +91,7 @@ public void loginFtpServer(String host, int port, String username, String passwo catch (JSchException e) { String message = String.format("Failed to connect %s:%s because: %s", host, port, e.getMessage()); LOG.error(message); - throw AddaxException.asAddaxException(FtpWriterErrorCode.FAIL_LOGIN, message, e); + throw AddaxException.asAddaxException(LOGIN_ERROR, message, e); } } @@ -126,7 +129,7 @@ public void mkdir(String directoryPath) } catch (SftpException e) { LOG.error("IOException occurred while create folder {}, {}", directoryPath, e); - throw AddaxException.asAddaxException(FtpWriterErrorCode.COMMAND_FTP_IO_EXCEPTION, e); + throw AddaxException.asAddaxException(IO_ERROR, e); } } } @@ -158,7 +161,7 @@ public void mkDirRecursive(String directoryPath) } catch (SftpException e) { LOG.error("IOException occurred while create folder {}, {}", directoryPath, e); - throw AddaxException.asAddaxException(FtpWriterErrorCode.COMMAND_FTP_IO_EXCEPTION, e); + throw AddaxException.asAddaxException(IO_ERROR, e); } } } @@ -192,14 +195,14 @@ public OutputStream getOutputStream(String filePath) OutputStream writeOutputStream = this.channelSftp.put(filePath, ChannelSftp.APPEND); String message = String.format("打开FTP文件[%s]获取写出流时出错,请确认文件%s有权限创建，有权限写出等", filePath, filePath); if (null == writeOutputStream) { - throw AddaxException.asAddaxException(FtpWriterErrorCode.OPEN_FILE_ERROR, message); + throw AddaxException.asAddaxException(IO_ERROR, message); } return writeOutputStream; } catch (SftpException e) { String message = String.format("写出文件[%s] 时出错,请确认文件%s有权限写出, errorMessage:%s", filePath, filePath, e.getMessage()); LOG.error(message); - throw AddaxException.asAddaxException(FtpWriterErrorCode.OPEN_FILE_ERROR, message); + throw AddaxException.asAddaxException(IO_ERROR, message); } } @@ -221,7 +224,7 @@ public String getRemoteFileContent(String filePath) catch (SftpException e) { String message = String.format("写出文件[%s] 时出错,请确认文件%s有权限写出, errorMessage:%s", filePath, filePath, e.getMessage()); LOG.error(message); - throw AddaxException.asAddaxException(FtpWriterErrorCode.OPEN_FILE_ERROR, message); + throw AddaxException.asAddaxException(IO_ERROR, message); } } @@ -247,7 +250,7 @@ public Set getAllFilesInDir(String dir, String prefixFileName) String message = String.format("获取path:[%s] 下文件列表时发生I/O异常,请确认与ftp服务器的连接正常,拥有目录ls权限, errorMessage:%s", dir, e.getMessage()); LOG.error(message); - throw AddaxException.asAddaxException(FtpWriterErrorCode.COMMAND_FTP_IO_EXCEPTION, message, e); + throw AddaxException.asAddaxException(IO_ERROR, message, e); } return allFilesWithPointedPrefix; } @@ -270,7 +273,7 @@ public void deleteFiles(Set filesToDelete) eachFile, e.getMessage()); LOG.error(message); throw AddaxException.asAddaxException( - FtpWriterErrorCode.COMMAND_FTP_IO_EXCEPTION, message, e); + IO_ERROR, message, e); } } diff --git a/plugin/writer/ftpwriter/src/main/java/com/wgzhao/addax/plugin/writer/ftpwriter/util/StandardFtpHelperImpl.java b/plugin/writer/ftpwriter/src/main/java/com/wgzhao/addax/plugin/writer/ftpwriter/util/StandardFtpHelperImpl.java index 4cd8add1c..5de6ba819 100644 --- a/plugin/writer/ftpwriter/src/main/java/com/wgzhao/addax/plugin/writer/ftpwriter/util/StandardFtpHelperImpl.java +++ b/plugin/writer/ftpwriter/src/main/java/com/wgzhao/addax/plugin/writer/ftpwriter/util/StandardFtpHelperImpl.java @@ -22,7 +22,6 @@ import com.alibaba.fastjson2.JSON; import com.alibaba.fastjson2.JSONWriter; import com.wgzhao.addax.common.exception.AddaxException; -import com.wgzhao.addax.plugin.writer.ftpwriter.FtpWriterErrorCode; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.net.ftp.FTPClient; @@ -38,6 +37,10 @@ import java.util.HashSet; import java.util.Set; +import static com.wgzhao.addax.common.spi.ErrorCode.CONNECT_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.EXECUTE_FAIL; +import static com.wgzhao.addax.common.spi.ErrorCode.IO_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.LOGIN_ERROR; import static org.apache.commons.net.ftp.FTP.BINARY_FILE_TYPE; public class StandardFtpHelperImpl @@ -74,7 +77,7 @@ public void loginFtpServer(String host, int port, String username, String passwo host, port, username, reply); LOG.error(message); throw AddaxException.asAddaxException( - FtpWriterErrorCode.FAIL_LOGIN, message); + LOGIN_ERROR, message); } } catch (UnknownHostException e) { @@ -83,7 +86,7 @@ public void loginFtpServer(String host, int port, String username, String passwo host, e.getMessage()); LOG.error(message); throw AddaxException.asAddaxException( - FtpWriterErrorCode.FAIL_LOGIN, message, e); + LOGIN_ERROR, message, e); } catch (IllegalArgumentException e) { String message = String.format( @@ -91,7 +94,7 @@ public void loginFtpServer(String host, int port, String username, String passwo e.getMessage()); LOG.error(message); throw AddaxException.asAddaxException( - FtpWriterErrorCode.FAIL_LOGIN, message, e); + LOGIN_ERROR, message, e); } catch (Exception e) { String message = String @@ -99,7 +102,7 @@ public void loginFtpServer(String host, int port, String username, String passwo host, port, username, e.getMessage()); LOG.error(message); throw AddaxException.asAddaxException( - FtpWriterErrorCode.FAIL_LOGIN, message, e); + LOGIN_ERROR, message, e); } } @@ -115,7 +118,7 @@ public void logoutFtpServer() "与ftp服务器断开连接失败, errorMessage:%s", e.getMessage()); LOG.error(message); throw AddaxException.asAddaxException( - FtpWriterErrorCode.FAIL_DISCONNECT, message, e); + CONNECT_ERROR, message, e); } finally { if (this.ftpClient.isConnected()) { @@ -150,7 +153,7 @@ public void mkdir(String directoryPath) if (replayCode != FTPReply.COMMAND_OK && replayCode != FTPReply.PATHNAME_CREATED) { throw AddaxException.asAddaxException( - FtpWriterErrorCode.COMMAND_FTP_IO_EXCEPTION, + EXECUTE_FAIL, message); } } @@ -160,7 +163,7 @@ public void mkdir(String directoryPath) e.getMessage()); LOG.error(message); throw AddaxException.asAddaxException( - FtpWriterErrorCode.COMMAND_FTP_IO_EXCEPTION, message, e); + IO_ERROR, message, e); } } @@ -179,7 +182,7 @@ public void mkDirRecursive(String directoryPath) dirPath.append(IOUtils.DIR_SEPARATOR_UNIX); if (!mkdirSuccess) { throw AddaxException.asAddaxException( - FtpWriterErrorCode.COMMAND_FTP_IO_EXCEPTION, + EXECUTE_FAIL, message); } } @@ -189,7 +192,7 @@ public void mkDirRecursive(String directoryPath) e.getMessage()); LOG.error(message); throw AddaxException.asAddaxException( - FtpWriterErrorCode.COMMAND_FTP_IO_EXCEPTION, message, e); + IO_ERROR, message, e); } } @@ -223,7 +226,7 @@ public OutputStream getOutputStream(String filePath) filePath); if (null == writeOutputStream) { throw AddaxException.asAddaxException( - FtpWriterErrorCode.OPEN_FILE_ERROR, message); + EXECUTE_FAIL, message); } return writeOutputStream; @@ -234,7 +237,7 @@ public OutputStream getOutputStream(String filePath) filePath, filePath, e.getMessage()); LOG.error(message); throw AddaxException.asAddaxException( - FtpWriterErrorCode.OPEN_FILE_ERROR, message); + IO_ERROR, message); } } @@ -260,7 +263,7 @@ public String getRemoteFileContent(String filePath) filePath, filePath, e.getMessage()); LOG.error(message); throw AddaxException.asAddaxException( - FtpWriterErrorCode.OPEN_FILE_ERROR, message); + IO_ERROR, message); } } @@ -272,7 +275,7 @@ public Set getAllFilesInDir(String dir, String prefixFileName) boolean isDirExist = this.ftpClient.changeWorkingDirectory(dir); if (!isDirExist) { throw AddaxException.asAddaxException( - FtpWriterErrorCode.COMMAND_FTP_IO_EXCEPTION, + EXECUTE_FAIL, String.format("进入目录[%s]失败", dir)); } this.printWorkingDirectory(); @@ -293,7 +296,7 @@ public Set getAllFilesInDir(String dir, String prefixFileName) dir, e.getMessage()); LOG.error(message); throw AddaxException.asAddaxException( - FtpWriterErrorCode.COMMAND_FTP_IO_EXCEPTION, message, e); + IO_ERROR, message, e); } return allFilesWithPointedPrefix; } @@ -313,7 +316,7 @@ public void deleteFiles(Set filesToDelete) String message = String.format( "删除文件:[%s] 时失败,请确认指定文件有删除权限", eachFile); throw AddaxException.asAddaxException( - FtpWriterErrorCode.COMMAND_FTP_IO_EXCEPTION, + IO_ERROR, message); } } @@ -324,7 +327,7 @@ public void deleteFiles(Set filesToDelete) eachFile, e.getMessage()); LOG.error(message); throw AddaxException.asAddaxException( - FtpWriterErrorCode.COMMAND_FTP_IO_EXCEPTION, message, e); + IO_ERROR, message, e); } } @@ -353,7 +356,7 @@ public void completePendingCommand() boolean isOk = this.ftpClient.completePendingCommand(); if (!isOk) { throw AddaxException.asAddaxException( - FtpWriterErrorCode.COMMAND_FTP_IO_EXCEPTION, + EXECUTE_FAIL, "完成ftp completePendingCommand操作发生异常"); } } @@ -363,7 +366,7 @@ public void completePendingCommand() e.getMessage()); LOG.error(message); throw AddaxException.asAddaxException( - FtpWriterErrorCode.COMMAND_FTP_IO_EXCEPTION, message, e); + EXECUTE_FAIL, message, e); } } } diff --git a/plugin/writer/greenplumwriter/src/main/java/com/wgzhao/addax/plugin/writer/greenplumwriter/CopyWriterTask.java b/plugin/writer/greenplumwriter/src/main/java/com/wgzhao/addax/plugin/writer/greenplumwriter/CopyWriterTask.java index 2fda288dd..06a9b0848 100644 --- a/plugin/writer/greenplumwriter/src/main/java/com/wgzhao/addax/plugin/writer/greenplumwriter/CopyWriterTask.java +++ b/plugin/writer/greenplumwriter/src/main/java/com/wgzhao/addax/plugin/writer/greenplumwriter/CopyWriterTask.java @@ -26,7 +26,6 @@ import com.wgzhao.addax.common.plugin.TaskPluginCollector; import com.wgzhao.addax.common.util.Configuration; import com.wgzhao.addax.rdbms.util.DBUtil; -import com.wgzhao.addax.rdbms.util.DBUtilErrorCode; import com.wgzhao.addax.rdbms.util.DataBaseType; import com.wgzhao.addax.rdbms.writer.CommonRdbmsWriter; import com.wgzhao.addax.rdbms.writer.util.WriterUtil; @@ -45,6 +44,7 @@ import static com.wgzhao.addax.common.base.Constant.DEFAULT_BATCH_SIZE; import static com.wgzhao.addax.common.base.Key.BATCH_SIZE; +import static com.wgzhao.addax.common.spi.ErrorCode.EXECUTE_FAIL; public class CopyWriterTask extends CommonRdbmsWriter.Task @@ -127,7 +127,7 @@ public void startWrite(RecordReceiver recordReceiver, Configuration writerSliceC } catch (Exception e) { - throw AddaxException.asAddaxException(DBUtilErrorCode.WRITE_DATA_ERROR, e); + throw AddaxException.asAddaxException(EXECUTE_FAIL, e); } finally { DBUtil.closeDBResources(null, null, connection); diff --git a/plugin/writer/hbase11xsqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xsqlwriter/HbaseSQLHelper.java b/plugin/writer/hbase11xsqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xsqlwriter/HbaseSQLHelper.java index db67e6c2e..6f16dcd7b 100644 --- a/plugin/writer/hbase11xsqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xsqlwriter/HbaseSQLHelper.java +++ b/plugin/writer/hbase11xsqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xsqlwriter/HbaseSQLHelper.java @@ -51,6 +51,12 @@ import java.util.List; import java.util.Map; +import static com.wgzhao.addax.common.spi.ErrorCode.CONNECT_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.EXECUTE_FAIL; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.LOGIN_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.PERMISSION_ERROR; + /** * @author yanghan.y */ @@ -128,7 +134,7 @@ public static void validateConfig(HbaseSQLWriterConfig cfg) schema = getTableSchema(conn, cfg.getNamespace(), cfg.getTableName(), cfg.isThinClient()); } catch (SQLException e) { - throw AddaxException.asAddaxException(HbaseSQLWriterErrorCode.GET_HBASE_CONNECTION_ERROR, + throw AddaxException.asAddaxException(CONNECT_ERROR, "Unable to get the metadata of table " + cfg.getTableName(), e); } List columnNames = cfg.getColumns(); @@ -140,12 +146,12 @@ public static void validateConfig(HbaseSQLWriterConfig cfg) } catch (ColumnNotFoundException e) { // 用户配置的列名在元数据中不存在 - throw AddaxException.asAddaxException(HbaseSQLWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "The column '" + e.getColumnName() + "' your configured does not exists in the target table " + cfg.getTableName(), e); } catch (SQLException e) { // 列名有二义性或者其他问题 - throw AddaxException.asAddaxException(HbaseSQLWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "The column validation of target table " + cfg.getTableName() + "has got failure", e); } } @@ -180,7 +186,7 @@ public static Connection getJdbcConnection(HbaseSQLWriterConfig cfg) conn.setAutoCommit(false); } catch (Throwable e) { - throw AddaxException.asAddaxException(HbaseSQLWriterErrorCode.GET_HBASE_CONNECTION_ERROR, + throw AddaxException.asAddaxException(CONNECT_ERROR, "Unable to connect to hbase cluster, please check the configuration and cluster status ", e); } LOG.debug("Connected to HBase cluster successfully."); @@ -198,7 +204,7 @@ private static void kerberosAuthentication(String kerberosPrincipal, String kerb String message = String.format("Kerberos authentication failed, please make sure that kerberosKeytabFilePath[%s] and kerberosPrincipal[%s] are correct", kerberosKeytabFilePath, kerberosPrincipal); LOG.error(message); - throw AddaxException.asAddaxException(HbaseSQLWriterErrorCode.KERBEROS_LOGIN_ERROR, e); + throw AddaxException.asAddaxException(LOGIN_ERROR, e); } } } @@ -222,7 +228,7 @@ public static Connection getThinClientJdbcConnection(HbaseSQLWriterConfig cfg) return conn; } catch (Exception e) { - throw AddaxException.asAddaxException(HbaseSQLWriterErrorCode.GET_HBASE_CONNECTION_ERROR, + throw AddaxException.asAddaxException(CONNECT_ERROR, "Can not connection to the namespace.", e); } } @@ -331,7 +337,7 @@ public static void truncateTable(Connection conn, String tableName) } catch (Throwable t) { // 清空表失败 - throw AddaxException.asAddaxException(HbaseSQLWriterErrorCode.TRUNCATE_HBASE_ERROR, + throw AddaxException.asAddaxException(EXECUTE_FAIL, "Failed to truncate " + tableName + ".", t); } finally { @@ -373,7 +379,7 @@ public static void checkTable(Connection conn, String tableName) checkTable(admin, getTableName(tableName)); } catch (SQLException | IOException t) { - throw AddaxException.asAddaxException(HbaseSQLWriterErrorCode.TRUNCATE_HBASE_ERROR, + throw AddaxException.asAddaxException(EXECUTE_FAIL, "The table " + tableName + "status check failed, please check the HBase cluster status.", t); } finally { @@ -387,15 +393,15 @@ private static void checkTable(Admin admin, TableName tableName) throws IOException { if (!admin.tableExists(tableName)) { - throw AddaxException.asAddaxException(HbaseSQLWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "The hbase table " + tableName + "doest not exists."); } if (!admin.isTableAvailable(tableName)) { - throw AddaxException.asAddaxException(HbaseSQLWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "The hbase table" + tableName + "is unavailable."); } if (admin.isTableDisabled(tableName)) { - throw AddaxException.asAddaxException(HbaseSQLWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "The hbase table " + tableName + "is disabled at current, please enable it before using"); } LOG.info("table {} exists", tableName); @@ -409,7 +415,7 @@ private static void closeAdmin(Admin admin) } } catch (IOException e) { - throw AddaxException.asAddaxException(HbaseSQLWriterErrorCode.CLOSE_HBASE_AMIN_ERROR, e); + throw AddaxException.asAddaxException(PERMISSION_ERROR, e); } } diff --git a/plugin/writer/hbase11xsqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xsqlwriter/HbaseSQLWriterConfig.java b/plugin/writer/hbase11xsqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xsqlwriter/HbaseSQLWriterConfig.java index 13ae2ddf7..16b09726a 100644 --- a/plugin/writer/hbase11xsqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xsqlwriter/HbaseSQLWriterConfig.java +++ b/plugin/writer/hbase11xsqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xsqlwriter/HbaseSQLWriterConfig.java @@ -34,6 +34,9 @@ import java.util.List; import java.util.Map; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; + /** * HBase SQL writer config * @@ -112,7 +115,7 @@ private static void parseClusterConfig(HbaseSQLWriterConfig cfg, Configuration j if (StringUtils.isBlank(hbaseCfg)) { // 集群配置必须存在且不为空 throw AddaxException.asAddaxException( - HbaseSQLWriterErrorCode.REQUIRED_VALUE, + REQUIRED_VALUE, String.format("%s must be configured with the following: %s and %s", HBaseKey.HBASE_CONFIG, HConstants.ZOOKEEPER_QUORUM, HConstants.ZOOKEEPER_ZNODE_PARENT)); } @@ -125,12 +128,12 @@ private static void parseClusterConfig(HbaseSQLWriterConfig cfg, Configuration j cfg.password = thinConnectConfig.get(HBaseKey.HBASE_THIN_CONNECT_PASSWORD); if (Strings.isNullOrEmpty(thinConnectStr)) { throw AddaxException.asAddaxException( - HbaseSQLWriterErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, "You must configure 'hbase.thin.connect.url' if your want use thinClient mode"); } if (Strings.isNullOrEmpty(cfg.namespace) || Strings.isNullOrEmpty(cfg.username) || Strings .isNullOrEmpty(cfg.password)) { - throw AddaxException.asAddaxException(HbaseSQLWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "The items 'hbase.thin.connect.namespace|username|password' must be configured if you want to use thinClient mode"); } cfg.connectionString = thinConnectStr; @@ -144,14 +147,14 @@ private static void parseClusterConfig(HbaseSQLWriterConfig cfg, Configuration j catch (Throwable t) { // 解析hbase配置错误 throw AddaxException.asAddaxException( - HbaseSQLWriterErrorCode.REQUIRED_VALUE, + REQUIRED_VALUE, "Failed to parse hbaseConfig，please check it."); } String zkQuorum = zkCfg.getFirst(); String znode = zkCfg.getSecond(); if (zkQuorum == null || zkQuorum.isEmpty() || znode == null || znode.isEmpty()) { throw AddaxException.asAddaxException( - HbaseSQLWriterErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, "The items hbase.zookeeper.quorum/zookeeper.znode.parent must be configured"); } @@ -167,14 +170,14 @@ private static void parseTableConfig(HbaseSQLWriterConfig cfg, Configuration job if (cfg.tableName == null || cfg.tableName.isEmpty()) { throw AddaxException.asAddaxException( - HbaseSQLWriterErrorCode.ILLEGAL_VALUE, "The item tableName must be configured."); + ILLEGAL_VALUE, "The item tableName must be configured."); } try { TableName.valueOf(cfg.tableName); } catch (Exception e) { throw AddaxException.asAddaxException( - HbaseSQLWriterErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, "The table " + cfg.tableName + " you configured has illegal symbols."); } @@ -182,7 +185,7 @@ private static void parseTableConfig(HbaseSQLWriterConfig cfg, Configuration job cfg.columns = jobConf.getList(HBaseKey.COLUMN, String.class); if (cfg.columns == null || cfg.columns.isEmpty()) { throw AddaxException.asAddaxException( - HbaseSQLWriterErrorCode.ILLEGAL_VALUE, "The item columns must be configured and can not be empty."); + ILLEGAL_VALUE, "The item columns must be configured and can not be empty."); } } diff --git a/plugin/writer/hbase11xsqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xsqlwriter/HbaseSQLWriterErrorCode.java b/plugin/writer/hbase11xsqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xsqlwriter/HbaseSQLWriterErrorCode.java deleted file mode 100644 index 956f85c7a..000000000 --- a/plugin/writer/hbase11xsqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xsqlwriter/HbaseSQLWriterErrorCode.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.writer.hbase11xsqlwriter; - -import com.wgzhao.addax.common.spi.ErrorCode; - -public enum HbaseSQLWriterErrorCode - implements ErrorCode -{ - REQUIRED_VALUE("Hbasewriter-00", "Missing mandatory configuration item."), - ILLEGAL_VALUE("Hbasewriter-01", "Illegal configuration value."), - GET_HBASE_CONNECTION_ERROR("Hbasewriter-02", "Failed to connect HBase cluster."), -// GET_HBASE_TABLE_ERROR("Hbasewriter-03", "获取 Hbase table时出错."), -// CLOSE_HBASE_CONNECTION_ERROR("Hbasewriter-04", "关闭Hbase连接时出错."), - CLOSE_HBASE_AMIN_ERROR("Hbasewriter-05", "Failed to close HBase admin handler."), -// CLOSE_HBASE_TABLE_ERROR("Hbasewriter-06", "关闭Hbase table时时出错."), - PUT_HBASE_ERROR("Hbasewriter-07", "IO exception occurred while writing to HBase."), -// DELETE_HBASE_ERROR("Hbasewriter-08", "delete hbase表时发生异常."), - TRUNCATE_HBASE_ERROR("Hbasewriter-09", "Exception occurred while truncating HBase table."), - KERBEROS_LOGIN_ERROR("HbaseWriter-10", "Kerberos authentication failed."); - - private final String code; - private final String description; - - HbaseSQLWriterErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s].", this.code, this.description); - } -} diff --git a/plugin/writer/hbase11xsqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xsqlwriter/HbaseSQLWriterTask.java b/plugin/writer/hbase11xsqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xsqlwriter/HbaseSQLWriterTask.java index 8c82395d8..4b6aef088 100644 --- a/plugin/writer/hbase11xsqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xsqlwriter/HbaseSQLWriterTask.java +++ b/plugin/writer/hbase11xsqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xsqlwriter/HbaseSQLWriterTask.java @@ -41,6 +41,10 @@ import java.util.Collections; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.EXECUTE_FAIL; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; + /** * @author yanghan.y */ @@ -75,7 +79,7 @@ public void startWriter(RecordReceiver lineReceiver, TaskPluginCollector taskPlu while ((record = lineReceiver.getFromReader()) != null) { // 校验列数量是否符合预期 if (record.getColumnNumber() != numberOfColumnsToRead) { - throw AddaxException.asAddaxException(HbaseSQLWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "The number of fields(" + record.getColumnNumber() + ") in the source and the number of fields(" + numberOfColumnsToRead + ") you configured in 'column' are not the same."); } @@ -95,7 +99,7 @@ public void startWriter(RecordReceiver lineReceiver, TaskPluginCollector taskPlu } catch (Throwable t) { // 确保所有异常都转化为 AddaxException - throw AddaxException.asAddaxException(HbaseSQLWriterErrorCode.PUT_HBASE_ERROR, t); + throw AddaxException.asAddaxException(RUNTIME_ERROR, t); } finally { close(); @@ -163,7 +167,7 @@ private void doBatchUpsert(List records) doSingleUpsert(records); } catch (Exception e) { - throw AddaxException.asAddaxException(HbaseSQLWriterErrorCode.PUT_HBASE_ERROR, e); + throw AddaxException.asAddaxException(EXECUTE_FAIL, e); } } @@ -304,7 +308,7 @@ private void setupColumn(int pos, int sqlType, Column col) break; default: - throw AddaxException.asAddaxException(HbaseSQLWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "The data type " + sqlType + "is unsupported."); } // end switch } @@ -324,7 +328,7 @@ private void setupColumn(int pos, int sqlType, Column col) default: // nullMode的合法性在初始化配置的时候已经校验过，这里一定不会出错 - throw AddaxException.asAddaxException(HbaseSQLWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "The nullMode type " + cfg.getNullMode() + " is unsupported, here are available nullMode:" + Arrays.asList(NullModeType.values())); } } @@ -389,7 +393,7 @@ private Object getEmptyValue(int sqlType) return new byte[0]; default: - throw AddaxException.asAddaxException(HbaseSQLWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "The data type " + sqlType + " is unsupported yet."); } } diff --git a/plugin/writer/hbase11xsqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xsqlwriter/NullModeType.java b/plugin/writer/hbase11xsqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xsqlwriter/NullModeType.java index 317674a82..d832c5253 100644 --- a/plugin/writer/hbase11xsqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xsqlwriter/NullModeType.java +++ b/plugin/writer/hbase11xsqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xsqlwriter/NullModeType.java @@ -23,6 +23,8 @@ import java.util.Arrays; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; + public enum NullModeType { SKIP("skip"), @@ -42,7 +44,7 @@ public static NullModeType getByTypeName(String modeName) return modeType; } } - throw AddaxException.asAddaxException(HbaseSQLWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "The nullMode type " + modeName + " is unsupported, here are available nullMode:" + Arrays.asList(values())); } diff --git a/plugin/writer/hbase11xwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xwriter/ColumnType.java b/plugin/writer/hbase11xwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xwriter/ColumnType.java index bc2fa64f6..c159d3d66 100644 --- a/plugin/writer/hbase11xwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xwriter/ColumnType.java +++ b/plugin/writer/hbase11xwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xwriter/ColumnType.java @@ -24,6 +24,9 @@ import java.util.Arrays; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.NOT_SUPPORT_TYPE; + /** * 只对 normal 模式读取时有用，多版本读取时，不存在列类型的 */ @@ -47,7 +50,7 @@ public enum ColumnType public static ColumnType getByTypeName(String typeName) { if (StringUtils.isBlank(typeName)) { - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(CONFIG_ERROR, String.format("The data type %s is unsupported, the currently supported data types: %s", typeName, Arrays.asList(values()))); } for (ColumnType columnType : values()) { @@ -56,7 +59,7 @@ public static ColumnType getByTypeName(String typeName) } } - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(NOT_SUPPORT_TYPE, String.format("The data type %s is unsupported, the currently supported data types: %s", typeName, Arrays.asList(values()))); } diff --git a/plugin/writer/hbase11xwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xwriter/Hbase11xHelper.java b/plugin/writer/hbase11xwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xwriter/Hbase11xHelper.java index e6aefa754..587ef827d 100644 --- a/plugin/writer/hbase11xwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xwriter/Hbase11xHelper.java +++ b/plugin/writer/hbase11xwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xwriter/Hbase11xHelper.java @@ -46,6 +46,13 @@ import java.util.List; import java.util.Map; +import static com.wgzhao.addax.common.spi.ErrorCode.CONNECT_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.IO_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.LOGIN_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; + public class Hbase11xHelper { @@ -56,7 +63,7 @@ private Hbase11xHelper() {} public static org.apache.hadoop.conf.Configuration getHbaseConfiguration(String hbaseConfig) { if (StringUtils.isBlank(hbaseConfig)) { - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.REQUIRED_VALUE, "The item hbaseConfig must be configured."); + throw AddaxException.asAddaxException(REQUIRED_VALUE, "The item hbaseConfig must be configured."); } org.apache.hadoop.conf.Configuration hConfiguration = HBaseConfiguration.create(); try { @@ -68,7 +75,7 @@ public static org.apache.hadoop.conf.Configuration getHbaseConfiguration(String } } catch (Exception e) { - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.HBASE_CONNECTION_ERROR, e); + throw AddaxException.asAddaxException(CONNECT_ERROR, e); } return hConfiguration; } @@ -83,7 +90,7 @@ public static org.apache.hadoop.hbase.client.Connection getHbaseConnection(Strin } catch (Exception e) { Hbase11xHelper.closeConnection(hConnection); - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.HBASE_CONNECTION_ERROR, e); + throw AddaxException.asAddaxException(CONNECT_ERROR, e); } return hConnection; } @@ -108,7 +115,7 @@ public static Table getTable(Configuration configuration) Hbase11xHelper.closeTable(hTable); Hbase11xHelper.closeAdmin(admin); Hbase11xHelper.closeConnection(hConnection); - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.HBASE_TABLE_ERROR, e); + throw AddaxException.asAddaxException(RUNTIME_ERROR, e); } return hTable; } @@ -136,7 +143,7 @@ public static BufferedMutator getBufferedMutator(Configuration configuration) Hbase11xHelper.closeBufferedMutator(bufferedMutator); Hbase11xHelper.closeAdmin(admin); Hbase11xHelper.closeConnection(hConnection); - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.GET_HBASE_BUFFERED_MUTATOR_ERROR, e); + throw AddaxException.asAddaxException(RUNTIME_ERROR, e); } return bufferedMutator; } @@ -156,7 +163,7 @@ public static void truncateTable(Configuration configuration) admin.truncateTable(hTableName, true); } catch (Exception e) { - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.TRUNCATE_HBASE_ERROR, e); + throw AddaxException.asAddaxException(RUNTIME_ERROR, e); } finally { Hbase11xHelper.closeAdmin(admin); @@ -172,7 +179,7 @@ public static void closeConnection(Connection hConnection) } } catch (IOException e) { - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.CLOSE_HBASE_CONNECTION_ERROR, e); + throw AddaxException.asAddaxException(IO_ERROR, e); } } @@ -184,7 +191,7 @@ public static void closeAdmin(Admin admin) } } catch (IOException e) { - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.CLOSE_HBASE_AMIN_ERROR, e); + throw AddaxException.asAddaxException(IO_ERROR, e); } } @@ -196,7 +203,7 @@ public static void closeBufferedMutator(BufferedMutator bufferedMutator) } } catch (IOException e) { - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.CLOSE_HBASE_BUFFERED_MUTATOR_ERROR, e); + throw AddaxException.asAddaxException(IO_ERROR, e); } } @@ -208,7 +215,7 @@ public static void closeTable(Table table) } } catch (IOException e) { - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.CLOSE_HBASE_TABLE_ERROR, e); + throw AddaxException.asAddaxException(IO_ERROR, e); } } @@ -216,33 +223,33 @@ private static void checkHbaseTable(Admin admin, TableName hTableName) throws IOException { if (!admin.tableExists(hTableName)) { - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.ILLEGAL_VALUE, "The table " + hTableName.toString() + "DOES NOT exists."); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "The table " + hTableName.toString() + "DOES NOT exists."); } if (!admin.isTableAvailable(hTableName)) { - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.ILLEGAL_VALUE, "The table " + hTableName.toString() + " is unavailable."); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "The table " + hTableName.toString() + " is unavailable."); } if (admin.isTableDisabled(hTableName)) { - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.ILLEGAL_VALUE, "The table " + hTableName.toString() + "is disabled,"); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "The table " + hTableName.toString() + "is disabled,"); } } public static void validateParameter(Configuration originalConfig) { - originalConfig.getNecessaryValue(HBaseKey.HBASE_CONFIG, Hbase11xWriterErrorCode.REQUIRED_VALUE); - originalConfig.getNecessaryValue(HBaseKey.TABLE, Hbase11xWriterErrorCode.REQUIRED_VALUE); + originalConfig.getNecessaryValue(HBaseKey.HBASE_CONFIG, REQUIRED_VALUE); + originalConfig.getNecessaryValue(HBaseKey.TABLE, REQUIRED_VALUE); Hbase11xHelper.validateMode(originalConfig); String encoding = originalConfig.getString(HBaseKey.ENCODING, HBaseConstant.DEFAULT_ENCODING); if (!Charset.isSupported(encoding)) { - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.ILLEGAL_VALUE, "The encoding[" + encoding + "] is unsupported "); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "The encoding[" + encoding + "] is unsupported "); } originalConfig.set(HBaseKey.ENCODING, encoding); // validate kerberos login if (originalConfig.getBool(Key.HAVE_KERBEROS, false)) { - String principal = originalConfig.getNecessaryValue(Key.KERBEROS_PRINCIPAL, Hbase11xWriterErrorCode.REQUIRED_VALUE); - String keytab = originalConfig.getNecessaryValue(Key.KERBEROS_KEYTAB_FILE_PATH, Hbase11xWriterErrorCode.REQUIRED_VALUE); + String principal = originalConfig.getNecessaryValue(Key.KERBEROS_PRINCIPAL, REQUIRED_VALUE); + String keytab = originalConfig.getNecessaryValue(Key.KERBEROS_KEYTAB_FILE_PATH, REQUIRED_VALUE); kerberosAuthentication(principal, keytab); } Boolean walFlag = originalConfig.getBool(HBaseKey.WAL_FLAG, false); @@ -253,7 +260,7 @@ public static void validateParameter(Configuration originalConfig) private static void validateMode(Configuration originalConfig) { - String mode = originalConfig.getNecessaryValue(HBaseKey.MODE, Hbase11xWriterErrorCode.REQUIRED_VALUE); + String mode = originalConfig.getNecessaryValue(HBaseKey.MODE, REQUIRED_VALUE); ModeType modeType = ModeType.getByTypeName(mode); if (modeType == ModeType.NORMAL) { validateRowkeyColumn(originalConfig); @@ -261,7 +268,7 @@ private static void validateMode(Configuration originalConfig) validateVersionColumn(originalConfig); } else { - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.ILLEGAL_VALUE, "The mode " + mode + "is unsupported"); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "The mode " + mode + "is unsupported"); } } @@ -269,21 +276,21 @@ private static void validateColumn(Configuration originalConfig) { List columns = originalConfig.getListConfiguration(HBaseKey.COLUMN); if (columns == null || columns.isEmpty()) { - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.REQUIRED_VALUE, + throw AddaxException.asAddaxException(REQUIRED_VALUE, "The item column must be configured, the form is 'column:[{\"index\": 0,\"name\": \"cf0:column0\",\"type\": " + "\"string\"},{\"index\": 1,\"name\": \"cf1:column1\",\"type\": \"long\"}]'"); } for (Configuration aColumn : columns) { Integer index = aColumn.getInt(HBaseKey.INDEX); - String type = aColumn.getNecessaryValue(HBaseKey.TYPE, Hbase11xWriterErrorCode.REQUIRED_VALUE); - String name = aColumn.getNecessaryValue(HBaseKey.NAME, Hbase11xWriterErrorCode.REQUIRED_VALUE); + String type = aColumn.getNecessaryValue(HBaseKey.TYPE, REQUIRED_VALUE); + String name = aColumn.getNecessaryValue(HBaseKey.NAME, REQUIRED_VALUE); ColumnType.getByTypeName(type); if (name.split(":").length != 2) { - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, String.format("The field's name[%s] is not valid, it must be configured as cf:qualifier", name)); } if (index == null || index < 0) { - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.ILLEGAL_VALUE, "The index of name is not valid, it must be non-negative number"); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "The index of name is not valid, it must be non-negative number"); } } } @@ -292,24 +299,24 @@ private static void validateRowkeyColumn(Configuration originalConfig) { List rowkeyColumn = originalConfig.getListConfiguration(HBaseKey.ROW_KEY_COLUMN); if (rowkeyColumn == null || rowkeyColumn.isEmpty()) { - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.REQUIRED_VALUE, + throw AddaxException.asAddaxException(REQUIRED_VALUE, "The item rowkeyColumn is required，such as rowkeyColumn:[{\"index\": 0,\"type\": \"string\"},{\"index\": -1,\"type\": \"string\",\"value\": \"_\"}]"); } int rowkeyColumnSize = rowkeyColumn.size(); for (Configuration aRowkeyColumn : rowkeyColumn) { Integer index = aRowkeyColumn.getInt(HBaseKey.INDEX); - String type = aRowkeyColumn.getNecessaryValue(HBaseKey.TYPE, Hbase11xWriterErrorCode.REQUIRED_VALUE); + String type = aRowkeyColumn.getNecessaryValue(HBaseKey.TYPE, REQUIRED_VALUE); ColumnType.getByTypeName(type); if (index == null) { - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.REQUIRED_VALUE, "The index of rowkeyColumn is required"); + throw AddaxException.asAddaxException(REQUIRED_VALUE, "The index of rowkeyColumn is required"); } //不能只有-1列,即rowkey连接串 if (rowkeyColumnSize == 1 && index == -1) { - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "The item rowkeyColumn can not all be constant, it must be specify more than one rowkey column"); } if (index == -1) { - aRowkeyColumn.getNecessaryValue(HBaseKey.VALUE, Hbase11xWriterErrorCode.REQUIRED_VALUE); + aRowkeyColumn.getNecessaryValue(HBaseKey.VALUE, REQUIRED_VALUE); } } } @@ -321,14 +328,14 @@ private static void validateVersionColumn(Configuration originalConfig) if (versionColumn != null) { Integer index = versionColumn.getInt(HBaseKey.INDEX); if (index == null) { - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.REQUIRED_VALUE, "The field[index] of versionColumn is required."); + throw AddaxException.asAddaxException(REQUIRED_VALUE, "The field[index] of versionColumn is required."); } if (index == -1) { //指定时间,需要index=-1,value - versionColumn.getNecessaryValue(HBaseKey.VALUE, Hbase11xWriterErrorCode.REQUIRED_VALUE); + versionColumn.getNecessaryValue(HBaseKey.VALUE, REQUIRED_VALUE); } else if (index < 0) { - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.ILLEGAL_VALUE, "The field[index] of versionColumn must be either -1 or non-negative number"); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "The field[index] of versionColumn must be either -1 or non-negative number"); } } } @@ -346,7 +353,7 @@ public static void kerberosAuthentication(String kerberosPrincipal, String kerbe String message = String.format("Kerberos authentication failed, please make sure that kerberosKeytabFilePath[%s] and kerberosPrincipal[%s] are correct", kerberosKeytabFilePath, kerberosPrincipal); LOG.error(message); - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.KERBEROS_LOGIN_ERROR, e); + throw AddaxException.asAddaxException(LOGIN_ERROR, e); } } } diff --git a/plugin/writer/hbase11xwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xwriter/Hbase11xWriter.java b/plugin/writer/hbase11xwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xwriter/Hbase11xWriter.java index ab0111f24..a5850a10b 100644 --- a/plugin/writer/hbase11xwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xwriter/Hbase11xWriter.java +++ b/plugin/writer/hbase11xwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xwriter/Hbase11xWriter.java @@ -28,6 +28,8 @@ import java.util.ArrayList; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; + /** * Hbase11xWriter * Created by shf on 16/3/17. @@ -89,7 +91,7 @@ public void init() this.hbaseTaskProxy = new NormalTask(taskConfig); } else { - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.ILLEGAL_VALUE, "The mode " + modeType + "is unsupported"); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "The mode " + modeType + "is unsupported"); } } diff --git a/plugin/writer/hbase11xwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xwriter/Hbase11xWriterErrorCode.java b/plugin/writer/hbase11xwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xwriter/Hbase11xWriterErrorCode.java deleted file mode 100644 index d1790cfd7..000000000 --- a/plugin/writer/hbase11xwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xwriter/Hbase11xWriterErrorCode.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.writer.hbase11xwriter; - -import com.wgzhao.addax.common.spi.ErrorCode; - -/** - * Hbase11xWriterErrorCode - * Created by shf on 16/3/8. - */ -public enum Hbase11xWriterErrorCode - implements ErrorCode -{ - REQUIRED_VALUE("Hbasewriter-00", "The mandatory parameter are missing"), - ILLEGAL_VALUE("Hbasewriter-01", "The value of parameter is illegal"), - HBASE_CONNECTION_ERROR("Hbasewriter-02", "Failed to connect to HBase cluster"), - HBASE_TABLE_ERROR("Hbasewriter-03", "Failed to access HBase table"), - CLOSE_HBASE_CONNECTION_ERROR("Hbasewriter-04", "Failed to close connection"), - CLOSE_HBASE_AMIN_ERROR("Hbasewriter-05", "Failed to close HBase admin handler"), - CLOSE_HBASE_TABLE_ERROR("Hbasewriter-06", "Failed to close HBase table handler"), - PUT_HBASE_ERROR("Hbasewriter-07", "IO Exception occurred while putting"), - TRUNCATE_HBASE_ERROR("Hbasewriter-09", "Exception occurred while truncating table"), - CONSTRUCT_ROWKEY_ERROR("Hbasewriter-10", "Failed to create rowkey."), - CONSTRUCT_VERSION_ERROR("Hbasewriter-11", "Failed to put version."), - GET_HBASE_BUFFERED_MUTATOR_ERROR("Hbasewriter-12", "Failed to get the BufferedMutator."), - CLOSE_HBASE_BUFFERED_MUTATOR_ERROR("Hbasewriter-13", "Failed to close BufferedMutator handler."), - KERBEROS_LOGIN_ERROR("Hbasewriter-14", "Kerberos authentication failed.") - ; - private final String code; - private final String description; - - Hbase11xWriterErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Error Code:[%s], Description:[%s].", this.code, this.description); - } -} diff --git a/plugin/writer/hbase11xwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xwriter/HbaseAbstractTask.java b/plugin/writer/hbase11xwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xwriter/HbaseAbstractTask.java index e0381a82a..64de0788c 100644 --- a/plugin/writer/hbase11xwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xwriter/HbaseAbstractTask.java +++ b/plugin/writer/hbase11xwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xwriter/HbaseAbstractTask.java @@ -38,6 +38,10 @@ import java.nio.charset.Charset; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.IO_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.NOT_SUPPORT_TYPE; + public abstract class HbaseAbstractTask { private static final Logger LOG = LoggerFactory.getLogger(HbaseAbstractTask.class); @@ -92,7 +96,7 @@ public void startWriter(RecordReceiver lineReceiver, TaskPluginCollector taskPlu } } catch (IOException e) { - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.PUT_HBASE_ERROR, e); + throw AddaxException.asAddaxException(IO_ERROR, e); } finally { Hbase11xHelper.closeBufferedMutator(this.bufferedMutator); @@ -133,7 +137,7 @@ public byte[] getColumnByte(ColumnType columnType, Column column) bytes = this.getValueByte(columnType, column.asString()); break; default: - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.ILLEGAL_VALUE, "The data type " + columnType + "is unsupported"); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "The data type " + columnType + "is unsupported"); } } else { @@ -145,7 +149,7 @@ public byte[] getColumnByte(ColumnType columnType, Column column) bytes = HConstants.EMPTY_BYTE_ARRAY; break; default: - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.ILLEGAL_VALUE, "The item nullMode must be configured either skip or empty"); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "The item nullMode must be configured either skip or empty"); } } return bytes; @@ -178,7 +182,7 @@ public byte[] getValueByte(ColumnType columnType, String value) bytes = value.getBytes(Charset.forName(encoding)); break; default: - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.ILLEGAL_VALUE, "The data type " + columnType + "is unsupported"); + throw AddaxException.asAddaxException(NOT_SUPPORT_TYPE, "The data type " + columnType + "is unsupported"); } } else { diff --git a/plugin/writer/hbase11xwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xwriter/ModeType.java b/plugin/writer/hbase11xwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xwriter/ModeType.java index 0b60aaa9f..8fa14ba5a 100644 --- a/plugin/writer/hbase11xwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xwriter/ModeType.java +++ b/plugin/writer/hbase11xwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xwriter/ModeType.java @@ -23,6 +23,8 @@ import java.util.Arrays; +import static com.wgzhao.addax.common.spi.ErrorCode.NOT_SUPPORT_TYPE; + public enum ModeType { NORMAL("normal"), @@ -42,7 +44,7 @@ public static ModeType getByTypeName(String modeName) return modeType; } } - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(NOT_SUPPORT_TYPE, String.format("The mode %s is unsupported. %s are supported yet.", modeName, Arrays.asList(values()))); } } diff --git a/plugin/writer/hbase11xwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xwriter/MultiVersionTask.java b/plugin/writer/hbase11xwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xwriter/MultiVersionTask.java index 8b2259027..ae870ad34 100644 --- a/plugin/writer/hbase11xwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xwriter/MultiVersionTask.java +++ b/plugin/writer/hbase11xwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xwriter/MultiVersionTask.java @@ -24,6 +24,8 @@ import com.wgzhao.addax.common.util.Configuration; import org.apache.hadoop.hbase.client.Put; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; + public class MultiVersionTask extends HbaseAbstractTask { @@ -40,7 +42,7 @@ public Put convertRecordToPut(Record record) // multi-version 模式下源头读取字段列数为4元组(rowkey,column,timestamp,value),目的端需告诉[] throw AddaxException .asAddaxException( - Hbase11xWriterErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, String.format( "The record should be a tuple of (rowkey,column,timestamp,value) in multi-version mode. actually get %s", record.getColumnNumber())); diff --git a/plugin/writer/hbase11xwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xwriter/NormalTask.java b/plugin/writer/hbase11xwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xwriter/NormalTask.java index 87a7081c1..ea638c27d 100644 --- a/plugin/writer/hbase11xwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xwriter/NormalTask.java +++ b/plugin/writer/hbase11xwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xwriter/NormalTask.java @@ -37,6 +37,9 @@ import java.text.SimpleDateFormat; import java.util.Date; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; + public class NormalTask extends HbaseAbstractTask { @@ -72,7 +75,7 @@ public Put convertRecordToPut(Record record) String[] cfAndQualifier = name.split(":"); Validate.isTrue(cfAndQualifier.length == 2 && StringUtils.isNotBlank(cfAndQualifier[0]) && StringUtils.isNotBlank(cfAndQualifier[1]), promptInfo); if (index >= record.getColumnNumber()) { - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, String.format("The field[index] of column is out-range, it should be less than %s. actually got %s.", record.getColumnNumber(), index)); } byte[] columnBytes = getColumnByte(columnType, record.getColumn(index)); @@ -97,7 +100,7 @@ public byte[] getRowkey(Record record) } else { if (index >= record.getColumnNumber()) { - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.CONSTRUCT_ROWKEY_ERROR, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, String.format("The field[index] of rowkeyColumn is out-range, it should be less than %s. actually got %s.", record.getColumnNumber(), index)); } byte[] value = getColumnByte(columnType, record.getColumn(index)); @@ -115,17 +118,17 @@ public long getVersion(Record record) //指定时间作为版本 timestamp = versionColumn.getLong(HBaseKey.VALUE); if (timestamp < 0) { - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.CONSTRUCT_VERSION_ERROR, "Illegal timestamp version"); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "Illegal timestamp version"); } } else { //指定列作为版本,long/doubleColumn直接record.asLong, 其它类型尝试用yyyy-MM-dd HH:mm:ss,yyyy-MM-dd HH:mm:ss SSS去format if (index >= record.getColumnNumber()) { - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.CONSTRUCT_VERSION_ERROR, + throw AddaxException.asAddaxException(CONFIG_ERROR, String.format("The field[index] of versionColumn is out-range, it should be less than %s. actually got %s.", record.getColumnNumber(), index)); } if (record.getColumn(index).getRawData() == null) { - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.CONSTRUCT_VERSION_ERROR, "The version is empty"); + throw AddaxException.asAddaxException(CONFIG_ERROR, "The version is empty"); } SimpleDateFormat dfSeconds = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); SimpleDateFormat dfMs = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS"); @@ -143,7 +146,7 @@ public long getVersion(Record record) } catch (ParseException e1) { LOG.info(String.format("The value of version %s can not parsed as Date type with 'yyyy-MM-dd HH:mm:ss' and 'yyyy-MM-dd HH:mm:ss SSS' format", index)); - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.CONSTRUCT_VERSION_ERROR, e1); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, e1); } } timestamp = date.getTime(); diff --git a/plugin/writer/hbase11xwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xwriter/NullModeType.java b/plugin/writer/hbase11xwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xwriter/NullModeType.java index 0ae6328cd..a743ed2f9 100644 --- a/plugin/writer/hbase11xwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xwriter/NullModeType.java +++ b/plugin/writer/hbase11xwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase11xwriter/NullModeType.java @@ -23,6 +23,8 @@ import java.util.Arrays; +import static com.wgzhao.addax.common.spi.ErrorCode.NOT_SUPPORT_TYPE; + public enum NullModeType { SKIP("skip"), @@ -42,7 +44,7 @@ public static NullModeType getByTypeName(String modeName) return modeType; } } - throw AddaxException.asAddaxException(Hbase11xWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(NOT_SUPPORT_TYPE, String.format("The nullMode %s is unsupported. %s are supported at current", modeName, Arrays.asList(values()))); } } diff --git a/plugin/writer/hbase20xsqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase20xsqlwriter/HBase20xSQLHelper.java b/plugin/writer/hbase20xsqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase20xsqlwriter/HBase20xSQLHelper.java index 589285852..c5d31c906 100644 --- a/plugin/writer/hbase20xsqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase20xsqlwriter/HBase20xSQLHelper.java +++ b/plugin/writer/hbase20xsqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase20xsqlwriter/HBase20xSQLHelper.java @@ -35,6 +35,13 @@ import java.util.ArrayList; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.CONNECT_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.EXECUTE_FAIL; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.LOGIN_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; + public class HBase20xSQLHelper { @@ -56,8 +63,8 @@ private HBase20xSQLHelper() {} public static void validateParameter(Configuration originalConfig) { // 表名和queryserver地址必须配置，否则抛异常 - String tableName = originalConfig.getNecessaryValue(HBaseKey.TABLE, HBase20xSQLWriterErrorCode.REQUIRED_VALUE); - String jdbcUrl = originalConfig.getNecessaryValue(HBaseKey.JDBC_URL, HBase20xSQLWriterErrorCode.REQUIRED_VALUE); + String tableName = originalConfig.getNecessaryValue(HBaseKey.TABLE, REQUIRED_VALUE); + String jdbcUrl = originalConfig.getNecessaryValue(HBaseKey.JDBC_URL, REQUIRED_VALUE); boolean isThinMode = jdbcUrl.contains(":thin:"); // 序列化格式，可不配置，默认PROTOBUF String serialization = originalConfig.getString(HBaseKey.SERIALIZATION_NAME, HBaseConstant.DEFAULT_SERIALIZATION); @@ -67,8 +74,8 @@ public static void validateParameter(Configuration originalConfig) } // check kerberos if (originalConfig.getBool(HBaseKey.HAVE_KERBEROS, false)) { - String principal = originalConfig.getNecessaryValue(HBaseKey.KERBEROS_PRINCIPAL, HBase20xSQLWriterErrorCode.REQUIRED_VALUE); - String keytab = originalConfig.getNecessaryValue(HBaseKey.KERBEROS_KEYTAB_FILE_PATH, HBase20xSQLWriterErrorCode.REQUIRED_VALUE); + String principal = originalConfig.getNecessaryValue(HBaseKey.KERBEROS_PRINCIPAL, REQUIRED_VALUE); + String keytab = originalConfig.getNecessaryValue(HBaseKey.KERBEROS_KEYTAB_FILE_PATH, REQUIRED_VALUE); kerberosAuthentication(principal, keytab); if (isThinMode) { connStr = connStr + "authentication=SPENGO;principal=" + principal + ";keytab=" + keytab; @@ -87,7 +94,7 @@ public static void validateParameter(Configuration originalConfig) List columnNames = originalConfig.getList(HBaseKey.COLUMN, String.class); if (columnNames == null || columnNames.isEmpty()) { throw AddaxException.asAddaxException( - HBase20xSQLWriterErrorCode.ILLEGAL_VALUE, "HBase的columns配置不能为空,请添加目标表的列名配置."); + CONFIG_ERROR, "HBase的columns配置不能为空,请添加目标表的列名配置."); } String schema = originalConfig.getString(HBaseKey.SCHEMA); // 检查表以及配置列是否存在 @@ -109,7 +116,7 @@ public static Connection getClientConnection(String connStr, String driverName) conn.setAutoCommit(false); } catch (Throwable e) { - throw AddaxException.asAddaxException(HBase20xSQLWriterErrorCode.GET_QUERYSERVER_CONNECTION_ERROR, + throw AddaxException.asAddaxException(CONNECT_ERROR, "无法连接QueryServer，配置不正确或服务未启动，请检查配置和服务状态或者联系HBase管理员.", e); } LOG.debug("Connected to QueryServer successfully."); @@ -118,7 +125,7 @@ public static Connection getClientConnection(String connStr, String driverName) public static Connection getJdbcConnection(Configuration conf) { - String queryServerAddress = conf.getNecessaryValue(HBaseKey.QUERY_SERVER_ADDRESS, HBase20xSQLWriterErrorCode.REQUIRED_VALUE); + String queryServerAddress = conf.getNecessaryValue(HBaseKey.QUERY_SERVER_ADDRESS, REQUIRED_VALUE); if (queryServerAddress.contains(":thin:")) { return getClientConnection(queryServerAddress, PHOENIX_JDBC_THIN_DRIVER); } else { @@ -139,8 +146,8 @@ public static void checkTable(Connection conn, String schema, String tableName, allColumns.add(rs.getString(1)); } else { - LOG.error("表 {} 不存在，请检查表名是否正确或是否已创建. {}", tableName, HBase20xSQLWriterErrorCode.GET_HBASE_TABLE_ERROR); - throw AddaxException.asAddaxException(HBase20xSQLWriterErrorCode.GET_HBASE_TABLE_ERROR, + LOG.error("表 {} 不存在，请检查表名是否正确或是否已创建. {}", tableName, CONFIG_ERROR); + throw AddaxException.asAddaxException(CONFIG_ERROR, tableName + "表不存在，请检查表名是否正确或是否已创建."); } while (rs.next()) { @@ -149,13 +156,13 @@ public static void checkTable(Connection conn, String schema, String tableName, for (String columnName : columnNames) { if (!allColumns.contains(columnName)) { // 用户配置的列名在元数据中不存在 - throw AddaxException.asAddaxException(HBase20xSQLWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "您配置的列" + columnName + "在目的表" + tableName + "的元数据中不存在，请检查您的配置或者联系HBase管理员."); } } } catch (SQLException t) { - throw AddaxException.asAddaxException(HBase20xSQLWriterErrorCode.GET_HBASE_TABLE_ERROR, + throw AddaxException.asAddaxException(EXECUTE_FAIL, "获取表" + tableName + "信息失败，请检查您的集群和表状态或者联系HBase管理员.", t); } finally { @@ -186,7 +193,7 @@ public static void closeJdbc(Connection connection, Statement statement, ResultS } } catch (SQLException e) { - LOG.warn("数据库连接关闭异常. {}", HBase20xSQLWriterErrorCode.CLOSE_HBASE_CONNECTION_ERROR); + LOG.warn("数据库连接关闭异常. {}", EXECUTE_FAIL); } } @@ -208,7 +215,7 @@ private static void kerberosAuthentication(String kerberosPrincipal, String kerb String message = String.format("Kerberos authentication failed, please make sure that kerberosKeytabFilePath[%s] and kerberosPrincipal[%s] are correct", kerberosKeytabFilePath, kerberosPrincipal); LOG.error(message); - throw AddaxException.asAddaxException(HBase20xSQLWriterErrorCode.KERBEROS_LOGIN_ERROR, e); + throw AddaxException.asAddaxException(LOGIN_ERROR, e); } } } diff --git a/plugin/writer/hbase20xsqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase20xsqlwriter/HBase20xSQLWriterErrorCode.java b/plugin/writer/hbase20xsqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase20xsqlwriter/HBase20xSQLWriterErrorCode.java deleted file mode 100644 index d32c5d721..000000000 --- a/plugin/writer/hbase20xsqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase20xsqlwriter/HBase20xSQLWriterErrorCode.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.writer.hbase20xsqlwriter; - -import com.wgzhao.addax.common.spi.ErrorCode; - -public enum HBase20xSQLWriterErrorCode - implements ErrorCode -{ - REQUIRED_VALUE("Hbasewriter-00", "您缺失了必须填写的参数值."), - ILLEGAL_VALUE("Hbasewriter-01", "您填写的参数值不合法."), - GET_QUERYSERVER_CONNECTION_ERROR("Hbasewriter-02", "获取QueryServer连接时出错."), - GET_HBASE_TABLE_ERROR("Hbasewriter-03", "获取 Hbase table时出错."), - CLOSE_HBASE_CONNECTION_ERROR("Hbasewriter-04", "关闭Hbase连接时出错."), - GET_TABLE_COLUMN_TYPE_ERROR("Hbasewriter-05", "获取表列类型时出错."), - PUT_HBASE_ERROR("Hbasewriter-07", "写入hbase时发生IO异常."), - KERBEROS_LOGIN_ERROR("HbaseWriter-08", "Kerberos authentication failed."); - ; - - private final String code; - private final String description; - - HBase20xSQLWriterErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s].", this.code, this.description); - } -} diff --git a/plugin/writer/hbase20xsqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase20xsqlwriter/HBase20xSQLWriterTask.java b/plugin/writer/hbase20xsqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase20xsqlwriter/HBase20xSQLWriterTask.java index 37ea7e7cb..bda24733e 100644 --- a/plugin/writer/hbase20xsqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase20xsqlwriter/HBase20xSQLWriterTask.java +++ b/plugin/writer/hbase20xsqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase20xsqlwriter/HBase20xSQLWriterTask.java @@ -44,6 +44,11 @@ import java.util.Arrays; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.EXECUTE_FAIL; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; + public class HBase20xSQLWriterTask { private static final Logger LOG = LoggerFactory.getLogger(HBase20xSQLWriterTask.class); @@ -83,7 +88,7 @@ public void startWriter(RecordReceiver lineReceiver, TaskPluginCollector taskPlu writeData(lineReceiver); } catch (Throwable e) { - throw AddaxException.asAddaxException(HBase20xSQLWriterErrorCode.PUT_HBASE_ERROR, e); + throw AddaxException.asAddaxException(RUNTIME_ERROR, e); } finally { // 关闭jdbc连接 @@ -106,7 +111,7 @@ private void initialize() nullModeType = NullModeType.getByTypeName(configuration.getString(HBaseKey.NULL_MODE, HBaseConstant.DEFAULT_NULL_MODE)); batchSize = configuration.getInt(HBaseKey.BATCH_SIZE, HBaseConstant.DEFAULT_BATCH_ROW_COUNT); String schema = configuration.getString(HBaseKey.SCHEMA); - String tableName = configuration.getNecessaryValue(HBaseKey.TABLE, HBase20xSQLWriterErrorCode.REQUIRED_VALUE); + String tableName = configuration.getNecessaryValue(HBaseKey.TABLE, REQUIRED_VALUE); fullTableName = "\"" + tableName + "\""; if (schema != null && !schema.isEmpty()) { fullTableName = "\"" + schema + "\".\"" + tableName + "\""; @@ -184,7 +189,7 @@ private int[] getColumnSqlType() } } catch (SQLException e) { - throw AddaxException.asAddaxException(HBase20xSQLWriterErrorCode.GET_TABLE_COLUMN_TYPE_ERROR, + throw AddaxException.asAddaxException(EXECUTE_FAIL, "获取表" + fullTableName + "列类型失败，请检查配置和服务状态或者联系HBase管理员.", e); } finally { @@ -208,7 +213,7 @@ private void writeData(RecordReceiver lineReceiver) while ((record = lineReceiver.getFromReader()) != null) { // 校验列数量是否符合预期 if (record.getColumnNumber() != numberOfColumnsToRead) { - throw AddaxException.asAddaxException(HBase20xSQLWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "数据源给出的列数量[" + record.getColumnNumber() + "]与您配置中的列数量[" + numberOfColumnsToRead + "]不同, 请检查您的配置 或者 联系 Hbase 管理员."); } @@ -260,7 +265,7 @@ private void doBatchUpsert(List records) doSingleUpsert(records); } catch (Exception e) { - throw AddaxException.asAddaxException(HBase20xSQLWriterErrorCode.PUT_HBASE_ERROR, e); + throw AddaxException.asAddaxException(RUNTIME_ERROR, e); } } @@ -364,7 +369,7 @@ private void setupColumn(int pos, int sqlType, Column col) break; default: - throw AddaxException.asAddaxException(HBase20xSQLWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "不支持您配置的列类型:" + sqlType + ", 请检查您的配置 或者 联系 Hbase 管理员."); } } @@ -384,7 +389,7 @@ private void setupColumn(int pos, int sqlType, Column col) default: // nullMode的合法性在初始化配置的时候已经校验过，这里一定不会出错 - throw AddaxException.asAddaxException(HBase20xSQLWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "Hbasewriter 不支持该 nullMode 类型: " + nullModeType + ", 目前支持的 nullMode 类型是:" + Arrays.asList(NullModeType.values())); } @@ -449,7 +454,7 @@ private Object getEmptyValue(int sqlType) return new byte[0]; default: - throw AddaxException.asAddaxException(HBase20xSQLWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "不支持您配置的列类型:" + sqlType + ", 请检查您的配置 或者 联系 Hbase 管理员."); } } diff --git a/plugin/writer/hbase20xsqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase20xsqlwriter/NullModeType.java b/plugin/writer/hbase20xsqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase20xsqlwriter/NullModeType.java index 01d2036d1..15aeddf1b 100644 --- a/plugin/writer/hbase20xsqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase20xsqlwriter/NullModeType.java +++ b/plugin/writer/hbase20xsqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/hbase20xsqlwriter/NullModeType.java @@ -23,6 +23,8 @@ import java.util.Arrays; +import static com.wgzhao.addax.common.spi.ErrorCode.NOT_SUPPORT_TYPE; + public enum NullModeType { SKIP("skip"), @@ -42,7 +44,7 @@ public static NullModeType getByTypeName(String modeName) return modeType; } } - throw AddaxException.asAddaxException(HBase20xSQLWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(NOT_SUPPORT_TYPE, "Hbasewriter 不支持该 nullMode 类型:" + modeName + ", 目前支持的 nullMode 类型是:" + Arrays.asList(values())); } } diff --git a/plugin/writer/hdfswriter/src/main/java/com/wgzhao/addax/plugin/writer/hdfswriter/HdfsHelper.java b/plugin/writer/hdfswriter/src/main/java/com/wgzhao/addax/plugin/writer/hdfswriter/HdfsHelper.java index 27c07ca09..d3b50e6bb 100644 --- a/plugin/writer/hdfswriter/src/main/java/com/wgzhao/addax/plugin/writer/hdfswriter/HdfsHelper.java +++ b/plugin/writer/hdfswriter/src/main/java/com/wgzhao/addax/plugin/writer/hdfswriter/HdfsHelper.java @@ -48,6 +48,11 @@ import static com.wgzhao.addax.common.base.Key.HAVE_KERBEROS; import static com.wgzhao.addax.common.base.Key.KERBEROS_KEYTAB_FILE_PATH; import static com.wgzhao.addax.common.base.Key.KERBEROS_PRINCIPAL; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.IO_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.LOGIN_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.NOT_SUPPORT_TYPE; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; public class HdfsHelper { @@ -94,12 +99,12 @@ protected void getFileSystem(Configuration taskConfig) String message = String.format("Network IO exception occurred while obtaining Filesystem with defaultFS: [%s]", defaultFS); LOG.error(message); - throw AddaxException.asAddaxException(HdfsWriterErrorCode.CONNECT_HDFS_IO_ERROR, e); + throw AddaxException.asAddaxException(IO_ERROR, e); } catch (Exception e) { String message = String.format("Failed to obtain Filesystem with defaultFS: [%s]", defaultFS); LOG.error(message); - throw AddaxException.asAddaxException(HdfsWriterErrorCode.CONNECT_HDFS_IO_ERROR, e); + throw AddaxException.asAddaxException(RUNTIME_ERROR, e); } } @@ -114,7 +119,7 @@ private void kerberosAuthentication(String kerberosPrincipal, String kerberosKey String message = String.format("kerberos authentication failed, keytab file: [%s], principal: [%s]", kerberosKeytabFilePath, kerberosPrincipal); LOG.error(message); - throw AddaxException.asAddaxException(HdfsWriterErrorCode.KERBEROS_LOGIN_ERROR, e); + throw AddaxException.asAddaxException(LOGIN_ERROR, e); } } } @@ -140,7 +145,7 @@ public Path[] hdfsDirList(String dir) catch (IOException e) { String message = String.format("Network IO exception occurred while fetching file list for directory [%s]", dir); LOG.error(message); - throw AddaxException.asAddaxException(HdfsWriterErrorCode.CONNECT_HDFS_IO_ERROR, e); + throw AddaxException.asAddaxException(IO_ERROR, e); } return files; } @@ -154,7 +159,7 @@ public boolean isPathExists(String filePath) } catch (IOException e) { LOG.error("Network IO exception occurred while checking if file path [{}] exists", filePath); - throw AddaxException.asAddaxException(HdfsWriterErrorCode.CONNECT_HDFS_IO_ERROR, e); + throw AddaxException.asAddaxException(IO_ERROR, e); } return exist; } @@ -168,7 +173,7 @@ public boolean isPathDir(String filePath) } catch (IOException e) { LOG.error("Network IO exception occurred while checking if path [{}] is directory or not.", filePath); - throw AddaxException.asAddaxException(HdfsWriterErrorCode.CONNECT_HDFS_IO_ERROR, e); + throw AddaxException.asAddaxException(IO_ERROR, e); } return isDir; } @@ -197,10 +202,10 @@ public void deleteFilesFromDir(Path dir, boolean skipTrash) } } catch (FileNotFoundException fileNotFoundException) { - throw new AddaxException(HdfsWriterErrorCode.FILE_NOT_FOUND, fileNotFoundException.getMessage()); + throw new AddaxException(CONFIG_ERROR, fileNotFoundException.getMessage()); } catch (IOException ioException) { - throw new AddaxException(HdfsWriterErrorCode.IO_ERROR, ioException.getMessage()); + throw new AddaxException(IO_ERROR, ioException.getMessage()); } } @@ -212,9 +217,9 @@ public void deleteDir(Path path) fileSystem.delete(path, true); } } - catch (Exception e) { + catch (IOException e) { LOG.error("IO exception occurred while delete temporary directory [{}].", path); - throw AddaxException.asAddaxException(HdfsWriterErrorCode.CONNECT_HDFS_IO_ERROR, e); + throw AddaxException.asAddaxException(IO_ERROR, e); } LOG.info("Finish deleting temporary dir [{}] .", path); } @@ -237,7 +242,7 @@ public void moveFilesToDest(Path sourceDir, Path targetDir) } } catch (IOException e) { - throw AddaxException.asAddaxException(HdfsWriterErrorCode.IO_ERROR, e); + throw AddaxException.asAddaxException(IO_ERROR, e); } LOG.info("Finish moving file(s)."); } @@ -250,7 +255,7 @@ public void closeFileSystem() } catch (IOException e) { LOG.error("IO exception occurred while closing Filesystem."); - throw AddaxException.asAddaxException(HdfsWriterErrorCode.CONNECT_HDFS_IO_ERROR, e); + throw AddaxException.asAddaxException(IO_ERROR, e); } } @@ -280,7 +285,7 @@ public Class getCompressCodec(String compress) codecClass = org.apache.hadoop.io.compress.DeflateCodec.class; break; default: - throw AddaxException.asAddaxException(HdfsWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(NOT_SUPPORT_TYPE, String.format("The compress mode [%s} is unsupported yet.", compress)); } return codecClass; diff --git a/plugin/writer/hdfswriter/src/main/java/com/wgzhao/addax/plugin/writer/hdfswriter/HdfsWriter.java b/plugin/writer/hdfswriter/src/main/java/com/wgzhao/addax/plugin/writer/hdfswriter/HdfsWriter.java index 4c36f399a..10b649d7b 100644 --- a/plugin/writer/hdfswriter/src/main/java/com/wgzhao/addax/plugin/writer/hdfswriter/HdfsWriter.java +++ b/plugin/writer/hdfswriter/src/main/java/com/wgzhao/addax/plugin/writer/hdfswriter/HdfsWriter.java @@ -45,6 +45,9 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; + public class HdfsWriter extends Writer { @@ -81,38 +84,38 @@ public void init() private void validateParameter() { - String defaultFS = this.writerSliceConfig.getNecessaryValue(Key.DEFAULT_FS, HdfsWriterErrorCode.REQUIRED_VALUE); + String defaultFS = this.writerSliceConfig.getNecessaryValue(Key.DEFAULT_FS, REQUIRED_VALUE); //fileType check - String fileType = this.writerSliceConfig.getNecessaryValue(Key.FILE_TYPE, HdfsWriterErrorCode.REQUIRED_VALUE).toUpperCase(); + String fileType = this.writerSliceConfig.getNecessaryValue(Key.FILE_TYPE, REQUIRED_VALUE).toUpperCase(); if (!SUPPORT_FORMAT.contains(fileType)) { String message = String.format("The file format [%s] is supported yet, the plugin currently only supports: [%s].", fileType, SUPPORT_FORMAT); - throw AddaxException.asAddaxException(HdfsWriterErrorCode.ILLEGAL_VALUE, message); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, message); } //path - this.path = this.writerSliceConfig.getNecessaryValue(Key.PATH, HdfsWriterErrorCode.REQUIRED_VALUE); + this.path = this.writerSliceConfig.getNecessaryValue(Key.PATH, REQUIRED_VALUE); if (!path.startsWith("/")) { String message = String.format("The path [%s] must be configured as an absolute path.", path); LOG.error(message); - throw AddaxException.asAddaxException(HdfsWriterErrorCode.ILLEGAL_VALUE, message); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, message); } if (path.contains("*") || path.contains("?")) { String message = String.format("The path [%s] cannot contain special characters like '*','?'.", path); LOG.error(message); - throw AddaxException.asAddaxException(HdfsWriterErrorCode.ILLEGAL_VALUE, message); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, message); } //fileName - this.fileName = this.writerSliceConfig.getNecessaryValue(Key.FILE_NAME, HdfsWriterErrorCode.REQUIRED_VALUE); + this.fileName = this.writerSliceConfig.getNecessaryValue(Key.FILE_NAME, REQUIRED_VALUE); //columns check List columns = this.writerSliceConfig.getListConfiguration(Key.COLUMN); if (null == columns || columns.isEmpty()) { - throw AddaxException.asAddaxException(HdfsWriterErrorCode.REQUIRED_VALUE, "The item columns should be configured"); + throw AddaxException.asAddaxException(REQUIRED_VALUE, "The item columns should be configured"); } else { boolean rewriteFlag = false; for (int i = 0; i < columns.size(); i++) { Configuration eachColumnConf = columns.get(i); - eachColumnConf.getNecessaryValue(Key.NAME, HdfsWriterErrorCode.COLUMN_REQUIRED_VALUE); - eachColumnConf.getNecessaryValue(Key.TYPE, HdfsWriterErrorCode.COLUMN_REQUIRED_VALUE); + eachColumnConf.getNecessaryValue(Key.NAME, REQUIRED_VALUE); + eachColumnConf.getNecessaryValue(Key.TYPE, REQUIRED_VALUE); if (eachColumnConf.getString(Key.TYPE).toUpperCase().startsWith("DECIMAL")) { String type = eachColumnConf.getString(Key.TYPE); eachColumnConf.set(Key.TYPE, "decimal"); @@ -127,9 +130,9 @@ private void validateParameter() } } //writeMode check - this.writeMode = this.writerSliceConfig.getNecessaryValue(Key.WRITE_MODE, HdfsWriterErrorCode.REQUIRED_VALUE); + this.writeMode = this.writerSliceConfig.getNecessaryValue(Key.WRITE_MODE, REQUIRED_VALUE); if (!Constant.SUPPORTED_WRITE_MODE.contains(writeMode)) { - throw AddaxException.asAddaxException(HdfsWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, String.format("The item writeMode only supports append, noConflict and overwrite, [%s] is unsupported yet.", writeMode)); } @@ -137,13 +140,13 @@ private void validateParameter() //fieldDelimiter check String fieldDelimiter = this.writerSliceConfig.getString(Key.FIELD_DELIMITER, null); if (StringUtils.isEmpty(fieldDelimiter)) { - throw AddaxException.asAddaxException(HdfsWriterErrorCode.REQUIRED_VALUE, + throw AddaxException.asAddaxException(REQUIRED_VALUE, String.format("The item [%s] should be configured and valid while write TEXT file.", Key.FIELD_DELIMITER)); } if (1 != fieldDelimiter.length()) { // warn: if it has, length must be one - throw AddaxException.asAddaxException(HdfsWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, String.format("The field delimiter is only support single character, your configure: [%s]", fieldDelimiter)); } } @@ -155,7 +158,7 @@ private void validateParameter() CompressionKind.valueOf(compress); } catch (IllegalArgumentException e) { - throw AddaxException.asAddaxException(HdfsWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, String.format("The ORC format only supports [%s] compression. your configure [%s] is unsupported yet.", Arrays.toString(CompressionKind.values()), compress)); } @@ -169,7 +172,7 @@ private void validateParameter() CompressionCodecName.fromConf(compress); } catch (Exception e) { - throw AddaxException.asAddaxException(HdfsWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, String.format("The PARQUET format only supports [%s] compression. your configure [%s] is unsupported yet.", Arrays.toString(CompressionCodecName.values()), compress)); } @@ -178,8 +181,8 @@ private void validateParameter() //Kerberos check boolean haveKerberos = this.writerSliceConfig.getBool(Key.HAVE_KERBEROS, false); if (haveKerberos) { - this.writerSliceConfig.getNecessaryValue(Key.KERBEROS_KEYTAB_FILE_PATH, HdfsWriterErrorCode.REQUIRED_VALUE); - this.writerSliceConfig.getNecessaryValue(Key.KERBEROS_PRINCIPAL, HdfsWriterErrorCode.REQUIRED_VALUE); + this.writerSliceConfig.getNecessaryValue(Key.KERBEROS_KEYTAB_FILE_PATH, REQUIRED_VALUE); + this.writerSliceConfig.getNecessaryValue(Key.KERBEROS_PRINCIPAL, REQUIRED_VALUE); } // encoding check String encoding = this.writerSliceConfig.getString(Key.ENCODING, Constant.DEFAULT_ENCODING); @@ -189,7 +192,7 @@ private void validateParameter() Charsets.toCharset(encoding); } catch (Exception e) { - throw AddaxException.asAddaxException(HdfsWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, String.format("The encoding [%s] is unsupported yet.", encoding), e); } @@ -206,7 +209,7 @@ public void prepare() // Verify whether the path is a directory if it exists. if (hdfsHelper.isPathExists(path)) { if (!hdfsHelper.isPathDir(path)) { - throw AddaxException.asAddaxException(HdfsWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, String.format("The item path you configured [%s] is exists ,but it is not directory", path)); } @@ -218,19 +221,19 @@ public void prepare() "Files with the prefix [{}] are written in the [{}] directory.", fileName, path); } else if ("nonConflict".equals(writeMode) && isExistFile) { - throw AddaxException.asAddaxException(HdfsWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, String.format("The current writeMode is set to 'nonConflict', but the directory [%s] is not empty, it includes the sub-path(s): [%s]", path, String.join(",", Arrays.stream(existFilePaths).map(Path::getName).collect(Collectors.toSet())))); } } else { - throw AddaxException.asAddaxException(HdfsWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, String.format("The directory [%s] does not exists. please create it first. ", path)); } // validate the write permission if (!hdfsHelper.isPathWritable(path)) { - throw AddaxException.asAddaxException(HdfsWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, String.format("The path [%s] is not writable or permission denied", path)); } } @@ -400,7 +403,7 @@ public void startWrite(RecordReceiver lineReceiver) hdfsHelper = new ParquetWriter(this.writerSliceConfig); break; default: - throw AddaxException.asAddaxException(HdfsWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, String.format("The file format [%s] is supported yet, the plugin currently only supports: [%s].", fileType, Job.SUPPORT_FORMAT)); } //得当的已经是绝对路径，eg：/user/hive/warehouse/writer.db/text/test.snappy diff --git a/plugin/writer/hdfswriter/src/main/java/com/wgzhao/addax/plugin/writer/hdfswriter/HdfsWriterErrorCode.java b/plugin/writer/hdfswriter/src/main/java/com/wgzhao/addax/plugin/writer/hdfswriter/HdfsWriterErrorCode.java deleted file mode 100644 index 106a3052b..000000000 --- a/plugin/writer/hdfswriter/src/main/java/com/wgzhao/addax/plugin/writer/hdfswriter/HdfsWriterErrorCode.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.writer.hdfswriter; - -import com.wgzhao.addax.common.spi.ErrorCode; - -/** - * Created by shf on 15/10/8. - */ -public enum HdfsWriterErrorCode - implements ErrorCode -{ - - // CONFIG_INVALID_EXCEPTION("HdfsWriter-00", "您的参数配置错误."), - REQUIRED_VALUE("HdfsWriter-01", "您缺失了必须填写的参数值."), - ILLEGAL_VALUE("HdfsWriter-02", "您填写的参数值不合法."), - // WRITER_FILE_WITH_CHARSET_ERROR("HdfsWriter-03", "您配置的编码未能正常写入."), - Write_FILE_IO_ERROR("HdfsWriter-04", "您配置的文件在写入时出现IO异常."), - // WRITER_RUNTIME_EXCEPTION("HdfsWriter-05", "出现运行时异常, 请联系我们."), - CONNECT_HDFS_IO_ERROR("HdfsWriter-06", "与HDFS建立连接时出现IO异常."), - COLUMN_REQUIRED_VALUE("HdfsWriter-07", "您column配置中缺失了必须填写的参数值."), - HDFS_RENAME_FILE_ERROR("HdfsWriter-08", "将文件移动到配置路径失败."), - KERBEROS_LOGIN_ERROR("HdfsWriter-09", "KERBEROS认证失败"), - FILE_NOT_FOUND("HdfsWriter-10", "文件或目录不存在."), - IO_ERROR("HdfsWriter-11", "IO异常."), - ; - - private final String code; - private final String description; - - HdfsWriterErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s].", this.code, - this.description); - } -} diff --git a/plugin/writer/hdfswriter/src/main/java/com/wgzhao/addax/plugin/writer/hdfswriter/OrcWriter.java b/plugin/writer/hdfswriter/src/main/java/com/wgzhao/addax/plugin/writer/hdfswriter/OrcWriter.java index f211cd919..78daad6b9 100644 --- a/plugin/writer/hdfswriter/src/main/java/com/wgzhao/addax/plugin/writer/hdfswriter/OrcWriter.java +++ b/plugin/writer/hdfswriter/src/main/java/com/wgzhao/addax/plugin/writer/hdfswriter/OrcWriter.java @@ -34,6 +34,10 @@ import java.util.List; import java.util.StringJoiner; +import static com.wgzhao.addax.common.spi.ErrorCode.IO_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.NOT_SUPPORT_TYPE; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; + public class OrcWriter extends HdfsHelper implements IHDFSWriter @@ -131,7 +135,7 @@ else if (colType == Column.Type.DATE) { default: throw AddaxException .asAddaxException( - HdfsWriterErrorCode.ILLEGAL_VALUE, + NOT_SUPPORT_TYPE, String.format("The columns configuration is incorrect. the field type is unsupported yet. Field name: [%s], Field type name:[%s].", eachColumnConf.getString(Key.NAME), eachColumnConf.getString(Key.TYPE))); @@ -139,7 +143,7 @@ else if (colType == Column.Type.DATE) { } catch (Exception e) { taskPluginCollector.collectDirtyRecord(record, e.getMessage()); - throw AddaxException.asAddaxException(HdfsWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(RUNTIME_ERROR, String.format("Failed to set ORC row, source field type: %s, destination field original type: %s, " + "destination field hive type: %s, field name: %s, source field value: %s, root cause:%n%s", record.getColumn(i).getType(), columnType, eachColumnConf.getString(Key.TYPE), @@ -190,7 +194,7 @@ public void write(RecordReceiver lineReceiver, Configuration config, String file logger.error("IO exception occurred while writing file [{}}.", fileName); Path path = new Path(fileName); deleteDir(path.getParent()); - throw AddaxException.asAddaxException(HdfsWriterErrorCode.Write_FILE_IO_ERROR, e); + throw AddaxException.asAddaxException(IO_ERROR, e); } } } diff --git a/plugin/writer/hdfswriter/src/main/java/com/wgzhao/addax/plugin/writer/hdfswriter/TextWriter.java b/plugin/writer/hdfswriter/src/main/java/com/wgzhao/addax/plugin/writer/hdfswriter/TextWriter.java index 9a3c990f6..35b876597 100644 --- a/plugin/writer/hdfswriter/src/main/java/com/wgzhao/addax/plugin/writer/hdfswriter/TextWriter.java +++ b/plugin/writer/hdfswriter/src/main/java/com/wgzhao/addax/plugin/writer/hdfswriter/TextWriter.java @@ -41,12 +41,16 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.sql.Timestamp; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.IO_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.NOT_SUPPORT_TYPE; + public class TextWriter extends HdfsHelper implements IHDFSWriter @@ -92,11 +96,11 @@ public void write(RecordReceiver lineReceiver, Configuration config, String file } writer.close(Reporter.NULL); } - catch (Exception e) { + catch (IOException e) { logger.error("IO exception occurred while writing text file [{}]", fileName); Path path = new Path(fileName); deleteDir(path.getParent()); - throw AddaxException.asAddaxException(HdfsWriterErrorCode.Write_FILE_IO_ERROR, e); + throw AddaxException.asAddaxException(IO_ERROR, e); } } @@ -172,7 +176,7 @@ public static MutablePair, Boolean> transportOneRecord( break; default: throw AddaxException.asAddaxException( - HdfsWriterErrorCode.ILLEGAL_VALUE, + NOT_SUPPORT_TYPE, String.format( "The configuration is incorrect. The database does not support writing this type of field. " + "Field name: [%s], field type: [%s].", diff --git a/plugin/writer/influxdb2writer/src/main/java/com/wgzhao/addax/plugin/writer/influxdb2writer/InfluxDB2Writer.java b/plugin/writer/influxdb2writer/src/main/java/com/wgzhao/addax/plugin/writer/influxdb2writer/InfluxDB2Writer.java index f8e41215b..621485414 100644 --- a/plugin/writer/influxdb2writer/src/main/java/com/wgzhao/addax/plugin/writer/influxdb2writer/InfluxDB2Writer.java +++ b/plugin/writer/influxdb2writer/src/main/java/com/wgzhao/addax/plugin/writer/influxdb2writer/InfluxDB2Writer.java @@ -42,6 +42,7 @@ import static com.wgzhao.addax.common.base.Key.CONNECTION; import static com.wgzhao.addax.common.base.Key.ENDPOINT; import static com.wgzhao.addax.common.base.Key.TABLE; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; public class InfluxDB2Writer extends Writer @@ -56,15 +57,15 @@ public static class Job public void init() { this.originalConfig = super.getPluginJobConf(); - originalConfig.getNecessaryValue(InfluxDB2Key.TOKEN, InfluxDB2WriterErrorCode.REQUIRED_VALUE); + originalConfig.getNecessaryValue(InfluxDB2Key.TOKEN, REQUIRED_VALUE); Configuration connConf = Configuration.from(originalConfig.getList(CONNECTION, Object.class).get(0).toString()); - connConf.getNecessaryValue(InfluxDB2Key.ENDPOINT, InfluxDB2WriterErrorCode.REQUIRED_VALUE); - connConf.getNecessaryValue(InfluxDB2Key.BUCKET, InfluxDB2WriterErrorCode.REQUIRED_VALUE); - connConf.getNecessaryValue(InfluxDB2Key.ORG, InfluxDB2WriterErrorCode.REQUIRED_VALUE); - connConf.getNecessaryValue(TABLE, InfluxDB2WriterErrorCode.REQUIRED_VALUE); + connConf.getNecessaryValue(InfluxDB2Key.ENDPOINT, REQUIRED_VALUE); + connConf.getNecessaryValue(InfluxDB2Key.BUCKET, REQUIRED_VALUE); + connConf.getNecessaryValue(InfluxDB2Key.ORG, REQUIRED_VALUE); + connConf.getNecessaryValue(TABLE, REQUIRED_VALUE); List columns = originalConfig.getList(COLUMN, String.class); if (columns == null || columns.isEmpty() || (columns.size() == 1 && "*".equals(columns.get(0)))) { - throw AddaxException.asAddaxException(InfluxDB2WriterErrorCode.REQUIRED_VALUE, + throw AddaxException.asAddaxException(REQUIRED_VALUE, "The column must be configured and '*' is not supported yet"); } } diff --git a/plugin/writer/influxdb2writer/src/main/java/com/wgzhao/addax/plugin/writer/influxdb2writer/InfluxDB2WriterErrorCode.java b/plugin/writer/influxdb2writer/src/main/java/com/wgzhao/addax/plugin/writer/influxdb2writer/InfluxDB2WriterErrorCode.java deleted file mode 100644 index 47a58da45..000000000 --- a/plugin/writer/influxdb2writer/src/main/java/com/wgzhao/addax/plugin/writer/influxdb2writer/InfluxDB2WriterErrorCode.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.writer.influxdb2writer; - -import com.wgzhao.addax.common.spi.ErrorCode; - -public enum InfluxDB2WriterErrorCode - implements ErrorCode -{ - REQUIRED_VALUE("InfluxDB2Writer-00", "Missing mandatory configuration items"); - - private final String code; - private final String description; - - InfluxDB2WriterErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s]. ", this.code, this.description); - } -} diff --git a/plugin/writer/influxdbwriter/src/main/java/com/wgzhao/addax/plugin/writer/influxdbwriter/InfluxDBWriter.java b/plugin/writer/influxdbwriter/src/main/java/com/wgzhao/addax/plugin/writer/influxdbwriter/InfluxDBWriter.java index c55b56f54..45d76983b 100644 --- a/plugin/writer/influxdbwriter/src/main/java/com/wgzhao/addax/plugin/writer/influxdbwriter/InfluxDBWriter.java +++ b/plugin/writer/influxdbwriter/src/main/java/com/wgzhao/addax/plugin/writer/influxdbwriter/InfluxDBWriter.java @@ -30,6 +30,10 @@ import java.util.ArrayList; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.CONNECT_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; + public class InfluxDBWriter extends Writer { @@ -50,15 +54,15 @@ public void init() this.originalConfig = super.getPluginJobConf(); List connList = originalConfig.getList(InfluxDBKey.CONNECTION); Configuration conn = Configuration.from(connList.get(0).toString()); - conn.getNecessaryValue(InfluxDBKey.TABLE, InfluxDBWriterErrorCode.REQUIRED_VALUE); - this.endpoint = conn.getNecessaryValue(InfluxDBKey.ENDPOINT, InfluxDBWriterErrorCode.REQUIRED_VALUE); - this.database = conn.getNecessaryValue(InfluxDBKey.DATABASE, InfluxDBWriterErrorCode.REQUIRED_VALUE); + conn.getNecessaryValue(InfluxDBKey.TABLE, REQUIRED_VALUE); + this.endpoint = conn.getNecessaryValue(InfluxDBKey.ENDPOINT, REQUIRED_VALUE); + this.database = conn.getNecessaryValue(InfluxDBKey.DATABASE, REQUIRED_VALUE); this.username = originalConfig.getString(InfluxDBKey.USERNAME); this.password = originalConfig.getString(InfluxDBKey.PASSWORD); List columns = originalConfig.getList(InfluxDBKey.COLUMN, String.class); if (columns == null || columns.isEmpty()) { throw AddaxException.asAddaxException( - InfluxDBWriterErrorCode.REQUIRED_VALUE, + REQUIRED_VALUE, "The parameter [" + InfluxDBKey.COLUMN + "] is not set."); } } @@ -76,7 +80,7 @@ public void prepare() } catch (Exception e) { throw AddaxException.asAddaxException( - InfluxDBWriterErrorCode.CONNECT_ERROR, e + CONNECT_ERROR, e ); } } @@ -105,7 +109,7 @@ public void post() } catch (Exception e) { throw AddaxException.asAddaxException( - InfluxDBWriterErrorCode.ILLEGAL_VALUE, e + ILLEGAL_VALUE, e ); } } diff --git a/plugin/writer/influxdbwriter/src/main/java/com/wgzhao/addax/plugin/writer/influxdbwriter/InfluxDBWriterErrorCode.java b/plugin/writer/influxdbwriter/src/main/java/com/wgzhao/addax/plugin/writer/influxdbwriter/InfluxDBWriterErrorCode.java deleted file mode 100644 index 6ade4e2fd..000000000 --- a/plugin/writer/influxdbwriter/src/main/java/com/wgzhao/addax/plugin/writer/influxdbwriter/InfluxDBWriterErrorCode.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.writer.influxdbwriter; - -import com.wgzhao.addax.common.spi.ErrorCode; - -public enum InfluxDBWriterErrorCode - implements ErrorCode -{ - REQUIRED_VALUE("InfluxDBWriter-00","缺失必要的值"), - ILLEGAL_VALUE("InfluxDBWriter-01","值非法"), - CONF_ERROR("InfluxDBWriter-02", "您的配置错误."), - CONNECT_ERROR("InfluxDBWriter-03","连接错误"), - WRITER_ERROR("InfluxDBWriter-04", "写入错误"); - - private final String code; - private final String description; - - InfluxDBWriterErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s]. ", this.code, - this.description); - } -} diff --git a/plugin/writer/influxdbwriter/src/main/java/com/wgzhao/addax/plugin/writer/influxdbwriter/InfluxDBWriterTask.java b/plugin/writer/influxdbwriter/src/main/java/com/wgzhao/addax/plugin/writer/influxdbwriter/InfluxDBWriterTask.java index cb6632404..dfa095956 100644 --- a/plugin/writer/influxdbwriter/src/main/java/com/wgzhao/addax/plugin/writer/influxdbwriter/InfluxDBWriterTask.java +++ b/plugin/writer/influxdbwriter/src/main/java/com/wgzhao/addax/plugin/writer/influxdbwriter/InfluxDBWriterTask.java @@ -37,6 +37,9 @@ import java.util.*; import java.util.concurrent.TimeUnit; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; + public class InfluxDBWriterTask { private static final Logger LOG = LoggerFactory.getLogger(InfluxDBWriterTask.class); @@ -169,7 +172,7 @@ public void startWrite(RecordReceiver recordReceiver, TaskPluginCollector taskPl while ((record = recordReceiver.getFromReader()) != null) { if (record.getColumnNumber() != this.columnNumber) { throw AddaxException.asAddaxException( - InfluxDBWriterErrorCode.CONF_ERROR, + CONFIG_ERROR, String.format( "列配置信息有错误. 因为您配置的任务中，源头读取字段数:%s 与 目的表要写入的字段数:%s 不相等. 请检查您的配置并作出修改.", record.getColumnNumber(), @@ -222,7 +225,7 @@ public void startWrite(RecordReceiver recordReceiver, TaskPluginCollector taskPl } catch (Exception e) { taskPluginCollector.collectDirtyRecord(record, e); - throw AddaxException.asAddaxException(InfluxDBWriterErrorCode.ILLEGAL_VALUE, e); + throw AddaxException.asAddaxException(RUNTIME_ERROR, e); } } } diff --git a/plugin/writer/kafkawriter/src/main/java/com/wgzhao/addax/plugin/writer/kafkawriter/KafkaWriter.java b/plugin/writer/kafkawriter/src/main/java/com/wgzhao/addax/plugin/writer/kafkawriter/KafkaWriter.java index e2f2aac5d..8b6709182 100644 --- a/plugin/writer/kafkawriter/src/main/java/com/wgzhao/addax/plugin/writer/kafkawriter/KafkaWriter.java +++ b/plugin/writer/kafkawriter/src/main/java/com/wgzhao/addax/plugin/writer/kafkawriter/KafkaWriter.java @@ -36,6 +36,10 @@ import static com.wgzhao.addax.common.base.Constant.DEFAULT_BATCH_SIZE; import static com.wgzhao.addax.common.base.Key.BATCH_SIZE; import static com.wgzhao.addax.common.base.Key.COLUMN; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; import static com.wgzhao.addax.plugin.writer.kafkawriter.KafkaKey.BROKER_LIST; import static com.wgzhao.addax.plugin.writer.kafkawriter.KafkaKey.PROPERTIES; import static com.wgzhao.addax.plugin.writer.kafkawriter.KafkaKey.TOPIC; @@ -57,11 +61,11 @@ public void init() private void validateParameter() { - config.getNecessaryValue(BROKER_LIST, KafkaWriterErrorCode.REQUIRED_VALUE); - config.getNecessaryValue(TOPIC, KafkaWriterErrorCode.REQUIRED_VALUE); + config.getNecessaryValue(BROKER_LIST, REQUIRED_VALUE); + config.getNecessaryValue(TOPIC, REQUIRED_VALUE); List columns = config.getList(COLUMN, String.class); if (columns == null || columns.isEmpty() || (columns.size() == 1 && Objects.equals(columns.get(0), "*"))) { - throw AddaxException.asAddaxException(KafkaWriterErrorCode.ILLEGAL_VALUE, "the item column must be configure and be not " + + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "the item column must be configure and be not " + "'*'"); } } @@ -130,7 +134,7 @@ public void startWrite(RecordReceiver lineReceiver) if (record.getColumnNumber() != columns.size()) { String msg = String.format("Your item column has %d , but the record has %d", columns.size(), record.getColumnNumber()); - throw AddaxException.asAddaxException(KafkaWriterErrorCode.NOT_MATCHED_COLUMNS, msg); + throw AddaxException.asAddaxException(CONFIG_ERROR, msg); } Map message = new HashMap<>(); for (int i = 0; i < record.getColumnNumber(); i++) { @@ -168,7 +172,7 @@ public void startWrite(RecordReceiver lineReceiver) kafkaProducer.send(producerRecord, (metadata, exception) -> { if (exception != null) { - throw AddaxException.asAddaxException(KafkaWriterErrorCode.PUT_KAFKA_ERROR, exception); + throw AddaxException.asAddaxException(RUNTIME_ERROR, exception); } }); } diff --git a/plugin/writer/kafkawriter/src/main/java/com/wgzhao/addax/plugin/writer/kafkawriter/KafkaWriterErrorCode.java b/plugin/writer/kafkawriter/src/main/java/com/wgzhao/addax/plugin/writer/kafkawriter/KafkaWriterErrorCode.java deleted file mode 100644 index cc4f58d24..000000000 --- a/plugin/writer/kafkawriter/src/main/java/com/wgzhao/addax/plugin/writer/kafkawriter/KafkaWriterErrorCode.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.writer.kafkawriter; - -import com.wgzhao.addax.common.spi.ErrorCode; - -public enum KafkaWriterErrorCode - implements ErrorCode -{ - REQUIRED_VALUE("KafkaWriter-00", "You are missing a required parameter value."), - ILLEGAL_VALUE("KafkaWriter-01", "You fill in the parameter values are not legitimate."), - GET_KAFKA_CONNECTION_ERROR("KafkaWriter-02", "Error getting KAFKA connection."), - GET_KAFKA_TABLE_ERROR("KafkaWriter-03", "Error getting KAFKA table."), - CLOSE_KAFKA_CONNECTION_ERROR("KafkaWriter-04", "Error closing KAFKA connection."), - CLOSE_KAFKA_SESSION_ERROR("KafkaWriter-06", "Error closing KAFKA table connection."), - PUT_KAFKA_ERROR("KafkaWriter-07", "IO exception occurred when writing to KAFKA."), - DELETE_KAFKA_ERROR("KafkaWriter-08", "An exception occurred while delete KAFKA table."), - CREATE_KAFKA_TABLE_ERROR("KafkaWriter-09", "Error creating KAFKA table."), - PARAMETER_NUM_ERROR("KafkaWriter-10", "The number of parameters does not match."), - TABLE_NOT_EXISTS("KafkaWriter-11", "The table you specified does not exists yet"), - COLUMN_NOT_EXISTS("KafkaWriter-12", "the column doest not exists"), - NOT_MATCHED_COLUMNS("KafkaWriter-13", "the number of columns does not match the record"); - - private final String code; - private final String description; - - KafkaWriterErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return code; - } - - @Override - public String getDescription() - { - return description; - } -} diff --git a/plugin/writer/kuduwriter/src/main/java/com/wgzhao/addax/plugin/writer/kuduwriter/KuduHelper.java b/plugin/writer/kuduwriter/src/main/java/com/wgzhao/addax/plugin/writer/kuduwriter/KuduHelper.java index ef9869998..4baf2ab2e 100644 --- a/plugin/writer/kuduwriter/src/main/java/com/wgzhao/addax/plugin/writer/kuduwriter/KuduHelper.java +++ b/plugin/writer/kuduwriter/src/main/java/com/wgzhao/addax/plugin/writer/kuduwriter/KuduHelper.java @@ -32,6 +32,9 @@ import java.util.ArrayList; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.CONNECT_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; + public class KuduHelper { @@ -52,7 +55,7 @@ public KuduHelper(String masterAddress, long timeout) .build(); } catch (Exception e) { - throw AddaxException.asAddaxException(KuduWriterErrorCode.GET_KUDU_CONNECTION_ERROR, e); + throw AddaxException.asAddaxException(CONNECT_ERROR, e); } } @@ -66,8 +69,8 @@ public KuduTable getKuduTable(String tableName) kuduTable = kuduClient.openTable(tableName); return kuduTable; } - catch (Exception e) { - throw AddaxException.asAddaxException(KuduWriterErrorCode.GET_KUDU_TABLE_ERROR, e); + catch (KuduException e) { + throw AddaxException.asAddaxException(RUNTIME_ERROR, e); } } } @@ -80,8 +83,8 @@ public boolean isTableExists(String tableName) try { return kuduClient.tableExists(tableName); } - catch (Exception e) { - throw AddaxException.asAddaxException(KuduWriterErrorCode.GET_KUDU_CONNECTION_ERROR, e); + catch (KuduException e) { + throw AddaxException.asAddaxException(RUNTIME_ERROR, e); } } diff --git a/plugin/writer/kuduwriter/src/main/java/com/wgzhao/addax/plugin/writer/kuduwriter/KuduWriter.java b/plugin/writer/kuduwriter/src/main/java/com/wgzhao/addax/plugin/writer/kuduwriter/KuduWriter.java index a5f42311e..cc05346cf 100644 --- a/plugin/writer/kuduwriter/src/main/java/com/wgzhao/addax/plugin/writer/kuduwriter/KuduWriter.java +++ b/plugin/writer/kuduwriter/src/main/java/com/wgzhao/addax/plugin/writer/kuduwriter/KuduWriter.java @@ -30,6 +30,9 @@ import java.util.ArrayList; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; + public class KuduWriter extends Writer { @@ -53,8 +56,8 @@ public void init() private void validateParameter() { - String tableName = config.getNecessaryValue(KuduKey.TABLE, KuduWriterErrorCode.REQUIRED_VALUE); - String masterAddress = config.getNecessaryValue(KuduKey.KUDU_MASTER_ADDRESSES, KuduWriterErrorCode.REQUIRED_VALUE); + String tableName = config.getNecessaryValue(KuduKey.TABLE, REQUIRED_VALUE); + String masterAddress = config.getNecessaryValue(KuduKey.KUDU_MASTER_ADDRESSES, REQUIRED_VALUE); long timeout = config.getInt(KuduKey.KUDU_TIMEOUT, DEFAULT_TIME_OUT) * 1000L; // write back default value with ms unit this.config.set(KuduKey.KUDU_TIMEOUT, timeout); @@ -63,14 +66,14 @@ private void validateParameter() KuduHelper kuduHelper = new KuduHelper(masterAddress, timeout); // check table exists or not if (!kuduHelper.isTableExists(tableName)) { - throw AddaxException.asAddaxException(KuduWriterErrorCode.TABLE_NOT_EXISTS, "table '" + tableName + "' does not exists"); + throw AddaxException.asAddaxException(CONFIG_ERROR, "table '" + tableName + "' does not exists"); } // column check List columns = this.config.getList(KuduKey.COLUMN, String.class); if (null == columns || columns.isEmpty()) { throw AddaxException.asAddaxException( - KuduWriterErrorCode.REQUIRED_VALUE, "the configuration 'column' must be specified" + REQUIRED_VALUE, "the configuration 'column' must be specified" ); } @@ -85,7 +88,7 @@ private void validateParameter() final Schema schema = kuduHelper.getSchema(tableName); for (String column : columns) { if (schema.getColumn(column) == null) { - throw AddaxException.asAddaxException(KuduWriterErrorCode.COLUMN_NOT_EXISTS, "column '" + column + "' does not exists"); + throw AddaxException.asAddaxException(CONFIG_ERROR, "column '" + column + "' does not exists"); } } } diff --git a/plugin/writer/kuduwriter/src/main/java/com/wgzhao/addax/plugin/writer/kuduwriter/KuduWriterErrorCode.java b/plugin/writer/kuduwriter/src/main/java/com/wgzhao/addax/plugin/writer/kuduwriter/KuduWriterErrorCode.java deleted file mode 100644 index 1d7a33af7..000000000 --- a/plugin/writer/kuduwriter/src/main/java/com/wgzhao/addax/plugin/writer/kuduwriter/KuduWriterErrorCode.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.writer.kuduwriter; - -import com.wgzhao.addax.common.spi.ErrorCode; - -public enum KuduWriterErrorCode - implements ErrorCode -{ - REQUIRED_VALUE("Kuduwriter-00", "You are missing a required parameter value."), - ILLEGAL_VALUE("Kuduwriter-01", "You fill in the parameter values are not legitimate."), - GET_KUDU_CONNECTION_ERROR("Kuduwriter-02", "Error getting Kudu connection."), - GET_KUDU_TABLE_ERROR("Kuduwriter-03", "Error getting Kudu table."), - CLOSE_KUDU_CONNECTION_ERROR("Kuduwriter-04", "Error closing Kudu connection."), - CLOSE_KUDU_SESSION_ERROR("Kuduwriter-06", "Error closing Kudu table connection."), - PUT_KUDU_ERROR("Kuduwriter-07", "IO exception occurred when writing to Kudu."), - DELETE_KUDU_ERROR("Kuduwriter-08", "An exception occurred while delete Kudu table."), - CREATE_KUDU_TABLE_ERROR("Kuduwriter-09", "Error creating Kudu table."), - PARAMETER_NUM_ERROR("Kuduwriter-10", "The number of parameters does not match."), - TABLE_NOT_EXISTS("Kuduwriter-11", "The table you specified does not exists yet"), - COLUMN_NOT_EXISTS("Kuduwriter-12", "the column doest not exists"); - - private final String code; - private final String description; - - KuduWriterErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return code; - } - - @Override - public String getDescription() - { - return description; - } -} diff --git a/plugin/writer/kuduwriter/src/main/java/com/wgzhao/addax/plugin/writer/kuduwriter/KuduWriterTask.java b/plugin/writer/kuduwriter/src/main/java/com/wgzhao/addax/plugin/writer/kuduwriter/KuduWriterTask.java index 9d6a24b87..edb94cacd 100644 --- a/plugin/writer/kuduwriter/src/main/java/com/wgzhao/addax/plugin/writer/kuduwriter/KuduWriterTask.java +++ b/plugin/writer/kuduwriter/src/main/java/com/wgzhao/addax/plugin/writer/kuduwriter/KuduWriterTask.java @@ -29,6 +29,7 @@ import org.apache.kudu.Schema; import org.apache.kudu.Type; import org.apache.kudu.client.Insert; +import org.apache.kudu.client.KuduException; import org.apache.kudu.client.KuduSession; import org.apache.kudu.client.KuduTable; import org.apache.kudu.client.PartialRow; @@ -42,6 +43,9 @@ import java.util.List; import static com.wgzhao.addax.common.base.Constant.DEFAULT_BATCH_SIZE; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.NOT_SUPPORT_TYPE; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; public class KuduWriterTask extends Writer @@ -83,7 +87,7 @@ public void startWriter(RecordReceiver lineReceiver, TaskPluginCollector taskPlu // List columnNames = KuduHelper.getColumnNames(columns); while ((record = lineReceiver.getFromReader()) != null) { if (record.getColumnNumber() != columns.size()) { - throw AddaxException.asAddaxException(KuduWriterErrorCode.PARAMETER_NUM_ERROR, + throw AddaxException.asAddaxException(CONFIG_ERROR, "The number of record fields (" + record.getColumnNumber() + ") is different from the number of configuration fields (" + columns.size() + ")"); } @@ -140,7 +144,7 @@ public void startWriter(RecordReceiver lineReceiver, TaskPluginCollector taskPlu break; default: throw AddaxException.asAddaxException( - KuduWriterErrorCode.ILLEGAL_VALUE, "The data type " + type + " is unsupported" + NOT_SUPPORT_TYPE, "The data type " + type + " is unsupported" ); } } // end a row @@ -157,14 +161,14 @@ public void startWriter(RecordReceiver lineReceiver, TaskPluginCollector taskPlu session.flush(); } } - catch (Exception e) { + catch (KuduException e) { LOG.error("Failed to write a record: ", e); if (isSkipFail) { LOG.warn("Since you have configured 'skipFail' to be true, this record will be skipped."); taskPluginCollector.collectDirtyRecord(record, e.getMessage()); } else { - throw AddaxException.asAddaxException(KuduWriterErrorCode.PUT_KUDU_ERROR, e.getMessage()); + throw AddaxException.asAddaxException(RUNTIME_ERROR, e.getMessage()); } } } @@ -173,14 +177,14 @@ public void startWriter(RecordReceiver lineReceiver, TaskPluginCollector taskPlu // try to flush last upsert/insert session.flush(); } - catch (Exception e) { + catch (KuduException e) { LOG.error("Failed to write a record: ", e); if (isSkipFail) { LOG.warn("Since you have configured 'skipFail' to be true, this record will be skipped !"); taskPluginCollector.collectDirtyRecord(record, e.getMessage()); } else { - throw AddaxException.asAddaxException(KuduWriterErrorCode.PUT_KUDU_ERROR, e.getMessage()); + throw AddaxException.asAddaxException(RUNTIME_ERROR, e.getMessage()); } } } diff --git a/plugin/writer/mongodbwriter/src/main/java/com/wgzhao/addax/plugin/writer/mongodbwriter/MongoDBWriter.java b/plugin/writer/mongodbwriter/src/main/java/com/wgzhao/addax/plugin/writer/mongodbwriter/MongoDBWriter.java index c867a9eb9..ec25b8e34 100644 --- a/plugin/writer/mongodbwriter/src/main/java/com/wgzhao/addax/plugin/writer/mongodbwriter/MongoDBWriter.java +++ b/plugin/writer/mongodbwriter/src/main/java/com/wgzhao/addax/plugin/writer/mongodbwriter/MongoDBWriter.java @@ -59,6 +59,8 @@ import static com.wgzhao.addax.common.base.Key.PRE_SQL; import static com.wgzhao.addax.common.base.Key.USERNAME; import static com.wgzhao.addax.common.base.Key.WRITE_MODE; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; public class MongoDBWriter extends Writer @@ -91,18 +93,18 @@ public void prepare() { super.prepare(); // parameters check - originalConfig.getNecessaryValue(CONNECTION, MongoDBWriterErrorCode.REQUIRED_VALUE); + originalConfig.getNecessaryValue(CONNECTION, REQUIRED_VALUE); Configuration connConf = Configuration.from(originalConfig.getList(CONNECTION, Object.class).get(0).toString()); List address = connConf.getList(KeyConstant.MONGO_ADDRESS, Object.class); if (address == null || address.isEmpty()) { throw AddaxException.asAddaxException( - MongoDBWriterErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, "The configuration address is illegal, please check your json file:" ); } - String dbName = connConf.getNecessaryValue(DATABASE, MongoDBWriterErrorCode.REQUIRED_VALUE); - String collection = connConf.getNecessaryValue(KeyConstant.MONGO_COLLECTION_NAME, MongoDBWriterErrorCode.REQUIRED_VALUE); + String dbName = connConf.getNecessaryValue(DATABASE, REQUIRED_VALUE); + String collection = connConf.getNecessaryValue(KeyConstant.MONGO_COLLECTION_NAME, REQUIRED_VALUE); String username = connConf.getString(USERNAME); String password = connConf.getString(PASSWORD); if (password != null && password.startsWith(Constant.ENC_PASSWORD_PREFIX)) { @@ -210,7 +212,7 @@ public void init() this.writeMode = writerSliceConfig.getString(WRITE_MODE, "insert"); if (this.writeMode.startsWith("update")) { if (!this.writeMode.contains("(")) { - throw AddaxException.asAddaxException(MongoDBWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "When specifying the mode is update, you MUST both specify the field to be updated"); } this.updateKey = this.writeMode.split("\\(")[1].replace(")", ""); @@ -279,8 +281,8 @@ else if (record.getColumn(i) instanceof StringColumn) { else if (KeyConstant.isArrayType(type.toLowerCase())) { String splitter = columnMeta.getJSONObject(i).getString(KeyConstant.COLUMN_SPLITTER); if (isNullOrEmpty((splitter))) { - throw AddaxException.asAddaxException(MongoDBWriterErrorCode.ILLEGAL_VALUE, - MongoDBWriterErrorCode.ILLEGAL_VALUE.getDescription()); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, + ILLEGAL_VALUE.getDescription()); } String itemType = columnMeta.getJSONObject(i).getString(KeyConstant.ITEM_TYPE); if (itemType != null && !itemType.isEmpty()) { diff --git a/plugin/writer/mongodbwriter/src/main/java/com/wgzhao/addax/plugin/writer/mongodbwriter/MongoDBWriterErrorCode.java b/plugin/writer/mongodbwriter/src/main/java/com/wgzhao/addax/plugin/writer/mongodbwriter/MongoDBWriterErrorCode.java deleted file mode 100644 index a3913a54a..000000000 --- a/plugin/writer/mongodbwriter/src/main/java/com/wgzhao/addax/plugin/writer/mongodbwriter/MongoDBWriterErrorCode.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.writer.mongodbwriter; - -import com.wgzhao.addax.common.spi.ErrorCode; - -public enum MongoDBWriterErrorCode - implements ErrorCode -{ - - ILLEGAL_VALUE("ILLEGAL_PARAMETER_VALUE", "参数不合法"), - ILLEGAL_ADDRESS("ILLEGAL_ADDRESS", "不合法的Mongo地址"), -// JSON_CAST_EXCEPTION("JSON_CAST_EXCEPTION", "json类型转换异常"), - UNKNOWN_EXCEPTION("UNKNOWN_EXCEPTION", "未知异常"), - REQUIRED_VALUE("REQUIRED_VALUE_EXCEPTION","Missing required parameters"); - - private final String code; - - private final String description; - - MongoDBWriterErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return code; - } - - @Override - public String getDescription() - { - return description; - } -} diff --git a/plugin/writer/mongodbwriter/src/main/java/com/wgzhao/addax/plugin/writer/mongodbwriter/util/MongoUtil.java b/plugin/writer/mongodbwriter/src/main/java/com/wgzhao/addax/plugin/writer/mongodbwriter/util/MongoUtil.java index 83f99fd20..20c092d73 100644 --- a/plugin/writer/mongodbwriter/src/main/java/com/wgzhao/addax/plugin/writer/mongodbwriter/util/MongoUtil.java +++ b/plugin/writer/mongodbwriter/src/main/java/com/wgzhao/addax/plugin/writer/mongodbwriter/util/MongoUtil.java @@ -20,7 +20,6 @@ package com.wgzhao.addax.plugin.writer.mongodbwriter.util; import com.wgzhao.addax.common.exception.AddaxException; -import com.wgzhao.addax.plugin.writer.mongodbwriter.MongoDBWriterErrorCode; import com.mongodb.MongoClient; import com.mongodb.MongoClientOptions; import com.mongodb.MongoCredential; @@ -30,6 +29,9 @@ import java.util.ArrayList; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; + public class MongoUtil { @@ -37,19 +39,19 @@ public static MongoClient initMongoClient(List addressList) { if (addressList == null || addressList.isEmpty()) { - throw AddaxException.asAddaxException(MongoDBWriterErrorCode.ILLEGAL_VALUE, "不合法参数"); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "不合法参数"); } try { return new MongoClient(parseServerAddress(addressList)); } catch (UnknownHostException e) { - throw AddaxException.asAddaxException(MongoDBWriterErrorCode.ILLEGAL_ADDRESS, "不合法的地址"); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "不合法的地址"); } catch (NumberFormatException e) { - throw AddaxException.asAddaxException(MongoDBWriterErrorCode.ILLEGAL_VALUE, "不合法参数"); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "不合法参数"); } catch (Exception e) { - throw AddaxException.asAddaxException(MongoDBWriterErrorCode.UNKNOWN_EXCEPTION, "未知异常"); + throw AddaxException.asAddaxException(RUNTIME_ERROR, "未知异常"); } } @@ -57,20 +59,20 @@ public static MongoClient initCredentialMongoClient(List addressList, St { if (!isHostPortPattern(addressList)) { - throw AddaxException.asAddaxException(MongoDBWriterErrorCode.ILLEGAL_VALUE, "不合法参数"); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "不合法参数"); } try { MongoCredential credential = MongoCredential.createCredential(userName, database, password.toCharArray()); return new MongoClient(parseServerAddress(addressList), credential, new MongoClientOptions.Builder().build()); } catch (UnknownHostException e) { - throw AddaxException.asAddaxException(MongoDBWriterErrorCode.ILLEGAL_ADDRESS, "不合法的地址"); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "不合法的地址"); } catch (NumberFormatException e) { - throw AddaxException.asAddaxException(MongoDBWriterErrorCode.ILLEGAL_VALUE, "不合法参数"); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "不合法参数"); } catch (Exception e) { - throw AddaxException.asAddaxException(MongoDBWriterErrorCode.UNKNOWN_EXCEPTION, "未知异常"); + throw AddaxException.asAddaxException(RUNTIME_ERROR, "未知异常"); } } diff --git a/plugin/writer/oraclewriter/src/main/java/com/wgzhao/addax/plugin/writer/oraclewriter/OracleWriter.java b/plugin/writer/oraclewriter/src/main/java/com/wgzhao/addax/plugin/writer/oraclewriter/OracleWriter.java index 5a0202f5f..eb8ecdee6 100644 --- a/plugin/writer/oraclewriter/src/main/java/com/wgzhao/addax/plugin/writer/oraclewriter/OracleWriter.java +++ b/plugin/writer/oraclewriter/src/main/java/com/wgzhao/addax/plugin/writer/oraclewriter/OracleWriter.java @@ -25,7 +25,6 @@ import com.wgzhao.addax.common.plugin.RecordReceiver; import com.wgzhao.addax.common.spi.Writer; import com.wgzhao.addax.common.util.Configuration; -import com.wgzhao.addax.rdbms.util.DBUtilErrorCode; import com.wgzhao.addax.rdbms.util.DataBaseType; import com.wgzhao.addax.rdbms.writer.CommonRdbmsWriter; @@ -36,6 +35,8 @@ import java.sql.Types; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; + public class OracleWriter extends Writer { @@ -62,7 +63,7 @@ public void init() String writeMode = originalConfig.getString(Key.WRITE_MODE); if (null != writeMode) { if (!"insert".equalsIgnoreCase(writeMode) && !writeMode.startsWith("update")) { - throw AddaxException.asAddaxException(DBUtilErrorCode.CONF_ERROR, + throw AddaxException.asAddaxException(CONFIG_ERROR, String.format("The item writeMode your configured [%s] is unsupported, it only supports insert and update mode.", writeMode)); } } diff --git a/plugin/writer/postgresqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/postgresqlwriter/PostgresqlWriter.java b/plugin/writer/postgresqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/postgresqlwriter/PostgresqlWriter.java index 8b50bbae4..ae693efdd 100644 --- a/plugin/writer/postgresqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/postgresqlwriter/PostgresqlWriter.java +++ b/plugin/writer/postgresqlwriter/src/main/java/com/wgzhao/addax/plugin/writer/postgresqlwriter/PostgresqlWriter.java @@ -25,7 +25,6 @@ import com.wgzhao.addax.common.plugin.RecordReceiver; import com.wgzhao.addax.common.spi.Writer; import com.wgzhao.addax.common.util.Configuration; -import com.wgzhao.addax.rdbms.util.DBUtilErrorCode; import com.wgzhao.addax.rdbms.util.DataBaseType; import com.wgzhao.addax.rdbms.writer.CommonRdbmsWriter; @@ -34,6 +33,8 @@ import java.sql.Types; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; + public class PostgresqlWriter extends Writer { @@ -55,7 +56,7 @@ public void init() if (!"insert".equalsIgnoreCase(writeMode) && !writeMode.startsWith("update")) { throw AddaxException.asAddaxException( - DBUtilErrorCode.CONF_ERROR, + ILLEGAL_VALUE, String.format("写入模式(writeMode)配置错误. PostgreSQL 仅支持insert, update两种模式." + " %s 不支持", writeMode)); diff --git a/plugin/writer/rdbmswriter/src/main/java/com/wgzhao/addax/plugin/writer/rdbmswriter/RdbmsWriter.java b/plugin/writer/rdbmswriter/src/main/java/com/wgzhao/addax/plugin/writer/rdbmswriter/RdbmsWriter.java index 190d72277..48112ac60 100644 --- a/plugin/writer/rdbmswriter/src/main/java/com/wgzhao/addax/plugin/writer/rdbmswriter/RdbmsWriter.java +++ b/plugin/writer/rdbmswriter/src/main/java/com/wgzhao/addax/plugin/writer/rdbmswriter/RdbmsWriter.java @@ -25,7 +25,6 @@ import com.wgzhao.addax.common.spi.Writer; import com.wgzhao.addax.common.util.Configuration; import com.wgzhao.addax.rdbms.util.DBUtil; -import com.wgzhao.addax.rdbms.util.DBUtilErrorCode; import com.wgzhao.addax.rdbms.util.DataBaseType; import com.wgzhao.addax.rdbms.writer.CommonRdbmsWriter; import org.apache.commons.lang3.StringUtils; @@ -33,6 +32,8 @@ import java.util.List; import static com.wgzhao.addax.common.base.Key.JDBC_DRIVER; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; public class RdbmsWriter extends Writer @@ -53,13 +54,13 @@ public void init() // warn：not like mysql, only support insert mode, don't use String writeMode = this.originalConfig.getString(Key.WRITE_MODE); if (null != writeMode) { - throw AddaxException.asAddaxException(DBUtilErrorCode.CONF_ERROR, + throw AddaxException.asAddaxException(CONFIG_ERROR, String.format("写入模式(writeMode)配置有误. 因为不支持配置参数项 writeMode: %s, 仅使用insert sql 插入数据. 请检查您的配置并作出修改.", writeMode)); } String jdbcDriver = this.originalConfig.getString(JDBC_DRIVER, null); if (jdbcDriver == null || StringUtils.isBlank(jdbcDriver)) { - throw AddaxException.asAddaxException(DBUtilErrorCode.REQUIRED_VALUE, "config 'driver' is required and must not be empty"); + throw AddaxException.asAddaxException(REQUIRED_VALUE, "config 'driver' is required and must not be empty"); } // use special jdbc driver class DATABASE_TYPE.setDriverClassName(jdbcDriver); diff --git a/plugin/writer/s3writer/src/main/java/com/wgzhao/addax/plugin/writer/s3writer/S3Util.java b/plugin/writer/s3writer/src/main/java/com/wgzhao/addax/plugin/writer/s3writer/S3Util.java index 16102f6e1..6e7f5db88 100644 --- a/plugin/writer/s3writer/src/main/java/com/wgzhao/addax/plugin/writer/s3writer/S3Util.java +++ b/plugin/writer/s3writer/src/main/java/com/wgzhao/addax/plugin/writer/s3writer/S3Util.java @@ -9,6 +9,8 @@ import java.net.URI; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; + public class S3Util { public static S3Client initS3Client(Configuration conf) { @@ -34,7 +36,7 @@ public static S3Client initS3Client(String endpoint, Region region, String acces .build(); } catch (IllegalArgumentException e) { throw AddaxException.asAddaxException( - S3WriterErrorCode.ILLEGAL_VALUE, e.getMessage()); + ILLEGAL_VALUE, e.getMessage()); } } } diff --git a/plugin/writer/s3writer/src/main/java/com/wgzhao/addax/plugin/writer/s3writer/S3Writer.java b/plugin/writer/s3writer/src/main/java/com/wgzhao/addax/plugin/writer/s3writer/S3Writer.java index 1514c8523..c707228c6 100644 --- a/plugin/writer/s3writer/src/main/java/com/wgzhao/addax/plugin/writer/s3writer/S3Writer.java +++ b/plugin/writer/s3writer/src/main/java/com/wgzhao/addax/plugin/writer/s3writer/S3Writer.java @@ -20,8 +20,6 @@ import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse; import software.amazon.awssdk.services.s3.model.Delete; import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; -import software.amazon.awssdk.services.s3.model.ListObjectsRequest; -import software.amazon.awssdk.services.s3.model.ListObjectsResponse; import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; import software.amazon.awssdk.services.s3.model.ObjectIdentifier; @@ -39,7 +37,11 @@ import java.util.Set; import java.util.StringJoiner; import java.util.UUID; -import java.util.function.Consumer; + +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.IO_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; public class S3Writer extends Writer @@ -70,11 +72,11 @@ public void destroy() private void validateParameter() { - this.writerSliceConfig.getNecessaryValue(S3Key.REGION, S3WriterErrorCode.REQUIRED_VALUE); - this.writerSliceConfig.getNecessaryValue(S3Key.ACCESS_ID, S3WriterErrorCode.REQUIRED_VALUE); - this.writerSliceConfig.getNecessaryValue(S3Key.ACCESS_KEY, S3WriterErrorCode.REQUIRED_VALUE); - this.writerSliceConfig.getNecessaryValue(S3Key.BUCKET, S3WriterErrorCode.REQUIRED_VALUE); - this.writerSliceConfig.getNecessaryValue(S3Key.OBJECT, S3WriterErrorCode.REQUIRED_VALUE); + this.writerSliceConfig.getNecessaryValue(S3Key.REGION, REQUIRED_VALUE); + this.writerSliceConfig.getNecessaryValue(S3Key.ACCESS_ID, REQUIRED_VALUE); + this.writerSliceConfig.getNecessaryValue(S3Key.ACCESS_KEY, REQUIRED_VALUE); + this.writerSliceConfig.getNecessaryValue(S3Key.BUCKET, REQUIRED_VALUE); + this.writerSliceConfig.getNecessaryValue(S3Key.OBJECT, REQUIRED_VALUE); StorageWriterUtil.validateParameter(this.writerSliceConfig); } @@ -97,7 +99,7 @@ else if ("nonConflict".equals(writeMode)) { List objs = listObjects(bucket, object); if (!objs.isEmpty()) { LOG.error("There have {} objects starts with {} in bucket {} ", objs.size(), object, bucket); - throw AddaxException.asAddaxException(S3WriterErrorCode.ILLEGAL_VALUE, "Object conflict"); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "Object conflict"); } } } @@ -199,7 +201,7 @@ private void deleteBucketObjects(String bucket, String objectName) s3Client.deleteObjects(dor); } catch (S3Exception e) { - throw AddaxException.asAddaxException(S3WriterErrorCode.S3_COMM_ERROR, e.getMessage()); + throw AddaxException.asAddaxException(RUNTIME_ERROR, e.getMessage()); } } } @@ -295,7 +297,7 @@ public void startWrite(RecordReceiver lineReceiver) } } catch (IOException e) { - throw AddaxException.asAddaxException(S3WriterErrorCode.S3_COMM_ERROR, e.getMessage()); + throw AddaxException.asAddaxException(IO_ERROR, e.getMessage()); } } // remain bytes diff --git a/plugin/writer/s3writer/src/main/java/com/wgzhao/addax/plugin/writer/s3writer/S3WriterErrorCode.java b/plugin/writer/s3writer/src/main/java/com/wgzhao/addax/plugin/writer/s3writer/S3WriterErrorCode.java deleted file mode 100644 index c08a954b2..000000000 --- a/plugin/writer/s3writer/src/main/java/com/wgzhao/addax/plugin/writer/s3writer/S3WriterErrorCode.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.wgzhao.addax.plugin.writer.s3writer; - -import com.wgzhao.addax.common.spi.ErrorCode; - -public enum S3WriterErrorCode implements ErrorCode -{ - CONFIG_INVALID_EXCEPTION("S3Writer-00", "Invalid configure."), - REQUIRED_VALUE("S3Writer-01", "Missing required parameters"), - ILLEGAL_VALUE("S3Writer-02", "Illegal value"), - WRITE_OBJECT_ERROR("S3Writer-03", "Failure to write object "), - S3_COMM_ERROR("S3Writer-05", "S3 object operation error"); - - private final String code; - private final String description; - - S3WriterErrorCode(String code, String description) { - this.code = code; - this.description = description; - } - - @Override - public String getCode() { - return this.code; - } - - @Override - public String getDescription() { - return this.description; - } - - @Override - public String toString() { - return String.format("Code:[%s], Description:[%s].", this.code, this.description); - } -} diff --git a/plugin/writer/sqlserverwriter/src/main/java/com/wgzhao/addax/plugin/writer/sqlserverwriter/SqlServerWriter.java b/plugin/writer/sqlserverwriter/src/main/java/com/wgzhao/addax/plugin/writer/sqlserverwriter/SqlServerWriter.java index d5e6bf128..b58f505f5 100644 --- a/plugin/writer/sqlserverwriter/src/main/java/com/wgzhao/addax/plugin/writer/sqlserverwriter/SqlServerWriter.java +++ b/plugin/writer/sqlserverwriter/src/main/java/com/wgzhao/addax/plugin/writer/sqlserverwriter/SqlServerWriter.java @@ -24,12 +24,13 @@ import com.wgzhao.addax.common.plugin.RecordReceiver; import com.wgzhao.addax.common.spi.Writer; import com.wgzhao.addax.common.util.Configuration; -import com.wgzhao.addax.rdbms.util.DBUtilErrorCode; import com.wgzhao.addax.rdbms.util.DataBaseType; import com.wgzhao.addax.rdbms.writer.CommonRdbmsWriter; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; + public class SqlServerWriter extends Writer { @@ -49,7 +50,7 @@ public void init() String writeMode = this.originalConfig.getString(Key.WRITE_MODE); if (null != writeMode) { if (!"insert".equalsIgnoreCase(writeMode) && !writeMode.startsWith("update")) { - throw AddaxException.asAddaxException(DBUtilErrorCode.CONF_ERROR, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, String.format("The writeMode '%s' is illegal, Only insert, update are supported.", writeMode)); } } diff --git a/plugin/writer/starrockswriter/src/main/java/com/wgzhao/addax/plugin/writer/starrockswriter/StarRocksWriter.java b/plugin/writer/starrockswriter/src/main/java/com/wgzhao/addax/plugin/writer/starrockswriter/StarRocksWriter.java index 885a6cc75..6531c28f9 100755 --- a/plugin/writer/starrockswriter/src/main/java/com/wgzhao/addax/plugin/writer/starrockswriter/StarRocksWriter.java +++ b/plugin/writer/starrockswriter/src/main/java/com/wgzhao/addax/plugin/writer/starrockswriter/StarRocksWriter.java @@ -10,7 +10,6 @@ import com.wgzhao.addax.plugin.writer.starrockswriter.row.StarRocksSerializerFactory; import com.wgzhao.addax.plugin.writer.starrockswriter.util.StarRocksWriterUtil; import com.wgzhao.addax.rdbms.util.DBUtil; -import com.wgzhao.addax.rdbms.util.DBUtilErrorCode; import com.wgzhao.addax.rdbms.util.DataBaseType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -19,6 +18,9 @@ import java.util.ArrayList; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.EXECUTE_FAIL; + public class StarRocksWriter extends Writer { @@ -126,7 +128,7 @@ public void startWrite(RecordReceiver recordReceiver) if (record.getColumnNumber() != options.getColumns().size()) { throw AddaxException .asAddaxException( - DBUtilErrorCode.CONF_ERROR, + CONFIG_ERROR, String.format( "列配置信息有错误. 因为您配置的任务中，源头读取字段数:%s 与 目的表要写入的字段数:%s 不相等. 请检查您的配置并作出修改.", record.getColumnNumber(), @@ -136,7 +138,7 @@ public void startWrite(RecordReceiver recordReceiver) } } catch (Exception e) { - throw AddaxException.asAddaxException(DBUtilErrorCode.WRITE_DATA_ERROR, e); + throw AddaxException.asAddaxException(EXECUTE_FAIL, e); } } @@ -147,7 +149,7 @@ public void post() writerManager.close(); } catch (Exception e) { - throw AddaxException.asAddaxException(DBUtilErrorCode.WRITE_DATA_ERROR, e); + throw AddaxException.asAddaxException(EXECUTE_FAIL, e); } } diff --git a/plugin/writer/starrockswriter/src/main/java/com/wgzhao/addax/plugin/writer/starrockswriter/StarRocksWriterOptions.java b/plugin/writer/starrockswriter/src/main/java/com/wgzhao/addax/plugin/writer/starrockswriter/StarRocksWriterOptions.java index 891ce1e27..6c2d9c99c 100644 --- a/plugin/writer/starrockswriter/src/main/java/com/wgzhao/addax/plugin/writer/starrockswriter/StarRocksWriterOptions.java +++ b/plugin/writer/starrockswriter/src/main/java/com/wgzhao/addax/plugin/writer/starrockswriter/StarRocksWriterOptions.java @@ -2,13 +2,15 @@ import com.wgzhao.addax.common.exception.AddaxException; import com.wgzhao.addax.common.util.Configuration; -import com.wgzhao.addax.rdbms.util.DBUtilErrorCode; import java.io.Serializable; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; + public class StarRocksWriterOptions implements Serializable { @@ -40,6 +42,7 @@ public class StarRocksWriterOptions private List infoCchemaColumns; private final List userSetColumns; private boolean isWildcardColumn; + public StarRocksWriterOptions(Configuration options) { this.options = options; @@ -165,7 +168,7 @@ private void validateStreamLoadUrl() List urlList = getLoadUrlList(); for (String host : urlList) { if (host.split(":").length < 2) { - throw AddaxException.asAddaxException(DBUtilErrorCode.CONF_ERROR, + throw AddaxException.asAddaxException(CONFIG_ERROR, "loadUrl的格式不正确，请输入 `fe_ip:fe_http_ip;fe_ip:fe_http_ip`。"); } } @@ -181,7 +184,7 @@ private void validateRequired() KEY_LOAD_URL }; for (String optionKey : requiredOptionKeys) { - options.getNecessaryValue(optionKey, DBUtilErrorCode.REQUIRED_VALUE); + options.getNecessaryValue(optionKey, REQUIRED_VALUE); } } diff --git a/plugin/writer/streamwriter/src/main/java/com/wgzhao/addax/plugin/writer/streamwriter/StreamWriter.java b/plugin/writer/streamwriter/src/main/java/com/wgzhao/addax/plugin/writer/streamwriter/StreamWriter.java index b1413d692..7aa73a41f 100644 --- a/plugin/writer/streamwriter/src/main/java/com/wgzhao/addax/plugin/writer/streamwriter/StreamWriter.java +++ b/plugin/writer/streamwriter/src/main/java/com/wgzhao/addax/plugin/writer/streamwriter/StreamWriter.java @@ -40,6 +40,11 @@ import java.util.ArrayList; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.IO_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.PERMISSION_ERROR; + public class StreamWriter extends Writer { @@ -91,7 +96,7 @@ private void validateParameter(String path, String fileName) if (dir.isFile()) { throw AddaxException .asAddaxException( - StreamWriterErrorCode.ILLEGAL_VALUE, + ILLEGAL_VALUE, String.format( "您配置的path: [%s] 不是一个合法的目录, 请您注意文件重名, 不合法目录名等情况.", path)); @@ -101,7 +106,7 @@ private void validateParameter(String path, String fileName) if (!createdOk) { throw AddaxException .asAddaxException( - StreamWriterErrorCode.CONFIG_INVALID_EXCEPTION, + CONFIG_ERROR, String.format("您指定的文件路径 : [%s] 创建失败.", path)); } @@ -115,14 +120,14 @@ private void validateParameter(String path, String fileName) } catch (IOException e) { throw AddaxException.asAddaxException( - StreamWriterErrorCode.RUNTIME_EXCEPTION, + IO_ERROR, String.format("删除文件失败 : [%s] ", fileFullPath), e); } } } catch (SecurityException se) { throw AddaxException.asAddaxException( - StreamWriterErrorCode.SECURITY_NOT_ENOUGH, + PERMISSION_ERROR, String.format("您没有权限创建文件路径 : [%s] ", path), se); } } @@ -189,10 +194,10 @@ public void init() this.sleepTime = writerSliceConfig.getLong(StreamKey.SLEEP_TIME, 0); this.nullFormat = writerSliceConfig.getString(StreamKey.NULL_FORMAT, NULL_FLAG); if (recordNumBeforeSleep < 0) { - throw AddaxException.asAddaxException(StreamWriterErrorCode.CONFIG_INVALID_EXCEPTION, "recordNumber 不能为负值"); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "recordNumber 不能为负值"); } if (sleepTime < 0) { - throw AddaxException.asAddaxException(StreamWriterErrorCode.CONFIG_INVALID_EXCEPTION, "sleep 不能为负值"); + throw AddaxException.asAddaxException(ILLEGAL_VALUE, "sleep 不能为负值"); } } @@ -219,8 +224,8 @@ else if (this.print) { } writer.flush(); } - catch (Exception e) { - throw AddaxException.asAddaxException(StreamWriterErrorCode.RUNTIME_EXCEPTION, e); + catch (IOException e) { + throw AddaxException.asAddaxException(IO_ERROR, e); } } } @@ -255,8 +260,8 @@ private void writeToFile(RecordReceiver recordReceiver, String path, String file } writer.flush(); } - catch (Exception e) { - throw AddaxException.asAddaxException(StreamWriterErrorCode.RUNTIME_EXCEPTION, e); + catch (IOException | InterruptedException e) { + throw AddaxException.asAddaxException(IO_ERROR, e); } } diff --git a/plugin/writer/streamwriter/src/main/java/com/wgzhao/addax/plugin/writer/streamwriter/StreamWriterErrorCode.java b/plugin/writer/streamwriter/src/main/java/com/wgzhao/addax/plugin/writer/streamwriter/StreamWriterErrorCode.java deleted file mode 100644 index baee9adc0..000000000 --- a/plugin/writer/streamwriter/src/main/java/com/wgzhao/addax/plugin/writer/streamwriter/StreamWriterErrorCode.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.writer.streamwriter; - -import com.wgzhao.addax.common.spi.ErrorCode; - -public enum StreamWriterErrorCode - implements ErrorCode -{ - RUNTIME_EXCEPTION("StreamWriter-00", "运行时异常"), - ILLEGAL_VALUE("StreamWriter-01", "您填写的参数值不合法."), - CONFIG_INVALID_EXCEPTION("StreamWriter-02", "您的参数配置错误."), - SECURITY_NOT_ENOUGH("TxtFileWriter-03", "您缺少权限执行相应的文件写入操作."); - - private final String code; - private final String description; - - StreamWriterErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s]. ", this.code, - this.description); - } -} diff --git a/plugin/writer/sybasewriter/src/main/java/com/wgzhao/addax/plugin/writer/sybasewriter/SybaseWriter.java b/plugin/writer/sybasewriter/src/main/java/com/wgzhao/addax/plugin/writer/sybasewriter/SybaseWriter.java index 63c53ecc4..eac961cbf 100644 --- a/plugin/writer/sybasewriter/src/main/java/com/wgzhao/addax/plugin/writer/sybasewriter/SybaseWriter.java +++ b/plugin/writer/sybasewriter/src/main/java/com/wgzhao/addax/plugin/writer/sybasewriter/SybaseWriter.java @@ -24,12 +24,13 @@ import com.wgzhao.addax.common.plugin.RecordReceiver; import com.wgzhao.addax.common.spi.Writer; import com.wgzhao.addax.common.util.Configuration; -import com.wgzhao.addax.rdbms.util.DBUtilErrorCode; import com.wgzhao.addax.rdbms.util.DataBaseType; import com.wgzhao.addax.rdbms.writer.CommonRdbmsWriter; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; + public class SybaseWriter extends Writer { @@ -49,7 +50,7 @@ public void init() String writeMode = this.originalConfig.getString(Key.WRITE_MODE); if (null != writeMode) { if (!"insert".equalsIgnoreCase(writeMode) && !writeMode.startsWith("update")) { - throw AddaxException.asAddaxException(DBUtilErrorCode.CONF_ERROR, + throw AddaxException.asAddaxException(CONFIG_ERROR, String.format("The writeMode '%s' is illegal, Only insert, update are supported.", writeMode)); } } diff --git a/plugin/writer/tdenginewriter/src/main/java/com/wgzhao/addax/plugin/writer/tdenginewriter/DefaultDataHandler.java b/plugin/writer/tdenginewriter/src/main/java/com/wgzhao/addax/plugin/writer/tdenginewriter/DefaultDataHandler.java index 50324778f..2d0a70962 100644 --- a/plugin/writer/tdenginewriter/src/main/java/com/wgzhao/addax/plugin/writer/tdenginewriter/DefaultDataHandler.java +++ b/plugin/writer/tdenginewriter/src/main/java/com/wgzhao/addax/plugin/writer/tdenginewriter/DefaultDataHandler.java @@ -41,6 +41,9 @@ import java.util.Map; import java.util.stream.Collectors; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.EXECUTE_FAIL; + public class DefaultDataHandler implements DataHandler { @@ -131,7 +134,7 @@ public int handle(RecordReceiver lineReceiver, TaskPluginCollector collector) } } catch (SQLException e) { - throw AddaxException.asAddaxException(TDengineWriterErrorCode.RUNTIME_EXCEPTION, e.getMessage()); + throw AddaxException.asAddaxException(EXECUTE_FAIL, e.getMessage()); } if (affectedRows != count) { @@ -234,7 +237,7 @@ private int executeUpdate(Connection conn, String sql) count = stmt.executeUpdate(sql); } catch (SQLException e) { - throw AddaxException.asAddaxException(TDengineWriterErrorCode.RUNTIME_EXCEPTION, e.getMessage()); + throw AddaxException.asAddaxException(EXECUTE_FAIL, e.getMessage()); } return count; } @@ -347,7 +350,7 @@ else if (column.getType() == Column.Type.STRING) { try { writer = new SchemalessWriter(conn); } catch (SQLException e) { - throw AddaxException.asAddaxException(TDengineWriterErrorCode.RUNTIME_EXCEPTION, e.getMessage()); + throw AddaxException.asAddaxException(EXECUTE_FAIL, e.getMessage()); } SchemalessTimestampType timestampType; switch (timestampPrecision) { @@ -367,7 +370,7 @@ else if (column.getType() == Column.Type.STRING) { writer.write(lines, SchemalessProtocolType.LINE, timestampType); } catch (SQLException e) { - throw AddaxException.asAddaxException(TDengineWriterErrorCode.RUNTIME_EXCEPTION, e.getMessage()); + throw AddaxException.asAddaxException(EXECUTE_FAIL, e.getMessage()); } LOG.warn("schemalessWriter does not return affected rows!"); @@ -566,7 +569,7 @@ private int indexOf(String colName) return i; } } - throw AddaxException.asAddaxException(TDengineWriterErrorCode.RUNTIME_EXCEPTION, + throw AddaxException.asAddaxException(CONFIG_ERROR, "cannot find col: " + colName + " in columns: " + columns); } } diff --git a/plugin/writer/tdenginewriter/src/main/java/com/wgzhao/addax/plugin/writer/tdenginewriter/OpentsdbDataHandler.java b/plugin/writer/tdenginewriter/src/main/java/com/wgzhao/addax/plugin/writer/tdenginewriter/OpentsdbDataHandler.java index 19036bb74..146b5171f 100644 --- a/plugin/writer/tdenginewriter/src/main/java/com/wgzhao/addax/plugin/writer/tdenginewriter/OpentsdbDataHandler.java +++ b/plugin/writer/tdenginewriter/src/main/java/com/wgzhao/addax/plugin/writer/tdenginewriter/OpentsdbDataHandler.java @@ -30,6 +30,10 @@ import java.sql.Connection; import java.sql.DriverManager; +import java.sql.SQLException; + +import static com.wgzhao.addax.common.spi.ErrorCode.EXECUTE_FAIL; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; public class OpentsdbDataHandler implements DataHandler @@ -60,8 +64,8 @@ public int handle(RecordReceiver lineReceiver, TaskPluginCollector collector) writer = new SchemalessWriter(conn); count = write(lineReceiver, batchSize); } - catch (Exception e) { - throw AddaxException.asAddaxException(TDengineWriterErrorCode.RUNTIME_EXCEPTION, e); + catch (SQLException e) { + throw AddaxException.asAddaxException(EXECUTE_FAIL, e); } return count; @@ -103,7 +107,7 @@ else if (recordIndex % batchSize == 0) { } } catch (Exception e) { - throw AddaxException.asAddaxException(TDengineWriterErrorCode.RUNTIME_EXCEPTION, e); + throw AddaxException.asAddaxException(RUNTIME_ERROR, e); } return recordIndex - 1; } diff --git a/plugin/writer/tdenginewriter/src/main/java/com/wgzhao/addax/plugin/writer/tdenginewriter/SchemaManager.java b/plugin/writer/tdenginewriter/src/main/java/com/wgzhao/addax/plugin/writer/tdenginewriter/SchemaManager.java index cd15ab170..b6271f495 100644 --- a/plugin/writer/tdenginewriter/src/main/java/com/wgzhao/addax/plugin/writer/tdenginewriter/SchemaManager.java +++ b/plugin/writer/tdenginewriter/src/main/java/com/wgzhao/addax/plugin/writer/tdenginewriter/SchemaManager.java @@ -29,6 +29,9 @@ import java.util.List; import java.util.Map; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.EXECUTE_FAIL; + public class SchemaManager { private static final Logger LOG = LoggerFactory.getLogger(SchemaManager.class); @@ -55,7 +58,7 @@ public TimestampPrecision loadDatabasePrecision() dbname = rs.getString("database()"); } if (dbname == null) { - throw AddaxException.asAddaxException(TDengineWriterErrorCode.RUNTIME_EXCEPTION, + throw AddaxException.asAddaxException(CONFIG_ERROR, "Database not specified or available"); } @@ -80,7 +83,7 @@ public TimestampPrecision loadDatabasePrecision() } } catch (SQLException e) { - throw AddaxException.asAddaxException(TDengineWriterErrorCode.RUNTIME_EXCEPTION, e.getMessage()); + throw AddaxException.asAddaxException(EXECUTE_FAIL, e.getMessage()); } return this.precision; } @@ -111,12 +114,12 @@ public Map loadTableMeta(List tables) for (String tbname : tables) { if (!tableMetas.containsKey(tbname)) { - throw AddaxException.asAddaxException(TDengineWriterErrorCode.RUNTIME_EXCEPTION, "table metadata of " + tbname + " is empty!"); + throw AddaxException.asAddaxException(CONFIG_ERROR, "table metadata of " + tbname + " is empty!"); } } } catch (SQLException e) { - throw AddaxException.asAddaxException(TDengineWriterErrorCode.RUNTIME_EXCEPTION, e.getMessage()); + throw AddaxException.asAddaxException(EXECUTE_FAIL, e.getMessage()); } return tableMetas; } @@ -136,7 +139,7 @@ public Map> loadColumnMetas(List tables) } } catch (SQLException e) { - throw AddaxException.asAddaxException(TDengineWriterErrorCode.RUNTIME_EXCEPTION, e.getMessage()); + throw AddaxException.asAddaxException(EXECUTE_FAIL, e.getMessage()); } if (columnMetaList.isEmpty()) { diff --git a/plugin/writer/tdenginewriter/src/main/java/com/wgzhao/addax/plugin/writer/tdenginewriter/TDengineWriter.java b/plugin/writer/tdenginewriter/src/main/java/com/wgzhao/addax/plugin/writer/tdenginewriter/TDengineWriter.java index 25c9b3d99..11045f903 100644 --- a/plugin/writer/tdenginewriter/src/main/java/com/wgzhao/addax/plugin/writer/tdenginewriter/TDengineWriter.java +++ b/plugin/writer/tdenginewriter/src/main/java/com/wgzhao/addax/plugin/writer/tdenginewriter/TDengineWriter.java @@ -12,6 +12,8 @@ import java.util.ArrayList; import java.util.List; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; + public class TDengineWriter extends Writer { @@ -33,21 +35,21 @@ public void init() // check user String user = this.originalConfig.getString(Key.USERNAME); if (StringUtils.isBlank(user)) { - throw AddaxException.asAddaxException(TDengineWriterErrorCode.REQUIRED_VALUE, "The parameter [" + throw AddaxException.asAddaxException(REQUIRED_VALUE, "The parameter [" + Key.USERNAME + "] is not set."); } // check password String password = this.originalConfig.getString(Key.PASSWORD); if (StringUtils.isBlank(password)) { - throw AddaxException.asAddaxException(TDengineWriterErrorCode.REQUIRED_VALUE, "The parameter [" + throw AddaxException.asAddaxException(REQUIRED_VALUE, "The parameter [" + Key.PASSWORD + "] is not set."); } // check connection List connection = this.originalConfig.getList(Key.CONNECTION); if (connection == null || connection.isEmpty()) { - throw AddaxException.asAddaxException(TDengineWriterErrorCode.REQUIRED_VALUE, "The parameter [" + throw AddaxException.asAddaxException(REQUIRED_VALUE, "The parameter [" + Key.CONNECTION + "] is not set."); } if (connection.size() > 1) { @@ -56,7 +58,7 @@ public void init() Configuration conn = Configuration.from(connection.get(0).toString()); String jdbcUrl = conn.getString(Key.JDBC_URL); if (StringUtils.isBlank(jdbcUrl)) { - throw AddaxException.asAddaxException(TDengineWriterErrorCode.REQUIRED_VALUE, "The parameter [" + throw AddaxException.asAddaxException(REQUIRED_VALUE, "The parameter [" + Key.JDBC_URL + "] of connection is not set."); } diff --git a/plugin/writer/tdenginewriter/src/main/java/com/wgzhao/addax/plugin/writer/tdenginewriter/TDengineWriterErrorCode.java b/plugin/writer/tdenginewriter/src/main/java/com/wgzhao/addax/plugin/writer/tdenginewriter/TDengineWriterErrorCode.java deleted file mode 100644 index eaaa91ddd..000000000 --- a/plugin/writer/tdenginewriter/src/main/java/com/wgzhao/addax/plugin/writer/tdenginewriter/TDengineWriterErrorCode.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.wgzhao.addax.plugin.writer.tdenginewriter; - -import com.wgzhao.addax.common.spi.ErrorCode; - -public enum TDengineWriterErrorCode - implements ErrorCode -{ - - REQUIRED_VALUE("TDengineWriter-00", "missing required value"), - ILLEGAL_VALUE("TDengineWriter-01", "illegal value"), - RUNTIME_EXCEPTION("TDengineWriter-02", "runtime exception"), - TYPE_ERROR("TDengineWriter-03", "Unknow type error"); - - private final String code; - private final String description; - - TDengineWriterErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s]. ", this.code, this.description); - } -} \ No newline at end of file diff --git a/plugin/writer/txtfilewriter/src/main/java/com/wgzhao/addax/plugin/writer/txtfilewriter/TxtFileWriter.java b/plugin/writer/txtfilewriter/src/main/java/com/wgzhao/addax/plugin/writer/txtfilewriter/TxtFileWriter.java index 74269ca98..fe88866d6 100644 --- a/plugin/writer/txtfilewriter/src/main/java/com/wgzhao/addax/plugin/writer/txtfilewriter/TxtFileWriter.java +++ b/plugin/writer/txtfilewriter/src/main/java/com/wgzhao/addax/plugin/writer/txtfilewriter/TxtFileWriter.java @@ -52,20 +52,30 @@ import static com.wgzhao.addax.common.base.Key.FORMAT; import static com.wgzhao.addax.common.base.Key.PATH; import static com.wgzhao.addax.common.base.Key.WRITE_MODE; +import static com.wgzhao.addax.common.spi.ErrorCode.CONFIG_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.EXECUTE_FAIL; +import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.IO_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.PERMISSION_ERROR; +import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE; +import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR; /** * Created by haiwei.luo on 14-9-17. */ public class TxtFileWriter - extends Writer { + extends Writer +{ public static class Job - extends Writer.Job { + extends Writer.Job + { private static final Logger LOG = LoggerFactory.getLogger(Job.class); private Configuration writerSliceConfig = null; @Override - public void init() { + public void init() + { this.writerSliceConfig = this.getPluginJobConf(); this.validateParameter(); String dateFormatOld = this.writerSliceConfig.getString(FORMAT); @@ -79,34 +89,32 @@ public void init() { StorageWriterUtil.validateParameter(this.writerSliceConfig); } - private void validateParameter() { - this.writerSliceConfig.getNecessaryValue(FILE_NAME, TxtFileWriterErrorCode.REQUIRED_VALUE); + private void validateParameter() + { + this.writerSliceConfig.getNecessaryValue(FILE_NAME, REQUIRED_VALUE); - String path = this.writerSliceConfig.getNecessaryValue(PATH, TxtFileWriterErrorCode.REQUIRED_VALUE); + String path = this.writerSliceConfig.getNecessaryValue(PATH, REQUIRED_VALUE); + File dir; + dir = new File(path); - try { - // warn: 这里用户需要配一个目录 - File dir = new File(path); - if (dir.isFile()) { - throw AddaxException.asAddaxException( - TxtFileWriterErrorCode.ILLEGAL_VALUE, - String.format("The path [%s] is a file, not a directory.", path)); - } - if (!dir.exists()) { - boolean createdOk = dir.mkdirs(); - if (!createdOk) { - throw AddaxException.asAddaxException(TxtFileWriterErrorCode.CONFIG_INVALID_EXCEPTION, - String.format("Failed to create the special path [%s].", path)); + if (dir.isFile()) { + throw AddaxException.asAddaxException( + CONFIG_ERROR, + String.format("The path [%s] is a file, not a directory.", path)); + } + + if (!dir.exists()) { + if (!dir.mkdirs()) { + throw AddaxException.asAddaxException( + EXECUTE_FAIL, + String.format("Failed to create directory [%s].", path)); } - } - } catch (SecurityException se) { - throw AddaxException.asAddaxException(TxtFileWriterErrorCode.SECURITY_NOT_ENOUGH, - String.format("Permission denied to access path [%s].", path), se); } } @Override - public void prepare() { + public void prepare() + { String path = this.writerSliceConfig.getString(Key.PATH); String fileName = this.writerSliceConfig.getString(FILE_NAME); String writeMode = this.writerSliceConfig.getString(WRITE_MODE); @@ -125,24 +133,29 @@ public void prepare() { LOG.info("delete file [{}].", eachFile.getName()); FileUtils.forceDelete(eachFile); } - } catch (NullPointerException npe) { - throw AddaxException.asAddaxException(TxtFileWriterErrorCode.WRITE_FILE_ERROR, + } + catch (NullPointerException npe) { + throw AddaxException.asAddaxException(RUNTIME_ERROR, String.format("NullPointException occurred when clean history files under this path [%s].", path), npe); - } catch (IllegalArgumentException iae) { - throw AddaxException.asAddaxException(TxtFileWriterErrorCode.SECURITY_NOT_ENOUGH, + } + catch (IllegalArgumentException iae) { + throw AddaxException.asAddaxException(PERMISSION_ERROR, String.format("IllegalArgumentException occurred when clean history files under this path [%s].", path), iae); - } catch (IOException e) { - throw AddaxException.asAddaxException(TxtFileWriterErrorCode.WRITE_FILE_ERROR, + } + catch (IOException e) { + throw AddaxException.asAddaxException(IO_ERROR, String.format("IOException occurred when clean history files under this path [%s].", path), e); } - } else if ("append".equals(writeMode)) { + } + else if ("append".equals(writeMode)) { LOG.info("You specify [{}] as writeMode, so we will NOT clean history files starts with [{}] under this path [{}].", writeMode, fileName, path); - } else if ("nonConflict".equals(writeMode)) { + } + else if ("nonConflict".equals(writeMode)) { LOG.info("You specify [{}] as writeMode, begin to check the files in [{}].", writeMode, path); // warn: check two times about exists, mkdir if (dir.exists()) { if (!dir.canRead()) { - throw AddaxException.asAddaxException(TxtFileWriterErrorCode.SECURITY_NOT_ENOUGH, + throw AddaxException.asAddaxException(PERMISSION_ERROR, String.format("Permission denied to access path [%s].", path)); } // fileName is not null @@ -155,42 +168,48 @@ public void prepare() { allFiles.add(eachFile.getName()); } LOG.error("The file(s) [{}] already exists under path [{}] with nonConflict writeMode.", StringUtils.join(allFiles, ","), path); - throw AddaxException.asAddaxException(TxtFileWriterErrorCode.ILLEGAL_VALUE, + throw AddaxException.asAddaxException(ILLEGAL_VALUE, String.format("The directory [%s] contains files.", path)); } - } else { + } + else { boolean createdOk = dir.mkdirs(); if (!createdOk) { - throw AddaxException.asAddaxException(TxtFileWriterErrorCode.CONFIG_INVALID_EXCEPTION, + throw AddaxException.asAddaxException(EXECUTE_FAIL, String.format("Failed to create the file [%s].", path)); } } - } else { - throw AddaxException.asAddaxException(TxtFileWriterErrorCode.ILLEGAL_VALUE, + } + else { + throw AddaxException.asAddaxException(ILLEGAL_VALUE, String.format("ONLY support truncate, append and nonConflict as writeMode, but you give [%s].", writeMode)); } } @Override - public void post() { + public void post() + { // } @Override - public void destroy() { + public void destroy() + { // } @Override - public List split(int mandatoryNumber) { + public List split(int mandatoryNumber) + { Set allFiles; String path = null; try { path = this.writerSliceConfig.getString(Key.PATH); File dir = new File(path); allFiles = new HashSet<>(Arrays.asList(Objects.requireNonNull(dir.list()))); - } catch (SecurityException se) { - throw AddaxException.asAddaxException(TxtFileWriterErrorCode.SECURITY_NOT_ENOUGH, + } + catch (SecurityException se) { + throw AddaxException.asAddaxException(PERMISSION_ERROR, String.format("Permission denied to access path [%s].", path), se); } return StorageWriterUtil.split(writerSliceConfig, allFiles, mandatoryNumber); @@ -198,7 +217,8 @@ public List split(int mandatoryNumber) { } public static class Task - extends Writer.Task { + extends Writer.Task + { private static final Logger LOG = LoggerFactory.getLogger(Task.class); private Configuration writerSliceConfig; @@ -209,7 +229,8 @@ public static class Task private String suffix = ""; @Override - public void init() { + public void init() + { this.writerSliceConfig = this.getPluginJobConf(); this.path = this.writerSliceConfig.getString(PATH); this.fileName = this.writerSliceConfig.getString(FILE_NAME); @@ -217,7 +238,8 @@ public void init() { } @Override - public void prepare() { + public void prepare() + { String compress = this.writerSliceConfig.getString(COMPRESS); suffix = FileHelper.getCompressFileSuffix(compress); @@ -225,7 +247,8 @@ public void prepare() { } @Override - public void startWrite(RecordReceiver lineReceiver) { + public void startWrite(RecordReceiver lineReceiver) + { LOG.info("begin do write..."); String fileFullPath = StorageWriterUtil.buildFilePath(this.path, this.fileName, this.suffix); LOG.info("write to file : [{}]", fileFullPath); @@ -238,25 +261,30 @@ public void startWrite(RecordReceiver lineReceiver) { outputStream = new FileOutputStream(newFile); StorageWriterUtil.writeToStream(lineReceiver, outputStream, this.writerSliceConfig, this.fileName, this.getTaskPluginCollector()); - } catch (SecurityException se) { - throw AddaxException.asAddaxException(TxtFileWriterErrorCode.SECURITY_NOT_ENOUGH, + } + catch (SecurityException se) { + throw AddaxException.asAddaxException(PERMISSION_ERROR, String.format("Permission denied to create file [%s].", fileFullPath), se); - } catch (IOException ioe) { - throw AddaxException.asAddaxException(TxtFileWriterErrorCode.WRITE_FILE_IO_ERROR, + } + catch (IOException ioe) { + throw AddaxException.asAddaxException(IO_ERROR, String.format("Fail to create file [%s].", this.fileName), ioe); - } finally { + } + finally { IOUtils.closeQuietly(outputStream, null); } LOG.info("end do write"); } @Override - public void post() { + public void post() + { // } @Override - public void destroy() { + public void destroy() + { // } } diff --git a/plugin/writer/txtfilewriter/src/main/java/com/wgzhao/addax/plugin/writer/txtfilewriter/TxtFileWriterErrorCode.java b/plugin/writer/txtfilewriter/src/main/java/com/wgzhao/addax/plugin/writer/txtfilewriter/TxtFileWriterErrorCode.java deleted file mode 100644 index 48f8b9d8f..000000000 --- a/plugin/writer/txtfilewriter/src/main/java/com/wgzhao/addax/plugin/writer/txtfilewriter/TxtFileWriterErrorCode.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wgzhao.addax.plugin.writer.txtfilewriter; - -import com.wgzhao.addax.common.spi.ErrorCode; - -/** - * Created by haiwei.luo on 14-9-17. - */ -public enum TxtFileWriterErrorCode - implements ErrorCode -{ - - CONFIG_INVALID_EXCEPTION("TxtFileWriter-00", "error value."), - REQUIRED_VALUE("TxtFileWriter-01", "missing mandatory parameters"), - ILLEGAL_VALUE("TxtFileWriter-02", "illegal value"), - WRITE_FILE_ERROR("TxtFileWriter-03", "failed to write file."), - WRITE_FILE_IO_ERROR("TxtFileWriter-04", "IOException occurred when writing the file."), - SECURITY_NOT_ENOUGH("TxtFileWriter-05", "permission denied"), - PATH_NOT_VALID("TxtFileWriter-06", "invalid path"), - PAHT_NOT_DIR("TxtFileWriter-06", "the path is not directory."); - - private final String code; - private final String description; - - TxtFileWriterErrorCode(String code, String description) - { - this.code = code; - this.description = description; - } - - @Override - public String getCode() - { - return this.code; - } - - @Override - public String getDescription() - { - return this.description; - } - - @Override - public String toString() - { - return String.format("Code:[%s], Description:[%s].", this.code, - this.description); - } -}