From e8374e3b8170b02d5e870617dd73537e2f835f59 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Wed, 3 Jan 2024 22:50:33 -0800 Subject: [PATCH] ORC-1569: Remove HadoopShimsPre2_3, HadoopShimsPre2_6, HadoopShimsPre2_7 classes ### What changes were proposed in this pull request? This PR aims to remove `HadoopShimsPre2_3`, `HadoopShimsPre2_6`, `HadoopShimsPre2_7` classes and use `HadoopShimsCurrent` always. ### Why are the changes needed? 1. `HadoopShimsCurrent` supports not only Apache Hadoop 3+ but also Apache Hadoop 2.7+. 2. Apache ORC 2.0 uses Hadoop 3.x shaded client doesn't need old shims for Hadoop 2.6 and olders. - #1509 In addition, Apache Spark community also has been using the shaded Hadoop client since Spark 3.2 (SPARK-33212) and dropped `Hadoop 2` profile via [SPARK-42452](https://issues.apache.org/jira/browse/SPARK-42452) completely at Spark 3.5.0. ### How was this patch tested? Pass the CIs. Closes #1724 from dongjoon-hyun/ORC-1569. Authored-by: Dongjoon Hyun Signed-off-by: William Hyun --- .../apache/orc/impl/HadoopShimsFactory.java | 18 +---- .../apache/orc/impl/HadoopShimsPre2_3.java | 60 ----------------- .../apache/orc/impl/HadoopShimsPre2_6.java | 60 ----------------- .../apache/orc/impl/HadoopShimsPre2_7.java | 66 ------------------- 4 files changed, 2 insertions(+), 202 deletions(-) delete mode 100644 java/shims/src/java/org/apache/orc/impl/HadoopShimsPre2_3.java delete mode 100644 java/shims/src/java/org/apache/orc/impl/HadoopShimsPre2_6.java delete mode 100644 java/shims/src/java/org/apache/orc/impl/HadoopShimsPre2_7.java diff --git a/java/core/src/java/org/apache/orc/impl/HadoopShimsFactory.java b/java/core/src/java/org/apache/orc/impl/HadoopShimsFactory.java index 1cd98e0670..1366d3d4f2 100644 --- a/java/core/src/java/org/apache/orc/impl/HadoopShimsFactory.java +++ b/java/core/src/java/org/apache/orc/impl/HadoopShimsFactory.java @@ -32,11 +32,6 @@ public class HadoopShimsFactory { private static final String CURRENT_SHIM_NAME = "org.apache.orc.impl.HadoopShimsCurrent"; - private static final String PRE_2_6_SHIM_NAME = - "org.apache.orc.impl.HadoopShimsPre2_6"; - private static final String PRE_2_7_SHIM_NAME = - "org.apache.orc.impl.HadoopShimsPre2_7"; - private static HadoopShims SHIMS = null; private static HadoopShims createShimByName(String name) { @@ -57,18 +52,9 @@ public static synchronized HadoopShims get() { int major = Integer.parseInt(versionParts[0]); int minor = Integer.parseInt(versionParts[1]); if (major < 2 || (major == 2 && minor < 7)) { - LOG.warn("Hadoop " + VersionInfo.getVersion() + " support is deprecated. " + - "Please upgrade to Hadoop 2.7.3 or above."); - } - if (major < 2 || (major == 2 && minor < 3)) { - SHIMS = new HadoopShimsPre2_3(); - } else if (major == 2 && minor < 6) { - SHIMS = createShimByName(PRE_2_6_SHIM_NAME); - } else if (major == 2 && minor < 7) { - SHIMS = createShimByName(PRE_2_7_SHIM_NAME); - } else { - SHIMS = createShimByName(CURRENT_SHIM_NAME); + LOG.warn("Hadoop " + VersionInfo.getVersion() + " support is dropped."); } + SHIMS = createShimByName(CURRENT_SHIM_NAME); } return SHIMS; } diff --git a/java/shims/src/java/org/apache/orc/impl/HadoopShimsPre2_3.java b/java/shims/src/java/org/apache/orc/impl/HadoopShimsPre2_3.java deleted file mode 100644 index d53a377b25..0000000000 --- a/java/shims/src/java/org/apache/orc/impl/HadoopShimsPre2_3.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.orc.impl; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; - -import java.io.IOException; -import java.io.OutputStream; -import java.util.Random; - -/** - * Shims for versions of Hadoop up to and including 2.2.x - */ -public class HadoopShimsPre2_3 implements HadoopShims { - - HadoopShimsPre2_3() { - } - - @Override - public DirectDecompressor getDirectDecompressor( - DirectCompressionType codec) { - return null; - } - - @Override - public ZeroCopyReaderShim getZeroCopyReader(FSDataInputStream in, - ByteBufferPoolShim pool - ) throws IOException { - /* not supported */ - return null; - } - - @Override - public boolean endVariableLengthBlock(OutputStream output) { - return false; - } - - @Override - public KeyProvider getHadoopKeyProvider(Configuration conf, Random random) { - return new NullKeyProvider(); - } - -} diff --git a/java/shims/src/java/org/apache/orc/impl/HadoopShimsPre2_6.java b/java/shims/src/java/org/apache/orc/impl/HadoopShimsPre2_6.java deleted file mode 100644 index b59171d506..0000000000 --- a/java/shims/src/java/org/apache/orc/impl/HadoopShimsPre2_6.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.orc.impl; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; - -import java.io.IOException; -import java.io.OutputStream; -import java.util.Random; - -/** - * Shims for versions of Hadoop less than 2.6 - *

- * Adds support for: - *

    - *
  • Direct buffer decompression
    • - *
  • Zero copy
    • - *
- */ -public class HadoopShimsPre2_6 implements HadoopShims { - - @Override - public DirectDecompressor getDirectDecompressor(DirectCompressionType codec) { - return HadoopShimsCurrent.getDecompressor(codec); - } - - @Override - public ZeroCopyReaderShim getZeroCopyReader(FSDataInputStream in, - ByteBufferPoolShim pool - ) throws IOException { - return ZeroCopyShims.getZeroCopyReader(in, pool); - } - - @Override - public boolean endVariableLengthBlock(OutputStream output) { - return false; - } - - @Override - public KeyProvider getHadoopKeyProvider(Configuration conf, Random random) { - return new NullKeyProvider(); - } -} diff --git a/java/shims/src/java/org/apache/orc/impl/HadoopShimsPre2_7.java b/java/shims/src/java/org/apache/orc/impl/HadoopShimsPre2_7.java deleted file mode 100644 index f8ad0ee06f..0000000000 --- a/java/shims/src/java/org/apache/orc/impl/HadoopShimsPre2_7.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.orc.impl; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.OutputStream; -import java.util.Random; - -/** - * Shims for versions of Hadoop less than 2.7. - *

- * Adds support for: - *

    - *
  • Crypto
    • - *
- */ -public class HadoopShimsPre2_7 implements HadoopShims { - - private static final Logger LOG = - LoggerFactory.getLogger(HadoopShimsPre2_7.class); - - - @Override - public DirectDecompressor getDirectDecompressor(DirectCompressionType codec) { - return HadoopShimsCurrent.getDecompressor(codec); - } - - @Override - public ZeroCopyReaderShim getZeroCopyReader(FSDataInputStream in, - ByteBufferPoolShim pool - ) throws IOException { - return ZeroCopyShims.getZeroCopyReader(in, pool); - } - - @Override - public boolean endVariableLengthBlock(OutputStream output) { - return false; - } - - @Override - public KeyProvider getHadoopKeyProvider(Configuration conf, - Random random) throws IOException { - return HadoopShimsCurrent.createKeyProvider(conf, random); - } -}