Merge branch 'apache:master' into HIVE-28655-part2

Author: DanielZhu58

.github/workflows/docker-GA-images.yml

@@ -43,13 +43,30 @@ jobs:
     if: github.event_name == 'workflow_dispatch' || github.event_name == 'create' && github.event.ref_type == 'tag' && startsWith(github.event.ref, 'rel/')
     runs-on: ubuntu-latest
     steps:
+      - name: Free disk space
+        run: |
+          echo '===== Before ====='
+          df -h
+          sudo rm -rf /opt/hostedtoolcache/go
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          sudo rm -rf /usr/lib/google-cloud-sdk
+          sudo rm -rf /usr/local/julia*
+          sudo rm -rf /usr/local/.ghcup
+          sudo rm -rf /usr/share/swift
+          sudo rm -rf /usr/share/dotnet
+          sudo apt-get clean
+          echo '===== After ====='
+          df -h
+
       - name: Checkout
         uses: actions/checkout@v3
 
       - name: 'Set up JDK 21'
-        uses: actions/setup-java@v1
+        uses: actions/setup-java@v5
         with:
-          java-version: 21
+          java-version: '21'
+          distribution: temurin
+          cache: maven
 
       - name: Prepare environment variables for Workflow Dispatch
         if: github.event_name == 'workflow_dispatch'
@@ -75,7 +92,7 @@ jobs:
       - name: Build Hive project
         if: github.event_name == 'create' && github.event.ref_type == 'tag' && startsWith(github.event.ref, 'rel/')
         run: |
-          mvn clean package -DskipTests -Pdist
+          mvn --batch-mode clean package -DskipTests -Pdist
           ls ./packaging/target/
           mv ./packaging/target/apache-hive-*-bin.tar.gz ./packaging/src/docker/
           rm -rf ./packaging/target
@@ -123,3 +140,9 @@ jobs:
             HIVE_VERSION=${{ env.HIVE_VERSION }}
             HADOOP_VERSION=${{ env.HADOOP_VERSION }}
             BUILD_ENV=${{ env.BUILD_ENV }}
+
+      - name: Dump disk usage on failure
+        if: failure()
+        run: |
+          df -h
+          du -xh / --max-depth=3 2>/dev/null | sort -h | tail -50

beeline/src/java/org/apache/hive/beeline/BeeLine.java

@@ -71,7 +71,6 @@
 import java.util.ResourceBundle;
 import java.util.ServiceLoader;
 import java.util.Set;
-import java.util.SortedSet;
 import java.util.StringTokenizer;
 import java.util.TreeMap;
 import java.util.TreeSet;
@@ -310,12 +309,6 @@ public class BeeLine implements Closeable {
 
   private final Completer beeLineCommandCompleter = new BeeLineCommandCompleter(Arrays.asList(commandHandlers));
 
-  static final SortedSet<String> KNOWN_DRIVERS = new TreeSet<String>(Arrays.asList(
-      new String[] {
-          "org.apache.hive.jdbc.HiveDriver",
-          "org.apache.hadoop.hive.jdbc.HiveDriver",
-      }));
-
   static {
     try {
       Class.forName("org.jline.reader.LineReader");
@@ -2370,23 +2363,17 @@ private Driver findRegisteredDriver(String url) {
     return null;
   }
 
-  public Driver findLocalDriver(String url) throws Exception {
+  public Driver findLocalDriver(String url) throws SQLException {
     Objects.requireNonNull(url);
 
     Collection<Driver> currentDrivers = drivers == null ? Collections.emptyList() : drivers;
-    for (Driver d : currentDrivers) {
-      try {
-        String clazzName = d.getClass().getName();
-        Driver driver = (Driver) Class.forName(clazzName, true,
-          Thread.currentThread().getContextClassLoader()).newInstance();
-        if (driver.acceptsURL(url) && isSupportedLocalDriver(driver)) {
-          return driver;
-        }
-      } catch (SQLException e) {
-        throw e;
+    for (Driver driver : currentDrivers) {
+      // The 'driver' is already an instance from the ServiceLoader.
+      // We can use it directly without creating a new one via reflection.
+      if (driver.acceptsURL(url) && isSupportedLocalDriver(driver)) {
+        return driver;
       }
     }
-
     return null;
   }
 

beeline/src/java/org/apache/hive/beeline/Commands.java

@@ -44,6 +44,7 @@
 import java.sql.Statement;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.LinkedList;
@@ -340,48 +341,48 @@ public boolean reconnect(String line) {
     return true;
   }
 
-
   public boolean scan(String line) throws IOException {
-    TreeSet<String> names = new TreeSet<String>();
-
     if (beeLine.getDrivers() == null) {
       beeLine.setDrivers(beeLine.scanDrivers());
     }
 
-    beeLine.info(beeLine.loc("drivers-found-count", beeLine.getDrivers().size()));
+    // Use a TreeSet to get a unique, sorted list of drivers by class name.
+    Set<Driver> drivers =
+        new TreeSet<>(Comparator.comparing(d -> d.getClass().getName()));
+    drivers.addAll(beeLine.getDrivers());
 
-    // unique the list
-    for (Iterator<Driver> i = beeLine.getDrivers().iterator(); i.hasNext();) {
-      names.add(i.next().getClass().getName());
-    }
+    // Get count of the unique driver in classpath
+    beeLine.info(beeLine.loc("drivers-found-count", drivers.size()));
 
-    beeLine.output(beeLine.getColorBuffer()
-        .bold(beeLine.getColorBuffer().pad(beeLine.loc("compliant"), 10).getMono())
-        .bold(beeLine.getColorBuffer().pad(beeLine.loc("jdbc-version"), 8).getMono())
-        .bold(beeLine.getColorBuffer(beeLine.loc("driver-class")).getMono()));
+    beeLine.output(
+        beeLine
+            .getColorBuffer()
+            .bold(beeLine.getColorBuffer().pad(beeLine.loc("compliant"), 10).getMono())
+            .bold(beeLine.getColorBuffer().pad(beeLine.loc("jdbc-version"), 8).getMono())
+            .bold(beeLine.getColorBuffer(beeLine.loc("driver-class")).getMono()));
 
-    for (Iterator<String> i = names.iterator(); i.hasNext();) {
-      String name = i.next().toString();
+    for (Driver driver : drivers) {
+      String name = driver.getClass().getName();
       try {
-        Driver driver = (Driver) Class.forName(name).newInstance();
-        ColorBuffer msg = beeLine.getColorBuffer()
-            .pad(driver.jdbcCompliant() ? "yes" : "no", 10)
-            .pad(driver.getMajorVersion() + "."
-                + driver.getMinorVersion(), 8)
-            .append(name);
+        // Use the driver instance that ServiceLoader already created for us.
+        ColorBuffer msg =
+            beeLine
+                .getColorBuffer()
+                .pad(driver.jdbcCompliant() ? "yes" : "no", 10)
+                .pad(driver.getMajorVersion() + "." + driver.getMinorVersion(), 8)
+                .append(name);
         if (driver.jdbcCompliant()) {
           beeLine.output(msg);
         } else {
           beeLine.output(beeLine.getColorBuffer().red(msg.getMono()));
         }
       } catch (Throwable t) {
-        beeLine.output(beeLine.getColorBuffer().red(name)); // error with driver
+        beeLine.error("Error processing driver " + name);
       }
     }
     return true;
   }
 
-
   public boolean save(String line) throws IOException {
     beeLine.info(beeLine.loc("saving-options", beeLine.getOpts().getPropertiesFile()));
     beeLine.getOpts().save();
@@ -1670,7 +1671,7 @@ public boolean connect(Properties props) throws IOException {
 
     try {
       beeLine.getDatabaseConnections().setConnection(
-          new DatabaseConnection(beeLine, driver, url, props));
+          new DatabaseConnection(beeLine, url, props));
       beeLine.getDatabaseConnection().getConnection();
 
       if (!beeLine.isBeeLine()) {

beeline/src/java/org/apache/hive/beeline/DatabaseConnection.java

@@ -49,7 +49,6 @@ class DatabaseConnection {
   private final BeeLine beeLine;
   private Connection connection;
   private DatabaseMetaData meta;
-  private final String driver;
   private final String url;
   private final Properties info;
   private Schema schema = null;
@@ -59,10 +58,8 @@ public boolean isClosed() {
     return (null == connection);
   }
 
-  public DatabaseConnection(BeeLine beeLine, String driver, String url,
-       Properties info) throws SQLException {
+  DatabaseConnection(BeeLine beeLine, String url, Properties info) {
     this.beeLine = beeLine;
-    this.driver = driver;
     this.url = url;
     this.info = info;
   }
@@ -85,14 +82,6 @@ void setCompletions(boolean skipmeta) throws SQLException, IOException {
    * Connection to the specified data source.
    */
   boolean connect() throws SQLException {
-    try {
-      if (driver != null && driver.length() != 0) {
-        Class.forName(driver);
-      }
-    } catch (ClassNotFoundException cnfe) {
-      return beeLine.error(cnfe);
-    }
-
     boolean isDriverRegistered = false;
     try {
       isDriverRegistered = DriverManager.getDriver(getUrl()) != null;
@@ -163,18 +152,19 @@ boolean connect() throws SQLException {
 
   public Connection getConnectionFromLocalDriver(String url, Properties properties) {
     Collection<Driver> drivers = beeLine.getDrivers();
-    for (Driver d : drivers) {
+    for (Driver driver : drivers) {
       try {
-        if (d.acceptsURL(url) && beeLine.isSupportedLocalDriver(d)) {
-          String clazzName = d.getClass().getName();
-          beeLine.debug("Driver name is " + clazzName);
-          Driver driver =
-            (Driver) Class.forName(clazzName, true, Thread.currentThread().getContextClassLoader())
-              .newInstance();
+        if (driver.acceptsURL(url) && beeLine.isSupportedLocalDriver(driver)) {
+          beeLine.debug("Driver name is " + driver.getClass().getName());
+          // The 'driver' is already an instance from the ServiceLoader, so we can use it directly.
           return driver.connect(url, properties);
         }
-      } catch (Exception e) {
-        beeLine.error("Fail to connect with a local driver due to the exception:" + e);
+      } catch (SQLException e) {
+        beeLine.error(
+            "Failed to connect with local driver "
+                + driver.getClass().getName()
+                + " due to exception: "
+                + e);
         beeLine.error(e);
       }
     }

beeline/src/test/org/apache/hive/beeline/ProxyAuthTest.java

@@ -44,7 +44,6 @@
  *      <HS2host> <HS2Port> <HS2-Server-principal> <client-principal>
  */
 public class ProxyAuthTest {
-  private static final String driverName = "org.apache.hive.jdbc.HiveDriver";
   private static final String BEELINE_EXIT = "beeline.system.exit";
   private static Connection con = null;
   private static boolean noClose = false;
@@ -68,7 +67,6 @@ public static void main(String[] args) throws Exception {
     File currentResultFile = null;
     String [] beeLineArgs = {};
 
-    Class.forName(driverName);
     String host = args[0];
     String port = args[1];
     String serverPrincipal = args[2];

beeline/src/test/org/apache/hive/beeline/TestShutdownHook.java

@@ -30,11 +30,11 @@ public void testShutdownHook() throws Exception {
     PrintStream ops = new PrintStream(os);
     BeeLine beeline = new BeeLine();
     DatabaseConnections dbConnections = beeline.getDatabaseConnections();
-    dbConnections.setConnection(new DatabaseConnection(beeline,null,null, null));
-    dbConnections.setConnection(new DatabaseConnection(beeline,null,null, null));
+    dbConnections.setConnection(new DatabaseConnection(beeline, null, null));
+    dbConnections.setConnection(new DatabaseConnection(beeline, null, null));
     Assert.assertEquals(2, dbConnections.size());
     beeline.setOutputStream(ops);
     beeline.getShutdownHook().run();
     Assert.assertEquals(0, dbConnections.size());
   }
-}
+}

bin/hive

@@ -28,7 +28,6 @@ bin=`cd "$bin"; pwd`
 SERVICE=""
 HELP=""
 SKIP_HBASECP=false
-SKIP_HADOOPVERSION=false
 EXECUTE_WITH_JAVA=false
 
 SERVICE_ARGS=()
@@ -60,7 +59,6 @@ while [ $# -gt 0 ]; do
       shift
       ;;
     --skiphadoopversion)
-      SKIP_HADOOPVERSION=true
       shift
       ;;
     --skiphbasecp)
@@ -105,10 +103,6 @@ if [[ "$SERVICE" =~ ^(help|version|orcfiledump|rcfilecat|schemaTool|cleardanglin
   SKIP_HBASECP=true
 fi
 
-if [[ "$SERVICE" =~ ^(help|schemaTool)$ ]] ; then
-  SKIP_HADOOPVERSION=true
-fi
-
 if [ -f "${HIVE_CONF_DIR}/hive-env.sh" ]; then
   . "${HIVE_CONF_DIR}/hive-env.sh"
 fi
@@ -298,37 +292,6 @@ if [ ! -f ${HADOOP} ]; then
   fi
 fi
 
-if [ "$EXECUTE_WITH_JAVA" == "false" ]; then
-  if [ "$SKIP_HADOOPVERSION" = false ]; then
-    # Make sure we're using a compatible version of Hadoop
-    if [ "x$HADOOP_VERSION" == "x" ]; then
-        HADOOP_VERSION=$($HADOOP version 2>&2 | awk -F"\t" '/Hadoop/ {print $0}' | cut -d' ' -f 2);
-    fi
-  
-    # Save the regex to a var to workaround quoting incompatabilities
-    # between Bash 3.1 and 3.2
-    hadoop_version_re="^([[:digit:]]+)\.([[:digit:]]+)(\.([[:digit:]]+))?.*$"
-  
-    if [[ "$HADOOP_VERSION" =~ $hadoop_version_re ]]; then
-      hadoop_major_ver=${BASH_REMATCH[1]}
-      hadoop_minor_ver=${BASH_REMATCH[2]}
-      hadoop_patch_ver=${BASH_REMATCH[4]}
-    else
-      echo "Unable to determine Hadoop version information."
-      echo "'hadoop version' returned:"
-      echo `$HADOOP version`
-      exit 5
-    fi
-  
-    if [ "$hadoop_major_ver" -lt "1" -a  "$hadoop_minor_ver$hadoop_patch_ver" -lt "201" ]; then
-      echo "Hive requires Hadoop 0.20.x (x >= 1)."
-      echo "'hadoop version' returned:"
-      echo `$HADOOP version`
-      exit 6
-    fi
-  fi
-fi
-
 if [ "${AUX_PARAM}" != "" ]; then
   if [[ "$SERVICE" != beeline ]]; then
     HIVE_OPTS="$HIVE_OPTS --hiveconf hive.aux.jars.path=${AUX_PARAM}"

common/pom.xml

@@ -414,18 +414,6 @@
       </testResource>
     </testResources>
     <plugins>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-compiler-plugin</artifactId>
-        <configuration>
-          <compilerArgs>
-            <arg>--add-opens</arg>
-            <arg>org.apache.hadoop/org.apache.hadoop.fs=ALL-UNNAMED</arg>
-            <arg>--add-opens</arg>
-            <arg>java.base/java.net=ALL-UNNAMED</arg>
-          </compilerArgs>
-        </configuration>
-      </plugin>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-antrun-plugin</artifactId>