chore(spark): update hadoop dependencies (#1297)

* chore(spark): update hdfs dependencies

* bump hbase version

* add maven repo links for logging libs

* Update spark-k8s/boil-config.toml

Co-authored-by: Sebastian Bernauer <[email protected]>

---------

Co-authored-by: Sebastian Bernauer <[email protected]>

Author: razvan

spark-k8s/boil-config.toml

@@ -1,56 +1,56 @@
 [versions."3.5.6".local-images]
-"hadoop/hadoop" = "3.4.1" # Current Stackable LTS version. Source of the AWS and Azure artifacts to Spark's classpath.
+"hadoop/hadoop" = "3.4.2"
 java-base = "17"
 java-devel = "17"
-hbase = "2.6.2" # Current Stackable LTS version. Used to build the HBase connector.
+hbase = "2.6.3"
 vector = "0.49.0"
 
 [versions."3.5.6".build-arguments]
 python-version = "3.11"
-aws-java-sdk-bundle-version = "2.24.6" # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.4.1
-azure-storage-version = "7.0.1" # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.4.1
-azure-keyvault-core-version = "1.0.0" # https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1
-jackson-dataformat-xml-version = "2.15.2" # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.5.2
-stax2-api-version = "4.2.1" # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2
-woodstox-core-version = "6.5.1" # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2
+aws-java-sdk-bundle-version = "2.29.52"   # needs to match the version shipped by Hadoop
+azure-storage-version = "7.0.1"           # needs to match the version shipped by Hadoop
+azure-keyvault-core-version = "1.0.0"     # needs to match the version shipped by Hadoop
+jackson-dataformat-xml-version = "2.15.2" # needs to match the version shipped by Spark https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.12/3.5.6
+stax2-api-version = "4.2.1"               # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2
+woodstox-core-version = "6.5.1"           # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2
 jmx-exporter-version = "1.3.0"
 tini-version = "0.19.0"
 hbase-connector-version = "1.0.1"
 
 [versions."3.5.7".local-images]
-"hadoop/hadoop" = "3.4.1" # Current Stackable LTS version. Source of the AWS and Azure artifacts to Spark's classpath.
+"hadoop/hadoop" = "3.4.2"
 java-base = "17"
 java-devel = "17"
-hbase = "2.6.2" # Current Stackable LTS version. Used to build the HBase connector.
+hbase = "2.6.3"
 vector = "0.49.0"
 
 [versions."3.5.7".build-arguments]
 python-version = "3.11"
-aws-java-sdk-bundle-version = "2.24.6" # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.4.1
-azure-storage-version = "7.0.1" # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.4.1
-azure-keyvault-core-version = "1.0.0" # https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1
-jackson-dataformat-xml-version = "2.15.2" # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.5.2
-stax2-api-version = "4.2.1" # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2
-woodstox-core-version = "6.5.1" # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2
+aws-java-sdk-bundle-version = "2.29.52"   # needs to match the version shipped by Hadoop
+azure-storage-version = "7.0.1"           # needs to match the version shipped by Hadoop
+azure-keyvault-core-version = "1.0.0"     # needs to match the version shipped by Hadoop
+jackson-dataformat-xml-version = "2.15.2" # needs to match the version shipped by Spark https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.12/3.5.7
+stax2-api-version = "4.2.1"               # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2
+woodstox-core-version = "6.5.1"           # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2
 jmx-exporter-version = "1.3.0"
 tini-version = "0.19.0"
 hbase-connector-version = "1.0.1"
 
 [versions."4.0.1".local-images]
-"hadoop/hadoop" = "3.4.1"
+"hadoop/hadoop" = "3.4.2"
 java-base = "17"
 java-devel = "17"
-hbase = "2.6.2"
+hbase = "2.6.3"
 vector = "0.49.0"
 
 [versions."4.0.1".build-arguments]
 python-version = "3.11"
-aws-java-sdk-bundle-version = "2.24.6"
-azure-storage-version = "7.0.1" # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.3.4
-azure-keyvault-core-version = "1.0.0" # https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1
-jackson-dataformat-xml-version = "2.15.2" # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.5.1
-stax2-api-version = "4.2.1" # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2
-woodstox-core-version = "6.5.1" # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2
+aws-java-sdk-bundle-version = "2.29.52"   # needs to match the version shipped by Hadoop
+azure-storage-version = "7.0.1"           # needs to match the version shipped by Hadoop
+azure-keyvault-core-version = "1.0.0"     # needs to match the version shipped by Hadoop
+jackson-dataformat-xml-version = "2.18.2" # needs to match the version shipped by Spark https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/4.0.1
+stax2-api-version = "4.2.2"               # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.18.2
+woodstox-core-version = "7.0.0"           # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.18.2
 jmx-exporter-version = "1.3.0"
 tini-version = "0.19.0"
-hbase-connector-version = "1.0.1" # This is not supported in Spark 4 yet.
+hbase-connector-version = "1.0.1"         # This is not supported in Spark 4 yet.