[ASTERIXDB-2955] Upgrade to Hadoop3
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Upgrade to Hadoop v3
Change-Id: Ia212ddbe5eed114042118fecc4cc08db93cc4e09
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12623
Reviewed-by: Wael Alkowaileet <wael.y.k@gmail.com>
Reviewed-by: Michael Blow <mblow@apache.org>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JInterval.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JInterval.java
index 14caa1e..39a8909 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JInterval.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JInterval.java
@@ -27,7 +27,7 @@
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
-import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.hyracks.api.exceptions.HyracksDataException;
public final class JInterval extends JObject<List<Long>> {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index dacf464..08d803e 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -41,7 +41,6 @@
import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CLIENT_SECRET_FIELD_NAME;
import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CONNECTION_STRING_FIELD_NAME;
import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.TENANT_ID_FIELD_NAME;
-import static org.apache.asterix.external.util.ExternalDataConstants.CONTAINER_NAME_FIELD_NAME;
import static org.apache.asterix.external.util.ExternalDataConstants.GCS.JSON_CREDENTIALS_FIELD_NAME;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_ADAPTER_NAME_GCS;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_DELIMITER;
@@ -92,6 +91,7 @@
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.AUnionType;
import org.apache.asterix.runtime.evaluators.common.NumberUtils;
+import org.apache.hadoop.fs.s3a.Constants;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hyracks.algebricks.common.exceptions.NotImplementedException;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -878,7 +878,6 @@
String accessKeyId = configuration.get(ExternalDataConstants.AwsS3.ACCESS_KEY_ID_FIELD_NAME);
String secretAccessKey = configuration.get(ExternalDataConstants.AwsS3.SECRET_ACCESS_KEY_FIELD_NAME);
String sessionToken = configuration.get(ExternalDataConstants.AwsS3.SESSION_TOKEN_FIELD_NAME);
- String regionId = configuration.get(ExternalDataConstants.AwsS3.REGION_FIELD_NAME);
String serviceEndpoint = configuration.get(ExternalDataConstants.AwsS3.SERVICE_END_POINT_FIELD_NAME);
/*
@@ -915,7 +914,8 @@
// Validation of the URL should be done at hadoop-aws level
conf.set(ExternalDataConstants.AwsS3.HADOOP_SERVICE_END_POINT, serviceEndpoint);
} else {
- conf.set(ExternalDataConstants.AwsS3.HADOOP_REGION, regionId);
+ //Region is ignored and buckets could be found by the central endpoint
+ conf.set(ExternalDataConstants.AwsS3.HADOOP_SERVICE_END_POINT, Constants.CENTRAL_ENDPOINT);
}
}
diff --git a/asterixdb/asterix-server/pom.xml b/asterixdb/asterix-server/pom.xml
index e76814c..96007c1 100644
--- a/asterixdb/asterix-server/pom.xml
+++ b/asterixdb/asterix-server/pom.xml
@@ -199,25 +199,11 @@
</gavs>
<noticeUrl>https://raw.githubusercontent.com/aws/aws-sdk-java-v2/2.10.83/NOTICE.txt</noticeUrl>
</override>
- <!-- Hadoop AWS SDK -->
<override>
- <gavs>
- <gav>com.amazonaws:aws-java-sdk-core:1.12.1</gav>
- <gav>com.amazonaws:jmespath-java:1.12.1</gav>
- <gav>com.amazonaws:aws-java-sdk-s3:1.12.1</gav>
- <gav>com.amazonaws:aws-java-sdk-kms:1.12.1</gav>
- </gavs>
+ <gav>com.amazonaws:aws-java-sdk-bundle:1.12.1</gav>
<noticeUrl>https://raw.githubusercontent.com/aws/aws-sdk-java/1.12.1/NOTICE.txt</noticeUrl>
</override>
<override>
- <gav>software.amazon.eventstream:eventstream:1.0.1</gav>
- <noticeUrl>https://raw.githubusercontent.com/awslabs/aws-eventstream-java/7be2dd80e12f8835674c8ffb0f4a2efb64c7b585/NOTICE</noticeUrl>
- </override>
- <override>
- <gav>software.amazon.ion:ion-java:1.0.2</gav>
- <noticeUrl>https://raw.githubusercontent.com/amzn/ion-java/v1.0.2/NOTICE</noticeUrl>
- </override>
- <override>
<gavs>
<gav>com.azure:azure-core:1.17.0</gav>
<gav>com.azure:azure-core-http-netty:1.10.0</gav>
@@ -456,6 +442,7 @@
<aliasUrl>https://raw.githubusercontent.com/googleapis/java-common-protos/v2.1.0/proto-google-common-protos/LICENSE</aliasUrl>
<aliasUrl>https://raw.githubusercontent.com/googleapis/java-iam/v1.0.13/proto-google-iam-v1/LICENSE</aliasUrl>
<aliasUrl>https://raw.githubusercontent.com/googleapis/java-storage/v1.114.0/LICENSE</aliasUrl>
+ <aliasUrl>http://repository.jboss.org/licenses/apache-2.0.txt</aliasUrl>
</aliasUrls>
<metric>1</metric>
</license>
@@ -528,6 +515,13 @@
<aliasUrl>https://raw.githubusercontent.com/luben/zstd-jni/v1.4.9-1/LICENSE</aliasUrl>
</aliasUrls>
</license>
+ <license>
+ <displayName>The Go license</displayName>
+ <url>http://golang.org/LICENSE</url>
+ <aliasUrls>
+ <aliasUrl>https://golang.org/LICENSE</aliasUrl>
+ </aliasUrls>
+ </license>
</licenses>
<templateProperties>
<packageName>Apache AsterixDB Server Install</packageName>
diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml
index 7fc4e26..71228df 100644
--- a/asterixdb/pom.xml
+++ b/asterixdb/pom.xml
@@ -84,7 +84,7 @@
<!-- Versions under dependencymanagement or used in many projects via properties -->
<algebricks.version>0.3.7-SNAPSHOT</algebricks.version>
<hyracks.version>0.3.7-SNAPSHOT</hyracks.version>
- <hadoop.version>2.8.5</hadoop.version>
+ <hadoop.version>3.3.1</hadoop.version>
<jacoco.version>0.7.6.201602180812</jacoco.version>
<log4j.version>2.14.1</log4j.version>
<awsjavasdk.version>2.10.83</awsjavasdk.version>
@@ -989,7 +989,7 @@
<artifactId>commons-logging</artifactId>
</exclusion>
<exclusion>
- <groupId>stax</groupId>
+ <groupId>org.codehaus.woodstox</groupId>
<artifactId>stax-api</artifactId>
</exclusion>
<exclusion>
@@ -1008,6 +1008,26 @@
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.fasterxml</groupId>
+ <artifactId>woodstox-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>dnsjava</groupId>
+ <artifactId>dnsjava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.wildfly.openssl</groupId>
+ <artifactId>wildfly-openssl</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.jline</groupId>
+ <artifactId>jline</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.ws.rs</groupId>
+ <artifactId>jsr311-api</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -1046,6 +1066,10 @@
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.jline</groupId>
+ <artifactId>jline</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -1767,6 +1791,17 @@
</exclusions>
</dependency>
<dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-bundle</artifactId>
+ <version>${hadoop-awsjavasdk.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
<version>${hadoop.version}</version>
diff --git a/asterixdb/src/main/appended-resources/supplemental-models.xml b/asterixdb/src/main/appended-resources/supplemental-models.xml
index 305176b..f99dc60 100644
--- a/asterixdb/src/main/appended-resources/supplemental-models.xml
+++ b/asterixdb/src/main/appended-resources/supplemental-models.xml
@@ -485,68 +485,19 @@
<!-- AWS SDK end -->
<!-- AWS Hadoop SDK start -->
- <!-- software.amazon.awssdk is ALv2, and does not contain any embedded LICENSE or NOTICE file -->
- <!-- license override not needed, ALv2 is specified in its pom.xml -->
- <!-- see https://github.com/aws/aws-sdk-java-v2/blob/master/LICENSE.txt -->
+ <!-- Contains (LICENSE and LICENSE.txt) and (NOTICE and NOTICE.txt) -->
+ <!-- Overridden to ignore the warning of having multiple licenses and notices -->
<supplement>
<project>
<groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-core</artifactId>
+ <artifactId>aws-java-sdk-bundle</artifactId>
<properties>
<license.ignoreMissingEmbeddedLicense>1.12.1</license.ignoreMissingEmbeddedLicense>
<license.ignoreMissingEmbeddedNotice>1.12.1</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreLicenseOverride>1.12.1</license.ignoreLicenseOverride>
<license.ignoreNoticeOverride>1.12.1</license.ignoreNoticeOverride>
- </properties>
- </project>
- </supplement>
-
- <supplement>
- <project>
- <groupId>com.amazonaws</groupId>
- <artifactId>jmespath-java</artifactId>
- <properties>
- <license.ignoreMissingEmbeddedLicense>1.12.1</license.ignoreMissingEmbeddedLicense>
- <license.ignoreMissingEmbeddedNotice>1.12.1</license.ignoreMissingEmbeddedNotice>
- <license.ignoreNoticeOverride>1.12.1</license.ignoreNoticeOverride>
- </properties>
- </project>
- </supplement>
-
- <supplement>
- <project>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-s3</artifactId>
- <properties>
- <license.ignoreMissingEmbeddedLicense>1.12.1</license.ignoreMissingEmbeddedLicense>
- <license.ignoreMissingEmbeddedNotice>1.12.1</license.ignoreMissingEmbeddedNotice>
- <license.ignoreNoticeOverride>1.12.1</license.ignoreNoticeOverride>
- </properties>
- </project>
- </supplement>
-
- <supplement>
- <project>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-kms</artifactId>
- <properties>
- <license.ignoreMissingEmbeddedLicense>1.12.1</license.ignoreMissingEmbeddedLicense>
- <license.ignoreMissingEmbeddedNotice>1.12.1</license.ignoreMissingEmbeddedNotice>
- <license.ignoreNoticeOverride>1.12.1</license.ignoreNoticeOverride>
- </properties>
- </project>
- </supplement>
-
- <!-- software.amazon.ion is ALv2, and does not contain any embedded LICENSE or NOTICE file -->
- <!-- license override not needed, ALv2 is specified in its pom.xml -->
- <!-- see https://github.com/amzn/ion-java -->
- <supplement>
- <project>
- <groupId>software.amazon.ion</groupId>
- <artifactId>ion-java</artifactId>
- <properties>
- <license.ignoreMissingEmbeddedLicense>1.0.2</license.ignoreMissingEmbeddedLicense>
- <license.ignoreMissingEmbeddedNotice>1.0.2</license.ignoreMissingEmbeddedNotice>
- <license.ignoreNoticeOverride>1.0.2</license.ignoreNoticeOverride>
+ <license.alternateLicenseFile>1.12.1:www.apache.org_licenses_LICENSE-2.0.txt</license.alternateLicenseFile>
+ <license.alternateNoticeFile>1.12.1:raw.githubusercontent.com_aws_aws-sdk-java_1.12.1_NOTICE.txt</license.alternateNoticeFile>
</properties>
</project>
</supplement>
@@ -561,7 +512,7 @@
<artifactId>netty-reactive-streams</artifactId>
<properties>
<license.ignoreMissingEmbeddedLicense>2.0.4</license.ignoreMissingEmbeddedLicense>
- <license.ignoreMissingEmbeddedNotice>2.0.4</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreMissingEmbeddedNotice>2.0.4</license.ignoreMissingEmbeddedNotice>doesn
</properties>
</project>
</supplement>
diff --git a/asterixdb/src/main/licenses/content/golang.org_LICENSE.txt b/asterixdb/src/main/licenses/content/golang.org_LICENSE.txt
new file mode 100644
index 0000000..ea5ea89
--- /dev/null
+++ b/asterixdb/src/main/licenses/content/golang.org_LICENSE.txt
@@ -0,0 +1,27 @@
+Copyright (c) 2009 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+ * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
diff --git a/hyracks-fullstack/pom.xml b/hyracks-fullstack/pom.xml
index 15c1c15..5f83aab 100644
--- a/hyracks-fullstack/pom.xml
+++ b/hyracks-fullstack/pom.xml
@@ -70,7 +70,7 @@
<test.includes>${global.test.includes}</test.includes>
<test.excludes>${global.test.excludes}</test.excludes>
<!-- Versions under dependencymanagement or used in many projects via properties -->
- <hadoop.version>2.8.5</hadoop.version>
+ <hadoop.version>3.3.1</hadoop.version>
<jacoco.version>0.7.6.201602180812</jacoco.version>
<log4j.version>2.14.1</log4j.version>
<snappy.version>1.1.8.4</snappy.version>