Change default Hadoop version to 2.2.0
This change will make the default Hadoop version used throughought AsterixDB v. 2.2.0. It also introduces
dependencyManagement and parameters in the top-level POM (tlp) for easing the management of this dependency
and other dependencies that are similarly pervasive throughout the code, like commons-io and json.
Change-Id: I51e22edb56717d2100d646d5f41a769881c6c86b
Reviewed-on: http://fulliautomatix.ics.uci.edu:8443/125
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <westmann@gmail.com>
diff --git a/asterix-app/pom.xml b/asterix-app/pom.xml
index c98c510..3ea149e 100644
--- a/asterix-app/pom.xml
+++ b/asterix-app/pom.xml
@@ -174,14 +174,12 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- <version>0.20.2</version>
+ <artifactId>hadoop-client</artifactId>
<type>jar</type>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-test</artifactId>
- <version>0.20.2</version>
+ <artifactId>hadoop-hdfs</artifactId>
<type>jar</type>
<scope>test</scope>
</dependency>
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/result/ResultUtils.java b/asterix-app/src/main/java/edu/uci/ics/asterix/result/ResultUtils.java
index a7c4fa0..3707e12 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/result/ResultUtils.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/result/ResultUtils.java
@@ -31,8 +31,7 @@
import org.json.JSONException;
import org.json.JSONObject;
-import com.sun.el.parser.ParseException;
-
+import org.apache.http.ParseException;
import edu.uci.ics.asterix.api.common.APIFramework.DisplayFormat;
import edu.uci.ics.asterix.api.http.servlet.APIServlet;
import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
diff --git a/asterix-app/src/test/java/edu/uci/ics/asterix/test/runtime/HDFSCluster.java b/asterix-app/src/test/java/edu/uci/ics/asterix/test/runtime/HDFSCluster.java
index 2318a04..34a6fea 100644
--- a/asterix-app/src/test/java/edu/uci/ics/asterix/test/runtime/HDFSCluster.java
+++ b/asterix-app/src/test/java/edu/uci/ics/asterix/test/runtime/HDFSCluster.java
@@ -21,7 +21,7 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
@@ -64,7 +64,13 @@
conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/mapred-site.xml"));
conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/hdfs-site.xml"));
cleanupLocal();
- dfsCluster = new MiniDFSCluster(nameNodePort, conf, numDataNodes, true, true, StartupOption.REGULAR, null);
+ //this constructor is deprecated in hadoop 2x
+ //dfsCluster = new MiniDFSCluster(nameNodePort, conf, numDataNodes, true, true, StartupOption.REGULAR, null);
+ MiniDFSCluster.Builder build = new MiniDFSCluster.Builder(conf);
+ build.nameNodePort(nameNodePort);
+ build.numDataNodes(numDataNodes);
+ build.startupOption(StartupOption.REGULAR);
+ dfsCluster = build.build();
dfs = FileSystem.get(conf);
loadData();
}
diff --git a/asterix-app/src/test/resources/hadoop/conf/hdfs-site.xml b/asterix-app/src/test/resources/hadoop/conf/hdfs-site.xml
index bff9a3e..4f3f777 100644
--- a/asterix-app/src/test/resources/hadoop/conf/hdfs-site.xml
+++ b/asterix-app/src/test/resources/hadoop/conf/hdfs-site.xml
@@ -26,7 +26,7 @@
<property>
<name>dfs.block.size</name>
- <value>65536</value>
+ <value>1048576</value>
</property>
</configuration>
diff --git a/asterix-common/pom.xml b/asterix-common/pom.xml
index e0adea2..ff393b2 100644
--- a/asterix-common/pom.xml
+++ b/asterix-common/pom.xml
@@ -126,7 +126,6 @@
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
- <version>1.4</version>
</dependency>
<dependency>
<groupId>commons-httpclient</groupId>
diff --git a/asterix-events/pom.xml b/asterix-events/pom.xml
index adfa543..6f74809 100644
--- a/asterix-events/pom.xml
+++ b/asterix-events/pom.xml
@@ -164,7 +164,6 @@
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
- <version>1.4</version>
</dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
diff --git a/asterix-external-data/pom.xml b/asterix-external-data/pom.xml
index 846bfc6..25337e7 100644
--- a/asterix-external-data/pom.xml
+++ b/asterix-external-data/pom.xml
@@ -130,8 +130,7 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- <version>0.20.2</version>
+ <artifactId>hadoop-client</artifactId>
<type>jar</type>
<scope>compile</scope>
</dependency>
@@ -176,7 +175,7 @@
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-exec</artifactId>
- <version>0.11.0</version>
+ <version>0.13.0</version>
</dependency>
<dependency>
<groupId>javax.jdo</groupId>
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/HDFSAdapter.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/HDFSAdapter.java
index 91e86dc..e0628be 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/HDFSAdapter.java
+++ b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/HDFSAdapter.java
@@ -88,4 +88,4 @@
}
}
-}
\ No newline at end of file
+}
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/indexing/input/AbstractHDFSReader.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/indexing/input/AbstractHDFSReader.java
index 0d340a3..b6bc2c0 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/indexing/input/AbstractHDFSReader.java
+++ b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/indexing/input/AbstractHDFSReader.java
@@ -90,6 +90,10 @@
@Override
public void progress() {
}
+
+ public float getProgress() {
+ return 0.0f;
+ }
};
return reporter;
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/indexing/input/TextualFullScanDataReader.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/indexing/input/TextualFullScanDataReader.java
index e80adb0..ee1e3ee 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/indexing/input/TextualFullScanDataReader.java
+++ b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/indexing/input/TextualFullScanDataReader.java
@@ -204,8 +204,13 @@
@Override
public void progress() {
}
+
+ @Override
+ public float getProgress() {
+ return 0.0f;
+ }
};
return reporter;
}
-}
\ No newline at end of file
+}
diff --git a/asterix-installer/pom.xml b/asterix-installer/pom.xml
index 2181b02..87bd705 100644
--- a/asterix-installer/pom.xml
+++ b/asterix-installer/pom.xml
@@ -234,7 +234,6 @@
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
- <version>1.4</version>
</dependency>
<dependency>
<groupId>edu.uci.ics.asterix</groupId>
diff --git a/asterix-maven-plugins/lexer-generator-maven-plugin/pom.xml b/asterix-maven-plugins/lexer-generator-maven-plugin/pom.xml
index 58ce450..ad9cb15 100644
--- a/asterix-maven-plugins/lexer-generator-maven-plugin/pom.xml
+++ b/asterix-maven-plugins/lexer-generator-maven-plugin/pom.xml
@@ -49,7 +49,6 @@
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-plugin-api</artifactId>
- <version>2.0.2</version>
</dependency>
</dependencies>
</project>
diff --git a/asterix-maven-plugins/pom.xml b/asterix-maven-plugins/pom.xml
index 8b4d520..4fc6dad 100644
--- a/asterix-maven-plugins/pom.xml
+++ b/asterix-maven-plugins/pom.xml
@@ -27,7 +27,6 @@
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-plugin-api</artifactId>
- <version>2.2.1</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
diff --git a/asterix-maven-plugins/record-manager-generator-maven-plugin/pom.xml b/asterix-maven-plugins/record-manager-generator-maven-plugin/pom.xml
index 858fbfa..60e9940 100644
--- a/asterix-maven-plugins/record-manager-generator-maven-plugin/pom.xml
+++ b/asterix-maven-plugins/record-manager-generator-maven-plugin/pom.xml
@@ -44,22 +44,20 @@
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-plugin-api</artifactId>
- <version>2.0.2</version>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-artifact</artifactId>
- <version>2.0.2</version>
+ <version>2.2.1</version>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-project</artifactId>
- <version>2.0.2</version>
+ <version>2.2.1</version>
</dependency>
<dependency>
<groupId>org.json</groupId>
<artifactId>json</artifactId>
- <version>20090211</version>
<type>jar</type>
</dependency>
</dependencies>
diff --git a/asterix-metadata/pom.xml b/asterix-metadata/pom.xml
index e657e26..624ff38 100644
--- a/asterix-metadata/pom.xml
+++ b/asterix-metadata/pom.xml
@@ -57,6 +57,12 @@
<scope>compile</scope>
</dependency>
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-storage-am-lsm-invertedindex</artifactId>
</dependency>
@@ -74,12 +80,5 @@
<version>${hyracks.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- <version>0.20.2</version>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
</dependencies>
</project>
diff --git a/asterix-runtime/pom.xml b/asterix-runtime/pom.xml
index c9ace97..d2e3e00 100644
--- a/asterix-runtime/pom.xml
+++ b/asterix-runtime/pom.xml
@@ -160,8 +160,7 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- <version>0.20.2</version>
+ <artifactId>hadoop-client</artifactId>
<type>jar</type>
<scope>compile</scope>
</dependency>
diff --git a/asterix-tools/pom.xml b/asterix-tools/pom.xml
index 886dbf7..22e1509 100644
--- a/asterix-tools/pom.xml
+++ b/asterix-tools/pom.xml
@@ -104,6 +104,35 @@
</plugins>
</build>
+ <repositories>
+ <repository>
+ <id>central</id>
+ <url>http://repo1.maven.org/maven2</url>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ <snapshots>
+ <enabled>false</enabled>
+ </snapshots>
+ </repository>
+ <repository>
+ <releases>
+ <enabled>true</enabled>
+ <updatePolicy>always</updatePolicy>
+ <checksumPolicy>warn</checksumPolicy>
+ </releases>
+ <snapshots>
+ <enabled>false</enabled>
+ <updatePolicy>never</updatePolicy>
+ <checksumPolicy>fail</checksumPolicy>
+ </snapshots>
+ <id>HDPReleases</id>
+ <name>HDP Releases</name>
+ <url>http://repo.hortonworks.com/content/repositories/releases</url>
+ <layout>default</layout>
+ </repository>
+ </repositories>
+
<dependencies>
<dependency>
<groupId>edu.uci.ics.asterix</groupId>
@@ -130,6 +159,66 @@
<scope>compile</scope>
</dependency>
<dependency>
+ <groupId>edu.uci.ics.asterix</groupId>
+ <artifactId>asterix-metadata</artifactId>
+ <version>0.8.7-SNAPSHOT</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.asterix</groupId>
+ <artifactId>asterix-events</artifactId>
+ <version>0.8.7-SNAPSHOT</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ <version>1.4</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-net</groupId>
+ <artifactId>commons-net</artifactId>
+ <version>3.1</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <version>2.5</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-collections</groupId>
+ <artifactId>commons-collections</artifactId>
+ <version>3.2.1</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-configuration</groupId>
+ <artifactId>commons-configuration</artifactId>
+ <version>1.6</version>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ <version>1.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-math</artifactId>
+ <version>2.1</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.2.2</version>
diff --git a/pom.xml b/pom.xml
index 21e7647..ad763e1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -21,8 +21,7 @@
<packaging>pom</packaging>
<properties>
- <algebricks.version>0.2.13</algebricks.version>
- <hyracks.version>0.2.13</hyracks.version>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<jvm.extraargs />
<runSlowAQLTests>false</runSlowAQLTests>
@@ -33,6 +32,14 @@
<invalid.tests>**/DmlTest.java</invalid.tests>
<global.test.includes>**/*TestSuite.java,**/*Test.java,${execution.tests}</global.test.includes>
<global.test.excludes>${optimizer.tests},${metadata.tests},${invalid.tests}</global.test.excludes>
+ <!-- Versions under dependencymanagement or used in many projects via properties -->
+ <algebricks.version>0.2.14-SNAPSHOT</algebricks.version>
+ <hyracks.version>0.2.14-SNAPSHOT</hyracks.version>
+ <hadoop.version>2.2.0</hadoop.version>
+ <junit.version>4.8.1</junit.version>
+ <commons.io.version>2.4</commons.io.version>
+ <servlet.api.version>2.5</servlet.api.version>
+ <json.version>20090211</json.version>
</properties>
<build>
@@ -222,8 +229,38 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
- <version>4.8.1</version>
+ <version>${junit.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.maven</groupId>
+ <artifactId>maven-plugin-api</artifactId>
+ <version>2.2.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-client</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>algebricks-compiler</artifactId>
@@ -314,15 +351,20 @@
<dependency>
<groupId>org.json</groupId>
<artifactId>json</artifactId>
- <version>20090211</version>
+ <version>${json.version}</version>
<type>jar</type>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
- <version>2.5</version>
+ <version>${servlet.api.version}</version>
<type>jar</type>
</dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>${commons.io.version}</version>
+ </dependency>
</dependencies>
</dependencyManagement>
</project>