YARN integration for AsterixDB
This is an initial version of YARN integration for AsterixDB.
- Uses static assignment of CC and NC nodes to NM locations
- Stores state locally on each NM, outside of HDFS
- "All or nothing" container allocation. We don't attempt to
move or rellocate containers the RM may kill (yet).
- Retains feature parity with managix.
Change-Id: I49c849179d17fc7faa446b9be57a0695df6836ab
Reviewed-on: https://asterix-gerrit.ics.uci.edu/161
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <hubailmor@gmail.com>
diff --git a/asterix-common/pom.xml b/asterix-common/pom.xml
index 70d384b..3afba1e 100644
--- a/asterix-common/pom.xml
+++ b/asterix-common/pom.xml
@@ -110,6 +110,40 @@
</bindingIncludes>
</configuration>
</execution>
+ <execution>
+ <id>yarn_cluster</id>
+ <goals>
+ <goal>generate</goal>
+ </goals>
+ <configuration>
+ <args>
+ <arg>-Xsetters</arg>
+ <arg>-Xvalue-constructor</arg>
+ </args>
+ <plugins>
+ <plugin>
+ <groupId>org.jvnet.jaxb2_commons</groupId>
+ <artifactId>jaxb2-basics</artifactId>
+ <version>0.6.2</version>
+ </plugin>
+ <plugin>
+ <groupId>org.jvnet.jaxb2_commons</groupId>
+ <artifactId>jaxb2-value-constructor</artifactId>
+ <version>3.0</version>
+ </plugin>
+ </plugins>
+ <schemaDirectory>src/main/resources/schema</schemaDirectory>
+ <schemaIncludes>
+ <include>yarn_cluster.xsd</include>
+ </schemaIncludes>
+ <generatePackage>edu.uci.ics.asterix.event.schema.yarnCluster</generatePackage>
+ <generateDirectory>${project.build.directory}/generated-sources/yarn_cluster</generateDirectory>
+ <bindingDirectory>src/main/resources/schema</bindingDirectory>
+ <bindingIncludes>
+ <bindingInclude>jaxb-bindings.xjb</bindingInclude>
+ </bindingIncludes>
+ </configuration>
+ </execution>
</executions>
</plugin>
<plugin>
diff --git a/asterix-common/src/main/resources/schema/asterix-conf.xsd b/asterix-common/src/main/resources/schema/asterix-conf.xsd
index 3033893..efda4b7 100644
--- a/asterix-common/src/main/resources/schema/asterix-conf.xsd
+++ b/asterix-common/src/main/resources/schema/asterix-conf.xsd
@@ -7,6 +7,7 @@
<xs:element name="instanceName" type="xs:string" />
+ <xs:element name="version" type="xs:string"/>
<xs:element name="metadataNode" type="xs:string" />
<xs:element name="coredumpPath" type="xs:string" />
<xs:element name="storeDirs" type="xs:string" />
@@ -60,6 +61,7 @@
<xs:complexType>
<xs:sequence>
<xs:element ref="mg:instanceName" minOccurs="0" />
+ <xs:element ref="mg:version" minOccurs="0" />
<xs:element ref="mg:metadataNode" minOccurs="0" />
<xs:element ref="mg:store" maxOccurs="unbounded" />
<xs:element ref="mg:coredump" maxOccurs="unbounded" />
diff --git a/asterix-common/src/main/resources/schema/yarn_cluster.xsd b/asterix-common/src/main/resources/schema/yarn_cluster.xsd
new file mode 100644
index 0000000..2aa76cf
--- /dev/null
+++ b/asterix-common/src/main/resources/schema/yarn_cluster.xsd
@@ -0,0 +1,103 @@
+<?xml version="1.0" encoding="ISO-8859-1" ?>
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
+ xmlns:cl="yarn_cluster" targetNamespace="yarn_cluster" elementFormDefault="qualified">
+
+ <!-- definition of simple types -->
+ <xs:element name="instance_name" type="xs:string" />
+ <xs:element name="cluster_name" type="xs:string" />
+ <xs:element name="log_dir" type="xs:string" />
+ <xs:element name="txn_log_dir" type="xs:string" />
+ <xs:element name="id" type="xs:string" />
+ <xs:element name="client_ip" type="xs:string" />
+ <xs:element name="cluster_ip" type="xs:string" />
+ <xs:element name="key" type="xs:string" />
+ <xs:element name="value" type="xs:string" />
+ <xs:element name="store" type="xs:string" />
+ <xs:element name="iodevices" type="xs:string" />
+ <xs:element name="web_port" type="xs:string" />
+ <xs:element name="client_port" type="xs:integer" />
+ <xs:element name="cluster_port" type="xs:integer" />
+ <xs:element name="http_port" type="xs:integer" />
+ <xs:element name="debug_port" type="xs:integer" />
+ <xs:element name="metadata_node" type="xs:string" />
+ <xs:element name="cc_container_mem" type="xs:string" />
+ <xs:element name="nc_container_mem" type="xs:string" />
+
+
+ <!-- definition of complex elements -->
+
+ <xs:element name="master_node">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element ref="cl:id" />
+ <xs:element ref="cl:client_ip" />
+ <xs:element ref="cl:cluster_ip" />
+ <xs:element ref="cl:log_dir" minOccurs="0" />
+ <xs:element ref="cl:client_port" />
+ <xs:element ref="cl:cluster_port" />
+ <xs:element ref="cl:http_port" />
+ <xs:element ref="cl:debug_port" minOccurs="0" />
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="property">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element ref="cl:key" />
+ <xs:element ref="cl:value" />
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="env">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element ref="cl:property" minOccurs="0" maxOccurs="unbounded" />
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="node">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element ref="cl:id" />
+ <xs:element ref="cl:cluster_ip" />
+ <xs:element ref="cl:log_dir" minOccurs="0" />
+ <xs:element ref="cl:txn_log_dir" minOccurs="0" />
+ <xs:element ref="cl:store" minOccurs="0" />
+ <xs:element ref="cl:iodevices" minOccurs="0" />
+ <xs:element ref="cl:debug_port" minOccurs="0" />
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="substitute_nodes">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element ref="cl:node" maxOccurs="unbounded" />
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="cluster">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element ref="cl:instance_name" />
+ <xs:element ref="cl:cluster_name" />
+ <xs:element ref="cl:cc_container_mem" minOccurs="0" />
+ <xs:element ref="cl:nc_container_mem" minOccurs="0" />
+ <xs:element ref="cl:env" minOccurs="0" />
+ <xs:element ref="cl:log_dir" minOccurs="0" />
+ <xs:element ref="cl:txn_log_dir" minOccurs="0" />
+ <xs:element ref="cl:store" minOccurs="0" />
+ <xs:element ref="cl:iodevices" minOccurs="0" />
+ <xs:element ref="cl:metadata_node" />
+ <xs:element ref="cl:master_node" />
+ <xs:element ref="cl:node" maxOccurs="unbounded" />
+ <xs:element ref="cl:substitute_nodes" />
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+</xs:schema>