Copied hyracks trunk into fullstack
git-svn-id: https://hyracks.googlecode.com/svn/branches/fullstack_staging@1958 123451ca-8445-de46-9d55-352943316053
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-am/pom.xml b/hyracks/hyracks-yarn/hyracks-yarn-am/pom.xml
new file mode 100644
index 0000000..9e453a6
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-am/pom.xml
@@ -0,0 +1,74 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hyracks-yarn-am</artifactId>
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-yarn</artifactId>
+ <version>0.2.1-SNAPSHOT</version>
+ </parent>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.0.2</version>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>appassembler-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <configuration>
+ <programs>
+ <program>
+ <mainClass>edu.uci.ics.hyracks.yarn.am.HyracksYarnApplicationMaster</mainClass>
+ <name>hyracks-yarn-am</name>
+ </program>
+ </programs>
+ <repositoryLayout>flat</repositoryLayout>
+ <repositoryName>lib</repositoryName>
+ </configuration>
+ <phase>package</phase>
+ <goals>
+ <goal>assemble</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.2-beta-5</version>
+ <executions>
+ <execution>
+ <configuration>
+ <descriptors>
+ <descriptor>src/main/assembly/binary-assembly.xml</descriptor>
+ </descriptors>
+ </configuration>
+ <phase>package</phase>
+ <goals>
+ <goal>attached</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>args4j</groupId>
+ <artifactId>args4j</artifactId>
+ <version>2.0.16</version>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-yarn-common</artifactId>
+ <version>0.2.1-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/assembly/binary-assembly.xml b/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/assembly/binary-assembly.xml
new file mode 100644
index 0000000..0500499
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/assembly/binary-assembly.xml
@@ -0,0 +1,19 @@
+<assembly>
+ <id>binary-assembly</id>
+ <formats>
+ <format>zip</format>
+ <format>dir</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>target/appassembler/bin</directory>
+ <outputDirectory>bin</outputDirectory>
+ <fileMode>0755</fileMode>
+ </fileSet>
+ <fileSet>
+ <directory>target/appassembler/lib</directory>
+ <outputDirectory>lib</outputDirectory>
+ </fileSet>
+ </fileSets>
+</assembly>
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/HyracksYarnApplicationMaster.java b/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/HyracksYarnApplicationMaster.java
new file mode 100644
index 0000000..a9c4520
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/HyracksYarnApplicationMaster.java
@@ -0,0 +1,233 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.yarn.am;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.Timer;
+import java.util.TimerTask;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.util.Records;
+import org.kohsuke.args4j.CmdLineParser;
+
+import edu.uci.ics.hyracks.yarn.am.manifest.AbstractProcess;
+import edu.uci.ics.hyracks.yarn.am.manifest.ContainerSpecification;
+import edu.uci.ics.hyracks.yarn.am.manifest.HyracksCluster;
+import edu.uci.ics.hyracks.yarn.am.manifest.ManifestParser;
+import edu.uci.ics.hyracks.yarn.am.manifest.NodeController;
+import edu.uci.ics.hyracks.yarn.common.protocols.amrm.AMRMConnection;
+
+public class HyracksYarnApplicationMaster {
+ private final Options options;
+
+ private final Timer timer;
+
+ private final List<ResourceRequest> asks;
+
+ private final Map<Resource, Set<AskRecord>> resource2AskMap;
+
+ private final Map<AbstractProcess, AskRecord> proc2AskMap;
+
+ private YarnConfiguration config;
+
+ private AMRMConnection amrmc;
+
+ private RegisterApplicationMasterResponse registration;
+
+ private HyracksCluster hcManifest;
+
+ private HyracksYarnApplicationMaster(Options options) {
+ this.options = options;
+ timer = new Timer(true);
+ asks = new ArrayList<ResourceRequest>();
+ resource2AskMap = new HashMap<Resource, Set<AskRecord>>();
+ proc2AskMap = new HashMap<AbstractProcess, AskRecord>();
+ }
+
+ private void run() throws Exception {
+ Configuration conf = new Configuration();
+ config = new YarnConfiguration(conf);
+ amrmc = new AMRMConnection(config);
+
+ performRegistration();
+ setupHeartbeats();
+ parseManifest();
+ setupAsks();
+ while (true) {
+ Thread.sleep(1000);
+ }
+ }
+
+ private synchronized void setupAsks() {
+ setupAsk(hcManifest.getClusterController());
+ for (NodeController nc : hcManifest.getNodeControllers()) {
+ setupAsk(nc);
+ }
+ }
+
+ private void setupAsk(AbstractProcess proc) {
+ ContainerSpecification cSpec = proc.getContainerSpecification();
+ ResourceRequest rsrcRequest = Records.newRecord(ResourceRequest.class);
+
+ rsrcRequest.setHostName(cSpec.getHostname());
+
+ Priority pri = Records.newRecord(Priority.class);
+ pri.setPriority(100);
+ rsrcRequest.setPriority(pri);
+
+ Resource capability = Records.newRecord(Resource.class);
+ capability.setMemory(cSpec.getMemory());
+ rsrcRequest.setCapability(capability);
+
+ rsrcRequest.setNumContainers(1);
+
+ AskRecord ar = new AskRecord();
+ ar.req = rsrcRequest;
+ ar.proc = proc;
+
+ Set<AskRecord> arSet = resource2AskMap.get(capability);
+ if (arSet == null) {
+ arSet = new HashSet<AskRecord>();
+ resource2AskMap.put(capability, arSet);
+ }
+ arSet.add(ar);
+ proc2AskMap.put(proc, ar);
+
+ System.err.println(proc + " -> " + rsrcRequest);
+
+ asks.add(rsrcRequest);
+ }
+
+ private void parseManifest() throws Exception {
+ String str = FileUtils.readFileToString(new File("manifest.xml"));
+ hcManifest = ManifestParser.parse(str);
+ }
+
+ private void setupHeartbeats() {
+ long heartbeatInterval = config.getLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS,
+ YarnConfiguration.DEFAULT_RM_AM_EXPIRY_INTERVAL_MS);
+ System.err.println("Heartbeat interval: " + heartbeatInterval);
+ heartbeatInterval = Math.min(heartbeatInterval, 1000);
+ System.err.println("Heartbeat interval: " + heartbeatInterval);
+ timer.schedule(new TimerTask() {
+ @Override
+ public void run() {
+ AllocateRequest hb = Records.newRecord(AllocateRequest.class);
+ populateAllocateRequest(hb);
+ hb.setApplicationAttemptId(amrmc.getApplicationAttemptId());
+ hb.setProgress(0);
+ try {
+ AllocateResponse allocateResponse = amrmc.getAMRMProtocol().allocate(hb);
+ List<Container> allocatedContainers = allocateResponse.getAMResponse().getAllocatedContainers();
+ List<ContainerStatus> completedContainers = allocateResponse.getAMResponse()
+ .getCompletedContainersStatuses();
+ processAllocation(allocatedContainers, completedContainers);
+ } catch (YarnRemoteException e) {
+ e.printStackTrace();
+ }
+ }
+ }, 0, heartbeatInterval);
+ }
+
+ private synchronized void populateAllocateRequest(AllocateRequest hb) {
+ hb.addAllAsks(asks);
+ asks.clear();
+ }
+
+ private synchronized void processAllocation(List<Container> allocatedContainers,
+ List<ContainerStatus> completedContainers) {
+ for (Container c : allocatedContainers) {
+ System.err.println("Got container: " + c.getContainerStatus());
+ NodeId nodeId = c.getNodeId();
+ Resource resource = c.getResource();
+
+ Set<AskRecord> arSet = resource2AskMap.get(resource);
+ boolean found = false;
+ if (arSet != null) {
+ AskRecord wildcardMatch = null;
+ AskRecord nameMatch = null;
+ for (AskRecord ar : arSet) {
+ ResourceRequest req = ar.req;
+ if (ar.allocation == null) {
+ if ("*".equals(req.getHostName()) && wildcardMatch == null) {
+ wildcardMatch = ar;
+ }
+ if (req.getHostName().equals(nodeId.getHost()) && nameMatch == null) {
+ nameMatch = ar;
+ break;
+ }
+ }
+ }
+ if (nameMatch != null) {
+ found = true;
+ nameMatch.allocation = c;
+ } else if (wildcardMatch != null) {
+ found = true;
+ wildcardMatch.allocation = c;
+ }
+ }
+ if (!found) {
+ System.err.println("Unknown request satisfied: " + resource);
+ }
+ }
+ }
+
+ private void performRegistration() throws YarnRemoteException {
+ RegisterApplicationMasterRequest appMasterRequest = Records.newRecord(RegisterApplicationMasterRequest.class);
+ appMasterRequest.setApplicationAttemptId(amrmc.getApplicationAttemptId());
+
+ registration = amrmc.getAMRMProtocol().registerApplicationMaster(appMasterRequest);
+ }
+
+ public static void main(String[] args) throws Exception {
+ Options options = new Options();
+ CmdLineParser parser = new CmdLineParser(options);
+ try {
+ parser.parseArgument(args);
+ } catch (Exception e) {
+ parser.printUsage(System.err);
+ return;
+ }
+ new HyracksYarnApplicationMaster(options).run();
+ }
+
+ private static class Options {
+ }
+
+ private static class AskRecord {
+ ResourceRequest req;
+ AbstractProcess proc;
+ Container allocation;
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/AbstractProcess.java b/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/AbstractProcess.java
new file mode 100644
index 0000000..be6e263
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/AbstractProcess.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.yarn.am.manifest;
+
+public abstract class AbstractProcess {
+ protected ContainerSpecification cSpec;
+
+ protected String cmdLineArgs;
+
+ public void setContainerSpecification(ContainerSpecification cSpec) {
+ this.cSpec = cSpec;
+ }
+
+ public ContainerSpecification getContainerSpecification() {
+ return cSpec;
+ }
+
+ public void setCommandLineArguments(String cmdLineArgs) {
+ this.cmdLineArgs = cmdLineArgs;
+ }
+
+ public String getCommandLineArguments() {
+ return cmdLineArgs;
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/ClusterController.java b/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/ClusterController.java
new file mode 100644
index 0000000..b230ada
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/ClusterController.java
@@ -0,0 +1,18 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.yarn.am.manifest;
+
+public class ClusterController extends AbstractProcess {
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/ContainerSpecification.java b/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/ContainerSpecification.java
new file mode 100644
index 0000000..07f12c8
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/ContainerSpecification.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.yarn.am.manifest;
+
+public class ContainerSpecification {
+ private String hostname;
+
+ private int memory;
+
+ public ContainerSpecification() {
+ hostname = "*";
+ }
+
+ public void setHostname(String hostname) {
+ this.hostname = hostname;
+ }
+
+ public String getHostname() {
+ return hostname;
+ }
+
+ public void setMemory(int memory) {
+ this.memory = memory;
+ }
+
+ public int getMemory() {
+ return memory;
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/HyracksCluster.java b/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/HyracksCluster.java
new file mode 100644
index 0000000..4a66ed6
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/HyracksCluster.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.yarn.am.manifest;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class HyracksCluster {
+ private String name;
+
+ private ClusterController cc;
+
+ private List<NodeController> ncs;
+
+ public HyracksCluster() {
+ ncs = new ArrayList<NodeController>();
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setClusterController(ClusterController cc) {
+ this.cc = cc;
+ }
+
+ public ClusterController getClusterController() {
+ return cc;
+ }
+
+ public void addNodeController(NodeController nc) {
+ ncs.add(nc);
+ }
+
+ public List<NodeController> getNodeControllers() {
+ return ncs;
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/ManifestParser.java b/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/ManifestParser.java
new file mode 100644
index 0000000..bc5ebae
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/ManifestParser.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.yarn.am.manifest;
+
+import java.io.StringReader;
+
+import org.apache.commons.digester.Digester;
+
+public class ManifestParser {
+ public static HyracksCluster parse(String mXML) throws Exception {
+ Digester d = createDigester();
+ return (HyracksCluster) d.parse(new StringReader(mXML));
+ }
+
+ private static Digester createDigester() {
+ Digester d = new Digester();
+ d.setValidating(false);
+
+ d.addObjectCreate("hyracks-cluster", HyracksCluster.class);
+ d.addSetProperties("hyracks-cluster");
+
+ d.addObjectCreate("hyracks-cluster/cluster-controller", ClusterController.class);
+ d.addSetProperties("hyracks-cluster/cluster-controller");
+ d.addSetNext("hyracks-cluster/cluster-controller", "setClusterController");
+
+ d.addObjectCreate("hyracks-cluster/node-controllers/node-controller", NodeController.class);
+ d.addSetProperties("hyracks-cluster/node-controllers/node-controller");
+ d.addSetNext("hyracks-cluster/node-controllers/node-controller", "addNodeController");
+
+ d.addObjectCreate("*/container-specification", ContainerSpecification.class);
+ d.addSetProperties("*/container-specification");
+ d.addSetNext("*/container-specification", "setContainerSpecification");
+ return d;
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/NodeController.java b/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/NodeController.java
new file mode 100644
index 0000000..f46eb7f
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/NodeController.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.yarn.am.manifest;
+
+public class NodeController extends AbstractProcess {
+ private String id;
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ public String getId() {
+ return id;
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-client/pom.xml b/hyracks/hyracks-yarn/hyracks-yarn-client/pom.xml
new file mode 100644
index 0000000..08935a7
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-client/pom.xml
@@ -0,0 +1,85 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hyracks-yarn-client</artifactId>
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-yarn</artifactId>
+ <version>0.2.1-SNAPSHOT</version>
+ </parent>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.0.2</version>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>appassembler-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <configuration>
+ <programs>
+ <program>
+ <mainClass>edu.uci.ics.hyracks.yarn.client.LaunchHyracksApplication</mainClass>
+ <name>launch-hyracks-application</name>
+ </program>
+ <program>
+ <mainClass>edu.uci.ics.hyracks.yarn.client.KillHyracksApplication</mainClass>
+ <name>kill-hyracks-application</name>
+ </program>
+ </programs>
+ <repositoryLayout>flat</repositoryLayout>
+ <repositoryName>lib</repositoryName>
+ </configuration>
+ <phase>package</phase>
+ <goals>
+ <goal>assemble</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.2-beta-5</version>
+ <executions>
+ <execution>
+ <configuration>
+ <descriptors>
+ <descriptor>src/main/assembly/binary-assembly.xml</descriptor>
+ </descriptors>
+ </configuration>
+ <phase>package</phase>
+ <goals>
+ <goal>attached</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>args4j</groupId>
+ <artifactId>args4j</artifactId>
+ <version>2.0.16</version>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-yarn-common</artifactId>
+ <version>0.2.1-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-yarn-am</artifactId>
+ <version>0.2.1-SNAPSHOT</version>
+ <type>zip</type>
+ <classifier>binary-assembly</classifier>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-client/src/main/assembly/binary-assembly.xml b/hyracks/hyracks-yarn/hyracks-yarn-client/src/main/assembly/binary-assembly.xml
new file mode 100644
index 0000000..b4fa339
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-client/src/main/assembly/binary-assembly.xml
@@ -0,0 +1,30 @@
+<assembly>
+ <id>binary-assembly</id>
+ <formats>
+ <format>zip</format>
+ <format>dir</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>target/appassembler/bin</directory>
+ <outputDirectory>bin</outputDirectory>
+ <fileMode>0755</fileMode>
+ </fileSet>
+ <fileSet>
+ <directory>target/appassembler/lib</directory>
+ <outputDirectory>lib</outputDirectory>
+ </fileSet>
+ </fileSets>
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>hyracks-yarn-am</outputDirectory>
+ <includes>
+ <include>hyracks-yarn-am*</include>
+ </includes>
+ <unpack>false</unpack>
+ <outputFileNameMapping>${artifact.artifactId}.${artifact.extension}</outputFileNameMapping>
+ <useTransitiveDependencies>false</useTransitiveDependencies>
+ </dependencySet>
+ </dependencySets>
+</assembly>
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-client/src/main/java/edu/uci/ics/hyracks/yarn/client/KillHyracksApplication.java b/hyracks/hyracks-yarn/hyracks-yarn-client/src/main/java/edu/uci/ics/hyracks/yarn/client/KillHyracksApplication.java
new file mode 100644
index 0000000..4789160
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-client/src/main/java/edu/uci/ics/hyracks/yarn/client/KillHyracksApplication.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.yarn.client;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.kohsuke.args4j.CmdLineParser;
+import org.kohsuke.args4j.Option;
+
+import edu.uci.ics.hyracks.yarn.common.protocols.clientrm.YarnClientRMConnection;
+
+public class KillHyracksApplication {
+ private final Options options;
+
+ private KillHyracksApplication(Options options) {
+ this.options = options;
+ }
+
+ private void run() throws Exception {
+ Configuration conf = new Configuration();
+ YarnConfiguration yconf = new YarnConfiguration(conf);
+ YarnClientRMConnection crmc = new YarnClientRMConnection(yconf);
+ crmc.killApplication(options.appId);
+ }
+
+ public static void main(String[] args) throws Exception {
+ Options options = new Options();
+ CmdLineParser parser = new CmdLineParser(options);
+ try {
+ parser.parseArgument(args);
+ } catch (Exception e) {
+ parser.printUsage(System.err);
+ return;
+ }
+ new KillHyracksApplication(options).run();
+ }
+
+ private static class Options {
+ @Option(name = "-application-id", required = true, usage = "Application Id")
+ String appId;
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-client/src/main/java/edu/uci/ics/hyracks/yarn/client/LaunchHyracksApplication.java b/hyracks/hyracks-yarn/hyracks-yarn-client/src/main/java/edu/uci/ics/hyracks/yarn/client/LaunchHyracksApplication.java
new file mode 100644
index 0000000..e12891e
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-client/src/main/java/edu/uci/ics/hyracks/yarn/client/LaunchHyracksApplication.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.yarn.client;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.kohsuke.args4j.CmdLineParser;
+import org.kohsuke.args4j.Option;
+
+import edu.uci.ics.hyracks.yarn.common.protocols.clientrm.YarnApplication;
+import edu.uci.ics.hyracks.yarn.common.protocols.clientrm.YarnClientRMConnection;
+import edu.uci.ics.hyracks.yarn.common.resources.LocalResourceHelper;
+import edu.uci.ics.hyracks.yarn.common.resources.ResourceHelper;
+
+public class LaunchHyracksApplication {
+ private final Options options;
+
+ private LaunchHyracksApplication(Options options) {
+ this.options = options;
+ }
+
+ private void run() throws Exception {
+ Configuration conf = new Configuration();
+ YarnConfiguration yconf = new YarnConfiguration(conf);
+ YarnClientRMConnection crmc = new YarnClientRMConnection(yconf);
+
+ YarnApplication app = crmc.createApplication(options.appName);
+
+ ContainerLaunchContext clCtx = app.getContainerLaunchContext();
+
+ Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
+
+ File amZipFile = new File(System.getProperty("basedir") + "/hyracks-yarn-am/hyracks-yarn-am.zip");
+ localResources.put("archive", LocalResourceHelper.createArchiveResource(conf, amZipFile));
+ localResources.put("manifest.xml", LocalResourceHelper.createFileResource(conf, options.hcManifest));
+ clCtx.setLocalResources(localResources);
+
+ String command = "./archive/bin/hyracks-yarn-am 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"
+ + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr";
+
+ List<String> commands = new ArrayList<String>();
+ commands.add(command);
+ clCtx.setCommands(commands);
+
+ clCtx.setResource(ResourceHelper.createMemoryCapability(options.amMemory));
+
+ app.submit();
+ }
+
+ public static void main(String[] args) throws Exception {
+ Options options = new Options();
+ CmdLineParser parser = new CmdLineParser(options);
+ try {
+ parser.parseArgument(args);
+ } catch (Exception e) {
+ parser.printUsage(System.err);
+ return;
+ }
+ new LaunchHyracksApplication(options).run();
+ }
+
+ private static class Options {
+ @Option(name = "-application-name", required = true, usage = "Application Name")
+ String appName;
+
+ @Option(name = "-am-memory", required = false, usage = "Application Master memory requirements")
+ int amMemory = 128;
+
+ @Option(name = "-hyracks-cluster-manifest", required = true, usage = "Hyracks Cluster Manifest file")
+ File hcManifest;
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-common/pom.xml b/hyracks/hyracks-yarn/hyracks-yarn-common/pom.xml
new file mode 100644
index 0000000..3aaf4a2
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-common/pom.xml
@@ -0,0 +1,40 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hyracks-yarn-common</artifactId>
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-yarn</artifactId>
+ <version>0.2.1-SNAPSHOT</version>
+ </parent>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.0.2</version>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <version>2.0.0-alpha</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <version>2.0.0-alpha</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>2.0.0-alpha</version>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/protocols/amrm/AMRMConnection.java b/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/protocols/amrm/AMRMConnection.java
new file mode 100644
index 0000000..7181bc3
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/protocols/amrm/AMRMConnection.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.yarn.common.protocols.amrm;
+
+import java.net.InetSocketAddress;
+import java.util.Map;
+
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.yarn.api.AMRMProtocol;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+
+public class AMRMConnection {
+ private final YarnConfiguration config;
+
+ private final ApplicationAttemptId appAttemptId;
+
+ private final AMRMProtocol amrmp;
+
+ public AMRMConnection(YarnConfiguration config) {
+ this.config = config;
+ Map<String, String> envs = System.getenv();
+ String containerIdString = envs.get(ApplicationConstants.AM_CONTAINER_ID_ENV);
+ if (containerIdString == null) {
+ throw new IllegalArgumentException("ContainerId not set in the environment");
+ }
+ ContainerId containerId = ConverterUtils.toContainerId(containerIdString);
+ appAttemptId = containerId.getApplicationAttemptId();
+ InetSocketAddress rmAddress = NetUtils.createSocketAddr(config.get(YarnConfiguration.RM_SCHEDULER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS));
+ YarnRPC rpc = YarnRPC.create(config);
+
+ amrmp = (AMRMProtocol) rpc.getProxy(AMRMProtocol.class, rmAddress, config);
+ }
+
+ public ApplicationAttemptId getApplicationAttemptId() {
+ return appAttemptId;
+ }
+
+ public AMRMProtocol getAMRMProtocol() {
+ return amrmp;
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/protocols/clientrm/YarnApplication.java b/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/protocols/clientrm/YarnApplication.java
new file mode 100644
index 0000000..ca099aa
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/protocols/clientrm/YarnApplication.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.yarn.common.protocols.clientrm;
+
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.util.Records;
+
+public class YarnApplication {
+ private final YarnClientRMConnection crmc;
+
+ private ApplicationSubmissionContext appCtx;
+
+ private ContainerLaunchContext clCtx;
+
+ YarnApplication(YarnClientRMConnection crmc, String appName) throws YarnRemoteException {
+ this.crmc = crmc;
+ appCtx = Records.newRecord(ApplicationSubmissionContext.class);
+ appCtx.setApplicationId(getNewApplicationId(crmc));
+ appCtx.setApplicationName(appName);
+ clCtx = Records.newRecord(ContainerLaunchContext.class);
+ }
+
+ public ContainerLaunchContext getContainerLaunchContext() {
+ return clCtx;
+ }
+
+ public void submit() throws YarnRemoteException {
+ appCtx.setAMContainerSpec(clCtx);
+ SubmitApplicationRequest appRequest = Records.newRecord(SubmitApplicationRequest.class);
+ appRequest.setApplicationSubmissionContext(appCtx);
+ crmc.getClientRMProtocol().submitApplication(appRequest);
+ }
+
+ private static ApplicationId getNewApplicationId(YarnClientRMConnection crmc) throws YarnRemoteException {
+ GetNewApplicationRequest request = Records.newRecord(GetNewApplicationRequest.class);
+ GetNewApplicationResponse response = crmc.getClientRMProtocol().getNewApplication(request);
+
+ return response.getApplicationId();
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/protocols/clientrm/YarnClientRMConnection.java b/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/protocols/clientrm/YarnClientRMConnection.java
new file mode 100644
index 0000000..bb42fab
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/protocols/clientrm/YarnClientRMConnection.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.yarn.common.protocols.clientrm;
+
+import java.net.InetSocketAddress;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.SecurityInfo;
+import org.apache.hadoop.yarn.api.ClientRMProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.security.client.ClientRMSecurityInfo;
+import org.apache.hadoop.yarn.util.Records;
+
+public class YarnClientRMConnection {
+ private final YarnConfiguration config;
+
+ private final ClientRMProtocol crmp;
+
+ public YarnClientRMConnection(YarnConfiguration config) {
+ this.config = config;
+ InetSocketAddress remoteAddress = NetUtils.createSocketAddr(config.get(YarnConfiguration.RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_ADDRESS));
+ Configuration appsManagerServerConf = new Configuration(config);
+ appsManagerServerConf.setClass(YarnConfiguration.YARN_SECURITY_SERVICE_AUTHORIZATION_CLIENT_RESOURCEMANAGER,
+ ClientRMSecurityInfo.class, SecurityInfo.class);
+ YarnRPC rpc = YarnRPC.create(appsManagerServerConf);
+ crmp = ((ClientRMProtocol) rpc.getProxy(ClientRMProtocol.class, remoteAddress, appsManagerServerConf));
+ }
+
+ public YarnApplication createApplication(String appName) throws YarnRemoteException {
+ return new YarnApplication(this, appName);
+ }
+
+ public ClientRMProtocol getClientRMProtocol() {
+ return crmp;
+ }
+
+ public void killApplication(String appId) throws Exception {
+ KillApplicationRequest killRequest = Records.newRecord(KillApplicationRequest.class);
+ ApplicationId aid = Records.newRecord(ApplicationId.class);
+ long ts = Long.parseLong(appId.substring(appId.indexOf('_') + 1, appId.lastIndexOf('_')));
+ aid.setClusterTimestamp(ts);
+ int id = Integer.parseInt(appId.substring(appId.lastIndexOf('_') + 1));
+ aid.setId(id);
+ killRequest.setApplicationId(aid);
+ crmp.forceKillApplication(killRequest);
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/resources/LocalResourceHelper.java b/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/resources/LocalResourceHelper.java
new file mode 100644
index 0000000..b30c4a1
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/resources/LocalResourceHelper.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.yarn.common.resources;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.URL;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.util.Records;
+
+public class LocalResourceHelper {
+ private static LocalResource createLocalResourceFromPath(Configuration config, File path) throws IOException {
+ LocalResource lr = Records.newRecord(LocalResource.class);
+ URL url = ConverterUtils.getYarnUrlFromPath(FileContext.getFileContext().makeQualified(new Path(path.toURI())));
+ lr.setResource(url);
+ lr.setVisibility(LocalResourceVisibility.APPLICATION);
+ lr.setTimestamp(path.lastModified());
+ lr.setSize(path.length());
+ return lr;
+ }
+
+ public static LocalResource createFileResource(Configuration config, File path) throws IOException {
+ LocalResource lr = createLocalResourceFromPath(config, path);
+ lr.setType(LocalResourceType.FILE);
+ return lr;
+ }
+
+ public static LocalResource createArchiveResource(Configuration config, File path) throws IOException {
+ LocalResource lr = createLocalResourceFromPath(config, path);
+ lr.setType(LocalResourceType.ARCHIVE);
+ return lr;
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/resources/ResourceHelper.java b/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/resources/ResourceHelper.java
new file mode 100644
index 0000000..20267b7
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/resources/ResourceHelper.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.yarn.common.resources;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.Records;
+
+public class ResourceHelper {
+ public static Resource createMemoryCapability(int memory) {
+ Resource capability = Records.newRecord(Resource.class);
+ capability.setMemory(memory);
+ return capability;
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-yarn/pom.xml b/hyracks/hyracks-yarn/pom.xml
new file mode 100644
index 0000000..af4b0b1
--- /dev/null
+++ b/hyracks/hyracks-yarn/pom.xml
@@ -0,0 +1,17 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hyracks-yarn</artifactId>
+ <packaging>pom</packaging>
+
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks</artifactId>
+ <version>0.2.1-SNAPSHOT</version>
+ </parent>
+
+ <modules>
+ <module>hyracks-yarn-common</module>
+ <module>hyracks-yarn-client</module>
+ <module>hyracks-yarn-am</module>
+ </modules>
+</project>