Merged hyracks_asterix_stabilization upto rev 1913
git-svn-id: https://hyracks.googlecode.com/svn/trunk@1924 123451ca-8445-de46-9d55-352943316053
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-common/pom.xml b/hyracks/hyracks-yarn/hyracks-yarn-common/pom.xml
new file mode 100644
index 0000000..3aaf4a2
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-common/pom.xml
@@ -0,0 +1,40 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hyracks-yarn-common</artifactId>
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-yarn</artifactId>
+ <version>0.2.1-SNAPSHOT</version>
+ </parent>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.0.2</version>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <version>2.0.0-alpha</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <version>2.0.0-alpha</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>2.0.0-alpha</version>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/protocols/amrm/AMRMConnection.java b/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/protocols/amrm/AMRMConnection.java
new file mode 100644
index 0000000..7181bc3
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/protocols/amrm/AMRMConnection.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.yarn.common.protocols.amrm;
+
+import java.net.InetSocketAddress;
+import java.util.Map;
+
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.yarn.api.AMRMProtocol;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+
+public class AMRMConnection {
+ private final YarnConfiguration config;
+
+ private final ApplicationAttemptId appAttemptId;
+
+ private final AMRMProtocol amrmp;
+
+ public AMRMConnection(YarnConfiguration config) {
+ this.config = config;
+ Map<String, String> envs = System.getenv();
+ String containerIdString = envs.get(ApplicationConstants.AM_CONTAINER_ID_ENV);
+ if (containerIdString == null) {
+ throw new IllegalArgumentException("ContainerId not set in the environment");
+ }
+ ContainerId containerId = ConverterUtils.toContainerId(containerIdString);
+ appAttemptId = containerId.getApplicationAttemptId();
+ InetSocketAddress rmAddress = NetUtils.createSocketAddr(config.get(YarnConfiguration.RM_SCHEDULER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS));
+ YarnRPC rpc = YarnRPC.create(config);
+
+ amrmp = (AMRMProtocol) rpc.getProxy(AMRMProtocol.class, rmAddress, config);
+ }
+
+ public ApplicationAttemptId getApplicationAttemptId() {
+ return appAttemptId;
+ }
+
+ public AMRMProtocol getAMRMProtocol() {
+ return amrmp;
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/protocols/clientrm/YarnApplication.java b/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/protocols/clientrm/YarnApplication.java
new file mode 100644
index 0000000..ca099aa
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/protocols/clientrm/YarnApplication.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.yarn.common.protocols.clientrm;
+
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.util.Records;
+
+public class YarnApplication {
+ private final YarnClientRMConnection crmc;
+
+ private ApplicationSubmissionContext appCtx;
+
+ private ContainerLaunchContext clCtx;
+
+ YarnApplication(YarnClientRMConnection crmc, String appName) throws YarnRemoteException {
+ this.crmc = crmc;
+ appCtx = Records.newRecord(ApplicationSubmissionContext.class);
+ appCtx.setApplicationId(getNewApplicationId(crmc));
+ appCtx.setApplicationName(appName);
+ clCtx = Records.newRecord(ContainerLaunchContext.class);
+ }
+
+ public ContainerLaunchContext getContainerLaunchContext() {
+ return clCtx;
+ }
+
+ public void submit() throws YarnRemoteException {
+ appCtx.setAMContainerSpec(clCtx);
+ SubmitApplicationRequest appRequest = Records.newRecord(SubmitApplicationRequest.class);
+ appRequest.setApplicationSubmissionContext(appCtx);
+ crmc.getClientRMProtocol().submitApplication(appRequest);
+ }
+
+ private static ApplicationId getNewApplicationId(YarnClientRMConnection crmc) throws YarnRemoteException {
+ GetNewApplicationRequest request = Records.newRecord(GetNewApplicationRequest.class);
+ GetNewApplicationResponse response = crmc.getClientRMProtocol().getNewApplication(request);
+
+ return response.getApplicationId();
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/protocols/clientrm/YarnClientRMConnection.java b/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/protocols/clientrm/YarnClientRMConnection.java
new file mode 100644
index 0000000..bb42fab
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/protocols/clientrm/YarnClientRMConnection.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.yarn.common.protocols.clientrm;
+
+import java.net.InetSocketAddress;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.SecurityInfo;
+import org.apache.hadoop.yarn.api.ClientRMProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.security.client.ClientRMSecurityInfo;
+import org.apache.hadoop.yarn.util.Records;
+
+public class YarnClientRMConnection {
+ private final YarnConfiguration config;
+
+ private final ClientRMProtocol crmp;
+
+ public YarnClientRMConnection(YarnConfiguration config) {
+ this.config = config;
+ InetSocketAddress remoteAddress = NetUtils.createSocketAddr(config.get(YarnConfiguration.RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_ADDRESS));
+ Configuration appsManagerServerConf = new Configuration(config);
+ appsManagerServerConf.setClass(YarnConfiguration.YARN_SECURITY_SERVICE_AUTHORIZATION_CLIENT_RESOURCEMANAGER,
+ ClientRMSecurityInfo.class, SecurityInfo.class);
+ YarnRPC rpc = YarnRPC.create(appsManagerServerConf);
+ crmp = ((ClientRMProtocol) rpc.getProxy(ClientRMProtocol.class, remoteAddress, appsManagerServerConf));
+ }
+
+ public YarnApplication createApplication(String appName) throws YarnRemoteException {
+ return new YarnApplication(this, appName);
+ }
+
+ public ClientRMProtocol getClientRMProtocol() {
+ return crmp;
+ }
+
+ public void killApplication(String appId) throws Exception {
+ KillApplicationRequest killRequest = Records.newRecord(KillApplicationRequest.class);
+ ApplicationId aid = Records.newRecord(ApplicationId.class);
+ long ts = Long.parseLong(appId.substring(appId.indexOf('_') + 1, appId.lastIndexOf('_')));
+ aid.setClusterTimestamp(ts);
+ int id = Integer.parseInt(appId.substring(appId.lastIndexOf('_') + 1));
+ aid.setId(id);
+ killRequest.setApplicationId(aid);
+ crmp.forceKillApplication(killRequest);
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/resources/LocalResourceHelper.java b/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/resources/LocalResourceHelper.java
new file mode 100644
index 0000000..b30c4a1
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/resources/LocalResourceHelper.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.yarn.common.resources;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.URL;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.util.Records;
+
+public class LocalResourceHelper {
+ private static LocalResource createLocalResourceFromPath(Configuration config, File path) throws IOException {
+ LocalResource lr = Records.newRecord(LocalResource.class);
+ URL url = ConverterUtils.getYarnUrlFromPath(FileContext.getFileContext().makeQualified(new Path(path.toURI())));
+ lr.setResource(url);
+ lr.setVisibility(LocalResourceVisibility.APPLICATION);
+ lr.setTimestamp(path.lastModified());
+ lr.setSize(path.length());
+ return lr;
+ }
+
+ public static LocalResource createFileResource(Configuration config, File path) throws IOException {
+ LocalResource lr = createLocalResourceFromPath(config, path);
+ lr.setType(LocalResourceType.FILE);
+ return lr;
+ }
+
+ public static LocalResource createArchiveResource(Configuration config, File path) throws IOException {
+ LocalResource lr = createLocalResourceFromPath(config, path);
+ lr.setType(LocalResourceType.ARCHIVE);
+ return lr;
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/resources/ResourceHelper.java b/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/resources/ResourceHelper.java
new file mode 100644
index 0000000..20267b7
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/resources/ResourceHelper.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.yarn.common.resources;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.Records;
+
+public class ResourceHelper {
+ public static Resource createMemoryCapability(int memory) {
+ Resource capability = Records.newRecord(Resource.class);
+ capability.setMemory(memory);
+ return capability;
+ }
+}
\ No newline at end of file