Copied hyracks trunk into fullstack
git-svn-id: https://hyracks.googlecode.com/svn/branches/fullstack_staging@1958 123451ca-8445-de46-9d55-352943316053
diff --git a/hyracks/hyracks-server/docs/README b/hyracks/hyracks-server/docs/README
new file mode 100644
index 0000000..04c0ed2
--- /dev/null
+++ b/hyracks/hyracks-server/docs/README
@@ -0,0 +1,91 @@
+Hyracks is a data-parallel platform that allows users to run jobs on a cluster of shared-nothing computers.
+
+QUICKSTART
+__________
+
+Hyracks is made up of two parts that need to run on a cluster to accept and run users' jobs. The Hyracks Cluster Controller must be run on one machine designated as the
+master node. This machine should be able to be accessed from the other nodes in the cluster that would do work as well as the client machines that would submit jobs to Hyracks.
+The worker nodes (machines) must run a Hyracks Node Controller.
+
+1. Starting the Hyracks Cluster Controller
+
+The simplest way to start the cluster controller is to run bin/hyrackscc.
+By default, the cluster controller listens on port 1099 for connections from Node Controllers. However, it can be made to listen on a different port by passing an optional
+parameter as shown below:
+
+bin/hyrackscc -port <port>
+
+2. Starting the Hyracks Node Controller
+
+The node controller is started by running bin/hyracksnc. It requires at least the following two command line arguments.
+
+ -cc-host VAL : Cluster Controller host name
+ -data-ip-address VAL : IP Address to bind data listener
+
+If the cluster controller was directed to listen on a port other than the default, you will need to pass one more argument to hyracksnc.
+
+ -cc-port N : Cluster Controller port (default: 1099)
+
+The data-ip-address is the interface on which the Node Controller must listen on -- in the event the machine is multi-homed it must listen on an IP that is reachable from
+other Node Controllers. Make sure that the value passed to the data-ip-address is a valid IPv4 address (four octets separated by .).
+
+3. Running a job on Hyracks
+
+There are a few examples in the source distribution under src/test/integration that outline the construction and issue of a Job to the Hyracks Cluster Controller. The
+basic steps that need to be performed on the client to execute jobs are:
+
+Registry registry = LocateRegistry.getRegistry(ccHost, ccPort);
+IClusterController cc = registry.lookup(IClusterController.class.getName()); // Get a handle to the Cluster Controller
+
+JobSpecification spec = createJob(); // User code to create a Job
+
+UUID jobId = cc.createJob(spec); // Install the Job on the Cluster Controller
+
+cc.start(jobId); // Start the job
+cc.waitForCompletion(jobId); // Jobs run asynchronously -- Wait for this one to complete
+
+
+
+BUILDING FROM SOURCE
+____________________
+
+Prerequisites:
+
+1. JDK 1.6
+2. Maven2 (maven.apache.org)
+
+Steps:
+
+1. Checkout source from SVN
+2. Download dcache-client-0.0.1.jar from the Downloads page at http://code.google.com/p/hyracks/downloads/list
+3. Run the following command: (Replacing /path/to/file with the path to the dcache-client jar file).
+
+ mvn install:install-file -DgroupId=edu.uci.ics.dcache -DartifactId=dcache-client -Dversion=0.0.1 -Dpackaging=jar -Dfile=/path/to/file/dcache-client-0.0.1.jar
+
+4. Download jol.jar from the Downloads page at http://code.google.com/p/hyracks/downloads/list
+5. Run the following command: (Replacing /path/to/file with the path to the jol jar file).
+
+ mvn install:install-file -DgroupId=jol -DartifactId=jol -Dversion=0.0.1 -Dpackaging=jar -Dfile=/path/to/file/jol.jar
+
+6. cd into the hyracks folder where the source was checked out
+7. Run
+
+ mvn package
+
+6. That's it!
+
+
+IMPORTING INTO ECLIPSE
+______________________
+
+Prerequisites:
+
+You will need to have the Eclipse Maven plugin (http://m2eclipse.sonatype.org/)
+
+1. Open eclipse
+2. Right click in the Package Explorer pane.
+3. Click on Import...
+4. Under "General" choose "Existing Projects into Workspace"
+5. Pick the project root option and browse to the hyracks folder
+6. Import all projects.
+7. Click on Finish
diff --git a/hyracks/hyracks-server/pom.xml b/hyracks/hyracks-server/pom.xml
new file mode 100644
index 0000000..d777a91
--- /dev/null
+++ b/hyracks/hyracks-server/pom.xml
@@ -0,0 +1,86 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hyracks-server</artifactId>
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks</artifactId>
+ <version>0.2.2-SNAPSHOT</version>
+ </parent>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.0.2</version>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>appassembler-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <configuration>
+ <programs>
+ <program>
+ <mainClass>edu.uci.ics.hyracks.control.cc.CCDriver</mainClass>
+ <name>hyrackscc</name>
+ </program>
+ <program>
+ <mainClass>edu.uci.ics.hyracks.control.nc.NCDriver</mainClass>
+ <name>hyracksnc</name>
+ </program>
+ <program>
+ <mainClass>edu.uci.ics.hyracks.server.drivers.VirtualClusterDriver</mainClass>
+ <name>hyracks-virtual-cluster</name>
+ </program>
+ </programs>
+ <repositoryLayout>flat</repositoryLayout>
+ <repositoryName>lib</repositoryName>
+ </configuration>
+ <phase>package</phase>
+ <goals>
+ <goal>assemble</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.2-beta-5</version>
+ <executions>
+ <execution>
+ <configuration>
+ <descriptors>
+ <descriptor>src/main/assembly/binary-assembly.xml</descriptor>
+ </descriptors>
+ </configuration>
+ <phase>package</phase>
+ <goals>
+ <goal>attached</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-control-cc</artifactId>
+ <version>0.2.2-SNAPSHOT</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-control-nc</artifactId>
+ <version>0.2.2-SNAPSHOT</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hyracks/hyracks-server/src/main/assembly/binary-assembly.xml b/hyracks/hyracks-server/src/main/assembly/binary-assembly.xml
new file mode 100644
index 0000000..cd598d9
--- /dev/null
+++ b/hyracks/hyracks-server/src/main/assembly/binary-assembly.xml
@@ -0,0 +1,23 @@
+<assembly>
+ <id>binary-assembly</id>
+ <formats>
+ <format>zip</format>
+ <format>dir</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>target/appassembler/bin</directory>
+ <outputDirectory>bin</outputDirectory>
+ <fileMode>0755</fileMode>
+ </fileSet>
+ <fileSet>
+ <directory>target/appassembler/lib</directory>
+ <outputDirectory>lib</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>docs</directory>
+ <outputDirectory>docs</outputDirectory>
+ </fileSet>
+ </fileSets>
+</assembly>
diff --git a/hyracks/hyracks-server/src/main/java/edu/uci/ics/hyracks/server/drivers/VirtualClusterDriver.java b/hyracks/hyracks-server/src/main/java/edu/uci/ics/hyracks/server/drivers/VirtualClusterDriver.java
new file mode 100644
index 0000000..8833fed
--- /dev/null
+++ b/hyracks/hyracks-server/src/main/java/edu/uci/ics/hyracks/server/drivers/VirtualClusterDriver.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.server.drivers;
+
+import org.kohsuke.args4j.CmdLineParser;
+import org.kohsuke.args4j.Option;
+
+import edu.uci.ics.hyracks.control.common.controllers.CCConfig;
+import edu.uci.ics.hyracks.control.common.controllers.NCConfig;
+import edu.uci.ics.hyracks.server.process.HyracksCCProcess;
+import edu.uci.ics.hyracks.server.process.HyracksNCProcess;
+
+public class VirtualClusterDriver {
+ private static class Options {
+ @Option(name = "-n", required = false, usage = "Number of node controllers (default: 2)")
+ public int n = 2;
+
+ @Option(name = "-cc-client-net-port", required = false, usage = "CC Port (default: 1098)")
+ public int ccClientNetPort = 1098;
+
+ @Option(name = "-cc-cluster-net-port", required = false, usage = "CC Port (default: 1099)")
+ public int ccClusterNetPort = 1099;
+
+ @Option(name = "-cc-http-port", required = false, usage = "CC Port (default: 16001)")
+ public int ccHttpPort = 16001;
+ }
+
+ public static void main(String[] args) throws Exception {
+ Options options = new Options();
+ CmdLineParser cp = new CmdLineParser(options);
+ try {
+ cp.parseArgument(args);
+ } catch (Exception e) {
+ System.err.println(e.getMessage());
+ cp.printUsage(System.err);
+ return;
+ }
+
+ CCConfig ccConfig = new CCConfig();
+ ccConfig.clusterNetIpAddress = "127.0.0.1";
+ ccConfig.clusterNetPort = options.ccClusterNetPort;
+ ccConfig.clientNetIpAddress = "127.0.0.1";
+ ccConfig.clientNetPort = options.ccClientNetPort;
+ ccConfig.httpPort = options.ccHttpPort;
+ HyracksCCProcess ccp = new HyracksCCProcess(ccConfig);
+ ccp.start();
+
+ Thread.sleep(5000);
+
+ HyracksNCProcess ncps[] = new HyracksNCProcess[options.n];
+ for (int i = 0; i < options.n; ++i) {
+ NCConfig ncConfig = new NCConfig();
+ ncConfig.ccHost = "127.0.0.1";
+ ncConfig.ccPort = options.ccClusterNetPort;
+ ncConfig.clusterNetIPAddress = "127.0.0.1";
+ ncConfig.nodeId = "nc" + i;
+ ncConfig.dataIPAddress = "127.0.0.1";
+ ncps[i] = new HyracksNCProcess(ncConfig);
+ ncps[i].start();
+ }
+
+ while (true) {
+ Thread.sleep(10000);
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-server/src/main/java/edu/uci/ics/hyracks/server/process/HyracksCCProcess.java b/hyracks/hyracks-server/src/main/java/edu/uci/ics/hyracks/server/process/HyracksCCProcess.java
new file mode 100644
index 0000000..cb1be95
--- /dev/null
+++ b/hyracks/hyracks-server/src/main/java/edu/uci/ics/hyracks/server/process/HyracksCCProcess.java
@@ -0,0 +1,24 @@
+package edu.uci.ics.hyracks.server.process;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.control.cc.CCDriver;
+import edu.uci.ics.hyracks.control.common.controllers.CCConfig;
+
+public class HyracksCCProcess extends HyracksServerProcess {
+ private CCConfig config;
+
+ public HyracksCCProcess(CCConfig config) {
+ this.config = config;
+ }
+
+ @Override
+ protected void addCmdLineArgs(List<String> cList) {
+ config.toCommandLine(cList);
+ }
+
+ @Override
+ protected String getMainClassName() {
+ return CCDriver.class.getName();
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-server/src/main/java/edu/uci/ics/hyracks/server/process/HyracksNCProcess.java b/hyracks/hyracks-server/src/main/java/edu/uci/ics/hyracks/server/process/HyracksNCProcess.java
new file mode 100644
index 0000000..cd110c8
--- /dev/null
+++ b/hyracks/hyracks-server/src/main/java/edu/uci/ics/hyracks/server/process/HyracksNCProcess.java
@@ -0,0 +1,24 @@
+package edu.uci.ics.hyracks.server.process;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.control.common.controllers.NCConfig;
+import edu.uci.ics.hyracks.control.nc.NCDriver;
+
+public class HyracksNCProcess extends HyracksServerProcess {
+ private NCConfig config;
+
+ public HyracksNCProcess(NCConfig config) {
+ this.config = config;
+ }
+
+ @Override
+ protected void addCmdLineArgs(List<String> cList) {
+ config.toCommandLine(cList);
+ }
+
+ @Override
+ protected String getMainClassName() {
+ return NCDriver.class.getName();
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-server/src/main/java/edu/uci/ics/hyracks/server/process/HyracksServerProcess.java b/hyracks/hyracks-server/src/main/java/edu/uci/ics/hyracks/server/process/HyracksServerProcess.java
new file mode 100644
index 0000000..6a0e45f
--- /dev/null
+++ b/hyracks/hyracks-server/src/main/java/edu/uci/ics/hyracks/server/process/HyracksServerProcess.java
@@ -0,0 +1,85 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.server.process;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+public abstract class HyracksServerProcess {
+ private static final Logger LOGGER = Logger.getLogger(HyracksServerProcess.class.getName());
+
+ protected Process process;
+
+ public void start() throws IOException {
+ String[] cmd = buildCommand();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Starting command: " + Arrays.toString(cmd));
+ }
+ process = Runtime.getRuntime().exec(cmd, null, null);
+ dump(process.getInputStream());
+ dump(process.getErrorStream());
+ }
+
+ private void dump(InputStream input) {
+ final int streamBufferSize = 1000;
+ final Reader in = new InputStreamReader(input);
+ new Thread(new Runnable() {
+ public void run() {
+ try {
+ char[] chars = new char[streamBufferSize];
+ int c;
+ while ((c = in.read(chars)) != -1) {
+ if (c > 0) {
+ System.out.print(String.valueOf(chars, 0, c));
+ }
+ }
+ } catch (IOException e) {
+ }
+ }
+ }).start();
+ }
+
+ private String[] buildCommand() {
+ List<String> cList = new ArrayList<String>();
+ cList.add(getJavaCommand());
+ cList.add("-Dbasedir=" + System.getProperty("basedir"));
+ cList.add("-Djava.rmi.server.hostname=127.0.0.1");
+ cList.add("-classpath");
+ cList.add(getClasspath());
+ cList.add(getMainClassName());
+ addCmdLineArgs(cList);
+ return cList.toArray(new String[cList.size()]);
+ }
+
+ protected abstract void addCmdLineArgs(List<String> cList);
+
+ protected abstract String getMainClassName();
+
+ private final String getClasspath() {
+ return System.getProperty("java.class.path");
+ }
+
+ protected final String getJavaCommand() {
+ return System.getProperty("java.home") + File.separator + "bin" + File.separator + "java";
+ }
+}
\ No newline at end of file