Merged fullstack_lsm_staging upto r3336
git-svn-id: https://hyracks.googlecode.com/svn/trunk/fullstack@3339 123451ca-8445-de46-9d55-352943316053
diff --git a/hivesterix-dist/src/main/java/edu/uci/ics/hivesterix/runtime/exec/HyracksExecutionEngine.java b/hivesterix-dist/src/main/java/edu/uci/ics/hivesterix/runtime/exec/HyracksExecutionEngine.java
index 379737f..53c9d42 100644
--- a/hivesterix-dist/src/main/java/edu/uci/ics/hivesterix/runtime/exec/HyracksExecutionEngine.java
+++ b/hivesterix-dist/src/main/java/edu/uci/ics/hivesterix/runtime/exec/HyracksExecutionEngine.java
@@ -258,7 +258,7 @@
builder.setTypeTraitProvider(HiveTypeTraitProvider.INSTANCE);
builder.setHashFunctionFamilyProvider(HiveBinaryHashFunctionFamilyProvider.INSTANCE);
- jobSpec = compiler.createJob(null);
+ jobSpec = compiler.createJob(null, null);
// set the policy
String policyStr = conf.get("hive.hyracks.connectorpolicy");
@@ -558,9 +558,8 @@
}
}
- String applicationName = "hivesterix";
long start = System.currentTimeMillis();
- JobId jobId = hcc.startJob(applicationName, job);
+ JobId jobId = hcc.startJob(job);
hcc.waitForCompletion(jobId);
// System.out.println("job finished: " + jobId.toString());
diff --git a/hivesterix-dist/src/test/java/edu/uci/ics/hivesterix/perf/base/AbstractPerfTestSuiteClass.java b/hivesterix-dist/src/test/java/edu/uci/ics/hivesterix/perf/base/AbstractPerfTestSuiteClass.java
index c882742..393378f 100644
--- a/hivesterix-dist/src/test/java/edu/uci/ics/hivesterix/perf/base/AbstractPerfTestSuiteClass.java
+++ b/hivesterix-dist/src/test/java/edu/uci/ics/hivesterix/perf/base/AbstractPerfTestSuiteClass.java
@@ -22,8 +22,6 @@
import org.apache.hadoop.mapred.MiniMRCluster;
import edu.uci.ics.hivesterix.common.config.ConfUtil;
-import edu.uci.ics.hyracks.api.client.HyracksConnection;
-import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
import edu.uci.ics.hyracks.control.common.controllers.CCConfig;
import edu.uci.ics.hyracks.control.common.controllers.NCConfig;
@@ -93,7 +91,6 @@
String ipAddress = hconf.get("hive.hyracks.host");
int clientPort = Integer.parseInt(hconf.get("hive.hyracks.port"));
int clusterPort = clientPort;
- String applicationName = hconf.get("hive.hyracks.app");
// start hyracks cc
CCConfig ccConfig = new CCConfig();
@@ -119,9 +116,6 @@
nc.start();
ncs.put(ncConfig.nodeId, nc);
}
-
- IHyracksClientConnection hcc = new HyracksConnection(ccConfig.clientNetIpAddress, clientPort);
- hcc.createApplication(applicationName, null);
}
protected void makeDir(String path) throws IOException {
diff --git a/hivesterix-dist/src/test/java/edu/uci/ics/hivesterix/test/base/AbstractTestSuiteClass.java b/hivesterix-dist/src/test/java/edu/uci/ics/hivesterix/test/base/AbstractTestSuiteClass.java
index 09b632a..5e2e64e 100644
--- a/hivesterix-dist/src/test/java/edu/uci/ics/hivesterix/test/base/AbstractTestSuiteClass.java
+++ b/hivesterix-dist/src/test/java/edu/uci/ics/hivesterix/test/base/AbstractTestSuiteClass.java
@@ -27,8 +27,6 @@
import org.apache.hadoop.mapred.JobConf;
import edu.uci.ics.hivesterix.common.config.ConfUtil;
-import edu.uci.ics.hyracks.api.client.HyracksConnection;
-import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
import edu.uci.ics.hyracks.control.common.controllers.CCConfig;
import edu.uci.ics.hyracks.control.common.controllers.NCConfig;
@@ -116,7 +114,6 @@
}
int clientPort = Integer.parseInt(clusterProps.getProperty("CC_CLIENTPORT"));
int netPort = Integer.parseInt(clusterProps.getProperty("CC_CLUSTERPORT"));
- String applicationName = "hivesterix";
// start hyracks cc
CCConfig ccConfig = new CCConfig();
@@ -143,9 +140,6 @@
nc.start();
ncs.put(ncConfig.nodeId, nc);
}
-
- IHyracksClientConnection hcc = new HyracksConnection(ccConfig.clientNetIpAddress, clientPort);
- hcc.createApplication(applicationName, null);
}
protected void makeDir(String path) throws IOException {
diff --git a/hivesterix-runtime/src/main/java/edu/uci/ics/hivesterix/runtime/jobgen/HiveMetaDataProvider.java b/hivesterix-runtime/src/main/java/edu/uci/ics/hivesterix/runtime/jobgen/HiveMetaDataProvider.java
index daf6a7f..d7a1d7c 100644
--- a/hivesterix-runtime/src/main/java/edu/uci/ics/hivesterix/runtime/jobgen/HiveMetaDataProvider.java
+++ b/hivesterix-runtime/src/main/java/edu/uci/ics/hivesterix/runtime/jobgen/HiveMetaDataProvider.java
@@ -8,8 +8,6 @@
import org.apache.hadoop.hive.ql.plan.PartitionDesc;
import edu.uci.ics.hivesterix.logical.expression.HiveFunctionInfo;
-import edu.uci.ics.hivesterix.runtime.jobgen.HiveScanRuntimeGenerator;
-import edu.uci.ics.hivesterix.runtime.jobgen.HiveWriteRuntimeGenerator;
import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
@@ -61,8 +59,8 @@
@Override
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getScannerRuntime(IDataSource<S> dataSource,
List<LogicalVariable> scanVariables, List<LogicalVariable> projectVariables, boolean projectPushed,
- IOperatorSchema opSchema, IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec)
- throws AlgebricksException {
+ IOperatorSchema opSchema, IVariableTypeEnvironment typeEnv, JobGenContext context,
+ JobSpecification jobSpec, Object implConfig) throws AlgebricksException {
S desc = dataSource.getId();
HiveScanRuntimeGenerator generator = new HiveScanRuntimeGenerator((PartitionDesc) desc);
@@ -79,17 +77,19 @@
}
@Override
- public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getDeleteRuntime(IDataSource<S> arg0,
- IOperatorSchema arg1, List<LogicalVariable> arg2, LogicalVariable arg3, RecordDescriptor arg4,
- JobGenContext arg5, JobSpecification arg6) throws AlgebricksException {
+ public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getInsertRuntime(IDataSource<S> dataSource,
+ IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv, List<LogicalVariable> keys,
+ LogicalVariable payLoadVar, RecordDescriptor recordDesc, JobGenContext context, JobSpecification jobSpec)
+ throws AlgebricksException {
// TODO Auto-generated method stub
return null;
}
@Override
- public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getInsertRuntime(IDataSource<S> arg0,
- IOperatorSchema arg1, List<LogicalVariable> arg2, LogicalVariable arg3, RecordDescriptor arg4,
- JobGenContext arg5, JobSpecification arg6) throws AlgebricksException {
+ public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getDeleteRuntime(IDataSource<S> dataSource,
+ IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv, List<LogicalVariable> keys,
+ LogicalVariable payLoadVar, RecordDescriptor recordDesc, JobGenContext context, JobSpecification jobSpec)
+ throws AlgebricksException {
// TODO Auto-generated method stub
return null;
}
@@ -133,5 +133,4 @@
// TODO Auto-generated method stub
return null;
}
-
}