Copied hyracks trunk into fullstack
git-svn-id: https://hyracks.googlecode.com/svn/branches/fullstack_staging@1958 123451ca-8445-de46-9d55-352943316053
diff --git a/hyracks/hyracks-examples/btree-example/btreehelper/pom.xml b/hyracks/hyracks-examples/btree-example/btreehelper/pom.xml
new file mode 100644
index 0000000..7b033be
--- /dev/null
+++ b/hyracks/hyracks-examples/btree-example/btreehelper/pom.xml
@@ -0,0 +1,49 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>edu.uci.ics.hyracks.examples.btree</groupId>
+ <artifactId>btreehelper</artifactId>
+ <parent>
+ <groupId>edu.uci.ics.hyracks.examples</groupId>
+ <artifactId>btree-example</artifactId>
+ <version>0.2.2-SNAPSHOT</version>
+ </parent>
+
+ <dependencies>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-dataflow-std</artifactId>
+ <version>0.2.2-SNAPSHOT</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-storage-am-btree</artifactId>
+ <version>0.2.2-SNAPSHOT</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-api</artifactId>
+ <version>0.2.2-SNAPSHOT</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-data-std</artifactId>
+ <version>0.2.2-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.0.2</version>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/DataGenOperatorDescriptor.java b/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/DataGenOperatorDescriptor.java
new file mode 100644
index 0000000..f342d23
--- /dev/null
+++ b/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/DataGenOperatorDescriptor.java
@@ -0,0 +1,166 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.examples.btree.helper;
+
+import java.io.DataOutput;
+import java.nio.ByteBuffer;
+import java.util.HashSet;
+import java.util.Random;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
+import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
+import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable;
+
+public class DataGenOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor {
+
+ private static final long serialVersionUID = 1L;
+ private final int numRecords;
+
+ private final int intMinVal;
+ private final int intMaxVal;
+ private final int maxStrLen;
+ private final int uniqueField;
+ private final long randomSeed;
+
+ public DataGenOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor outputRecord, int numRecords,
+ int uniqueField, int intMinVal, int intMaxVal, int maxStrLen, long randomSeed) {
+ super(spec, 0, 1);
+ this.numRecords = numRecords;
+ this.uniqueField = uniqueField;
+ this.intMinVal = intMinVal;
+ this.intMaxVal = intMaxVal;
+ this.maxStrLen = maxStrLen;
+ this.randomSeed = randomSeed;
+ recordDescriptors[0] = outputRecord;
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+
+ final ByteBuffer outputFrame = ctx.allocateFrame();
+ final FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
+ final RecordDescriptor recDesc = recordDescriptors[0];
+ final ArrayTupleBuilder tb = new ArrayTupleBuilder(recDesc.getFields().length);
+ final Random rnd = new Random(randomSeed);
+ final int maxUniqueAttempts = 20;
+
+ return new AbstractUnaryOutputSourceOperatorNodePushable() {
+
+ // for quick & dirty exclusion of duplicates
+ // WARNING: could contain numRecord entries and use a lot of memory
+ HashSet<String> stringHs = new HashSet<String>();
+ HashSet<Integer> intHs = new HashSet<Integer>();
+
+ @Override
+ public void initialize() throws HyracksDataException {
+ writer.open();
+ try {
+ appender.reset(outputFrame, true);
+ for (int i = 0; i < numRecords; i++) {
+ tb.reset();
+ for (int j = 0; j < recDesc.getFieldCount(); j++) {
+ genField(tb, j);
+ }
+
+ if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
+ FrameUtils.flushFrame(outputFrame, writer);
+ appender.reset(outputFrame, true);
+ if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
+ throw new IllegalStateException();
+ }
+ }
+ }
+ FrameUtils.flushFrame(outputFrame, writer);
+ } catch (Exception e) {
+ writer.fail();
+ throw new HyracksDataException(e);
+ } finally {
+ writer.close();
+ }
+ }
+
+ private void genField(ArrayTupleBuilder tb, int fieldIndex) throws HyracksDataException {
+ DataOutput dos = tb.getDataOutput();
+ if (recDesc.getFields()[fieldIndex] instanceof IntegerSerializerDeserializer) {
+ int val = -1;
+ if (fieldIndex == uniqueField) {
+ int attempt = 0;
+ while (attempt < maxUniqueAttempts) {
+ int tmp = Math.abs(rnd.nextInt()) % (intMaxVal - intMinVal) + intMinVal;
+ if (intHs.contains(tmp))
+ attempt++;
+ else {
+ val = tmp;
+ intHs.add(val);
+ break;
+ }
+ }
+ if (attempt == maxUniqueAttempts)
+ throw new HyracksDataException("MaxUnique attempts reached in datagen");
+ } else {
+ val = Math.abs(rnd.nextInt()) % (intMaxVal - intMinVal) + intMinVal;
+ }
+ recDesc.getFields()[fieldIndex].serialize(val, dos);
+ tb.addFieldEndOffset();
+ } else if (recDesc.getFields()[fieldIndex] instanceof UTF8StringSerializerDeserializer) {
+ String val = null;
+ if (fieldIndex == uniqueField) {
+ int attempt = 0;
+ while (attempt < maxUniqueAttempts) {
+ String tmp = randomString(maxStrLen, rnd);
+ if (stringHs.contains(tmp))
+ attempt++;
+ else {
+ val = tmp;
+ stringHs.add(val);
+ break;
+ }
+ }
+ if (attempt == maxUniqueAttempts)
+ throw new HyracksDataException("MaxUnique attempts reached in datagen");
+ } else {
+ val = randomString(maxStrLen, rnd);
+ }
+ recDesc.getFields()[fieldIndex].serialize(val, dos);
+ tb.addFieldEndOffset();
+ } else {
+ throw new HyracksDataException(
+ "Type unsupported in data generator. Only integers and strings allowed");
+ }
+ }
+
+ private String randomString(int length, Random random) {
+ String s = Long.toHexString(Double.doubleToLongBits(random.nextDouble()));
+ StringBuilder strBuilder = new StringBuilder();
+ for (int i = 0; i < s.length() && i < length; i++) {
+ strBuilder.append(s.charAt(Math.abs(random.nextInt()) % s.length()));
+ }
+ return strBuilder.toString();
+ }
+ };
+ }
+}
diff --git a/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/IndexRegistryProvider.java b/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/IndexRegistryProvider.java
new file mode 100644
index 0000000..d1f40d8
--- /dev/null
+++ b/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/IndexRegistryProvider.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.examples.btree.helper;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexRegistry;
+
+public class IndexRegistryProvider implements IIndexRegistryProvider<IIndex> {
+ private static final long serialVersionUID = 1L;
+
+ public static final IndexRegistryProvider INSTANCE = new IndexRegistryProvider();
+
+ private IndexRegistryProvider() {
+ }
+
+ @Override
+ public IndexRegistry<IIndex> getRegistry(IHyracksTaskContext ctx) {
+ return RuntimeContext.get(ctx).getIndexRegistry();
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/NCBootstrap.java b/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/NCBootstrap.java
new file mode 100644
index 0000000..ea55e7a
--- /dev/null
+++ b/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/NCBootstrap.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.examples.btree.helper;
+
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.application.INCApplicationContext;
+import edu.uci.ics.hyracks.api.application.INCBootstrap;
+
+public class NCBootstrap implements INCBootstrap {
+ private static final Logger LOGGER = Logger.getLogger(NCBootstrap.class.getName());
+
+ private INCApplicationContext appCtx;
+
+ @Override
+ public void start() throws Exception {
+ LOGGER.info("Starting NC Bootstrap");
+ RuntimeContext rCtx = new RuntimeContext(appCtx);
+ appCtx.setApplicationObject(rCtx);
+ LOGGER.info("Initialized RuntimeContext: " + rCtx);
+ }
+
+ @Override
+ public void stop() throws Exception {
+ LOGGER.info("Stopping Asterix NC Bootstrap");
+ RuntimeContext rCtx = (RuntimeContext) appCtx.getApplicationObject();
+ rCtx.close();
+ }
+
+ @Override
+ public void setApplicationContext(INCApplicationContext appCtx) {
+ this.appCtx = appCtx;
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/RuntimeContext.java b/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/RuntimeContext.java
new file mode 100644
index 0000000..3336d6d
--- /dev/null
+++ b/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/RuntimeContext.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.examples.btree.helper;
+
+import edu.uci.ics.hyracks.api.application.INCApplicationContext;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexRegistry;
+import edu.uci.ics.hyracks.storage.common.buffercache.BufferCache;
+import edu.uci.ics.hyracks.storage.common.buffercache.ClockPageReplacementStrategy;
+import edu.uci.ics.hyracks.storage.common.buffercache.HeapBufferAllocator;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICacheMemoryAllocator;
+import edu.uci.ics.hyracks.storage.common.buffercache.IPageReplacementStrategy;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapManager;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
+import edu.uci.ics.hyracks.storage.common.smi.TransientFileMapManager;
+
+public class RuntimeContext {
+ private IndexRegistry<IIndex> indexRegistry;
+ private IBufferCache bufferCache;
+ private IFileMapManager fileMapManager;
+
+ public RuntimeContext(INCApplicationContext appCtx) {
+ fileMapManager = new TransientFileMapManager();
+ ICacheMemoryAllocator allocator = new HeapBufferAllocator();
+ IPageReplacementStrategy prs = new ClockPageReplacementStrategy();
+ bufferCache = new BufferCache(appCtx.getRootContext().getIOManager(), allocator, prs, fileMapManager, 32768, 50, 100);
+ indexRegistry = new IndexRegistry<IIndex>();
+ }
+
+ public void close() {
+ bufferCache.close();
+ }
+
+ public IBufferCache getBufferCache() {
+ return bufferCache;
+ }
+
+ public IFileMapProvider getFileMapManager() {
+ return fileMapManager;
+ }
+
+ public IndexRegistry<IIndex> getIndexRegistry() {
+ return indexRegistry;
+ }
+
+ public static RuntimeContext get(IHyracksTaskContext ctx) {
+ return (RuntimeContext) ctx.getJobletContext().getApplicationContext().getApplicationObject();
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/StorageManagerInterface.java b/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/StorageManagerInterface.java
new file mode 100644
index 0000000..4c6363b
--- /dev/null
+++ b/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/StorageManagerInterface.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.examples.btree.helper;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
+
+public class StorageManagerInterface implements IStorageManagerInterface {
+ private static final long serialVersionUID = 1L;
+
+ public static final StorageManagerInterface INSTANCE = new StorageManagerInterface();
+
+ private StorageManagerInterface() {
+ }
+
+ @Override
+ public IBufferCache getBufferCache(IHyracksTaskContext ctx) {
+ return RuntimeContext.get(ctx).getBufferCache();
+ }
+
+ @Override
+ public IFileMapProvider getFileMapProvider(IHyracksTaskContext ctx) {
+ return RuntimeContext.get(ctx).getFileMapManager();
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-examples/btree-example/btreehelper/src/main/resources/hyracks-deployment.properties b/hyracks/hyracks-examples/btree-example/btreehelper/src/main/resources/hyracks-deployment.properties
new file mode 100644
index 0000000..ab0ecb3
--- /dev/null
+++ b/hyracks/hyracks-examples/btree-example/btreehelper/src/main/resources/hyracks-deployment.properties
@@ -0,0 +1 @@
+nc.bootstrap.class=edu.uci.ics.hyracks.examples.btree.helper.NCBootstrap
\ No newline at end of file