Improve Error Handling in Local Directory Feeds
This change improves handling of two error types for filesystem
based feeds. The first one is the handling of IO Errors which
causes the input stream to be closed, and the second one is
reacting to missed filesystem events. In both cases, we scan the
directory and compare it with the history we have in order to
resume from where we last left off.
In addition, this change includes some refactoring in external
data. Particularly, we get rid of the stream provider layer and
instead, stream factories create input streams directly. This
is consistent with record reader factories which create readers
directly without reader providers.
Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Reviewed-on: https://asterix-gerrit.ics.uci.edu/720
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <hubailmor@gmail.com>
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AInputStream.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/AsterixInputStream.java
similarity index 69%
rename from asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AInputStream.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/api/AsterixInputStream.java
index b78f96d..83d7a3a 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AInputStream.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/AsterixInputStream.java
@@ -16,21 +16,29 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.external.input.stream;
+package org.apache.asterix.external.api;
import java.io.InputStream;
import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
import org.apache.asterix.external.util.FeedLogManager;
-public abstract class AInputStream extends InputStream {
- public abstract boolean skipError() throws Exception;
+public abstract class AsterixInputStream extends InputStream {
+
+ protected AbstractFeedDataFlowController controller;
+ protected FeedLogManager logManager;
public abstract boolean stop() throws Exception;
- // TODO: Find a better way to send notifications
- public abstract void setController(AbstractFeedDataFlowController controller);
+ public abstract boolean handleException(Throwable th);
// TODO: Find a better way to send notifications
- public abstract void setFeedLogManager(FeedLogManager logManager);
+ public void setController(AbstractFeedDataFlowController controller) {
+ this.controller = controller;
+ }
+
+ // TODO: Find a better way to send notifications
+ public void setFeedLogManager(FeedLogManager logManager) {
+ this.logManager = logManager;
+ }
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalIndexer.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalIndexer.java
index 01ffd99..c261ae3 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalIndexer.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalIndexer.java
@@ -36,7 +36,7 @@
* the new reader
* @throws Exception
*/
- public void reset(IRecordReader<?> reader) throws IOException;
+ public void reset(IIndexingDatasource reader) throws IOException;
/**
* This method is called by the dataflow controller with each tuple. the indexer is expected to append record ids to the tuple.
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingDatasource.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingDatasource.java
index c247ef6..5381ef7 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingDatasource.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingDatasource.java
@@ -18,6 +18,33 @@
*/
package org.apache.asterix.external.api;
+import java.util.List;
+
+import org.apache.asterix.external.indexing.ExternalFile;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapred.RecordReader;
+
+/**
+ * An interface for external data sources which support indexing
+ */
public interface IIndexingDatasource {
+ /**
+ * @return an external indexer that is used to write RID fields for each record
+ */
public IExternalIndexer getIndexer();
+
+ /**
+ * @return a list of external files being accessed
+ */
+ public List<ExternalFile> getSnapshot();
+
+ /**
+ * @return the index of the currently being read file
+ */
+ public int getCurrentSplitIndex();
+
+ /**
+ * @return an HDFS record reader that is used to get the current position in the file
+ */
+ public RecordReader<?, ? extends Writable> getReader();
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProviderFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamFactory.java
similarity index 82%
rename from asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProviderFactory.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamFactory.java
index f52f7d3..a460e49 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProviderFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamFactory.java
@@ -21,10 +21,9 @@
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-public interface IInputStreamProviderFactory extends IExternalDataSourceFactory {
+public interface IInputStreamFactory extends IExternalDataSourceFactory {
- public IInputStreamProvider createInputStreamProvider(IHyracksTaskContext ctx, int partition)
- throws HyracksDataException;
+ public AsterixInputStream createInputStream(IHyracksTaskContext ctx, int partition) throws HyracksDataException;
@Override
public default DataSourceType getDataSourceType() {
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
index b4d67d4..11e2472 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
@@ -21,6 +21,7 @@
import java.io.Closeable;
import java.io.IOException;
+import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
import org.apache.asterix.external.util.FeedLogManager;
/**
@@ -54,7 +55,7 @@
* set a pointer to the controller of the feed. the controller can be used to flush()
* parsed records when waiting for more records to be pushed
*/
- public void setController(IDataFlowController controller);
+ public void setController(AbstractFeedDataFlowController controller);
// TODO: Find a better way to perform logging. this doesn't fit here
/**
@@ -62,4 +63,9 @@
* progress and errors
*/
public void setFeedLogManager(FeedLogManager feedLogManager);
+
+ /**
+ * gives the record reader a chance to recover from IO errors during feed intake
+ */
+ public boolean handleException(Throwable th);
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
index 2cc3c66..6401234 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
@@ -67,22 +67,29 @@
continue;
}
tb.reset();
- dataParser.parse(record, tb.getDataOutput());
+ try {
+ dataParser.parse(record, tb.getDataOutput());
+ } catch (Exception e) {
+ e.printStackTrace();
+ LOGGER.warn(ExternalDataConstants.ERROR_PARSE_RECORD, e);
+ feedLogManager.logRecord(record.toString(), ExternalDataConstants.ERROR_PARSE_RECORD);
+ continue;
+ }
tb.addFieldEndOffset();
addMetaPart(tb, record);
addPrimaryKeys(tb, record);
if (tb.getSize() > tupleForwarder.getMaxRecordSize()) {
// log
- feedLogManager.logRecord(record.toString(), ExternalDataConstants.LARGE_RECORD_ERROR_MESSAGE);
+ feedLogManager.logRecord(record.toString(), ExternalDataConstants.ERROR_LARGE_RECORD);
continue;
}
tupleForwarder.addTuple(tb);
}
- } catch (Throwable th) {
+ } catch (Exception e) {
failed = true;
tupleForwarder.flush();
- LOGGER.warn("Failure during while operating a feed source", th);
- throw new HyracksDataException(th);
+ LOGGER.warn("Failure while operating a feed source", e);
+ throw new HyracksDataException(e);
}
try {
tupleForwarder.close();
@@ -156,6 +163,7 @@
@Override
public boolean handleException(Throwable th) {
- return true;
+ // This is not a parser record. most likely, this error happened in the record reader.
+ return recordReader.handleException(th);
}
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedStreamDataFlowController.java b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedStreamDataFlowController.java
index f233971..ed5e355 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedStreamDataFlowController.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedStreamDataFlowController.java
@@ -18,8 +18,8 @@
*/
package org.apache.asterix.external.dataflow;
+import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.api.IStreamDataParser;
-import org.apache.asterix.external.input.stream.AInputStream;
import org.apache.asterix.external.util.FeedLogManager;
import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.hyracks.api.context.IHyracksTaskContext;
@@ -28,10 +28,11 @@
public class FeedStreamDataFlowController extends AbstractFeedDataFlowController {
private final IStreamDataParser dataParser;
- private final AInputStream stream;
+ private final AsterixInputStream stream;
public FeedStreamDataFlowController(IHyracksTaskContext ctx, FeedTupleForwarder tupleForwarder,
- FeedLogManager feedLogManager, int numOfFields, IStreamDataParser streamParser, AInputStream inputStream) {
+ FeedLogManager feedLogManager, int numOfFields, IStreamDataParser streamParser,
+ AsterixInputStream inputStream) {
super(ctx, tupleForwarder, feedLogManager, numOfFields);
this.dataParser = streamParser;
this.stream = inputStream;
@@ -73,7 +74,7 @@
public boolean handleException(Throwable th) {
boolean handled = true;
try {
- handled &= stream.skipError();
+ handled &= stream.handleException(th);
if (handled) {
handled &= dataParser.reset(stream);
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java
index 73de838..63787ee 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java
@@ -77,7 +77,7 @@
public void addTuple(ArrayTupleBuilder tb) throws HyracksDataException {
if (tb.getSize() > maxRecordSize) {
try {
- feedLogManager.logRecord(tb.toString(), ExternalDataConstants.LARGE_RECORD_ERROR_MESSAGE);
+ feedLogManager.logRecord(tb.toString(), ExternalDataConstants.ERROR_LARGE_RECORD);
} catch (IOException e) {
throw new HyracksDataException(e);
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedFrameHandler.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedFrameHandler.java
index 9cb0feb..3b37500 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedFrameHandler.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedFrameHandler.java
@@ -26,9 +26,9 @@
public interface IFeedFrameHandler {
- public void handleFrame(ByteBuffer frame) throws HyracksDataException;
+ public void handleFrame(ByteBuffer frame) throws HyracksDataException, InterruptedException;
- public void handleDataBucket(DataBucket bucket);
+ public void handleDataBucket(DataBucket bucket) throws InterruptedException;
public void close();
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedJoint.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedJoint.java
index 2a3932d..477ec36 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedJoint.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedJoint.java
@@ -20,7 +20,7 @@
import java.util.List;
-import org.apache.asterix.external.feed.api.IFeedLifecycleListener.ConnectionLocation;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
import org.apache.asterix.external.feed.management.FeedConnectionId;
import org.apache.asterix.external.feed.management.FeedConnectionRequest;
import org.apache.asterix.external.feed.management.FeedId;
@@ -71,7 +71,7 @@
/**
* @return the subscription location {@link ConnectionLocation} associated with the FeedJoint
*/
- public ConnectionLocation getConnectionLocation();
+ public FeedRuntimeType getConnectionLocation();
/**
* @return the unique {@link FeedJointKey} associated with the FeedJoint
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedLifecycleListener.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedLifecycleListener.java
index ce82aaf..28b713e 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedLifecycleListener.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedLifecycleListener.java
@@ -27,12 +27,6 @@
import org.apache.hyracks.api.job.IJobLifecycleListener;
public interface IFeedLifecycleListener extends IJobLifecycleListener, IClusterEventsSubscriber {
-
- public enum ConnectionLocation {
- SOURCE_FEED_INTAKE_STAGE,
- SOURCE_FEED_COMPUTE_STAGE
- }
-
public IFeedJoint getAvailableFeedJoint(FeedJointKey feedJoinKey);
public boolean isFeedJointAvailable(FeedJointKey feedJoinKey);
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IMessageReceiver.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IMessageReceiver.java
index bdfbfdb..e58d99e 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IMessageReceiver.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IMessageReceiver.java
@@ -20,7 +20,7 @@
public interface IMessageReceiver<T> {
- public void sendMessage(T message);
+ public void sendMessage(T message) throws InterruptedException;
public void close(boolean processPending);
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/DistributeFeedFrameWriter.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/DistributeFeedFrameWriter.java
index d314f74..8b7e2ba 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/DistributeFeedFrameWriter.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/DistributeFeedFrameWriter.java
@@ -66,7 +66,7 @@
public DistributeFeedFrameWriter(IHyracksTaskContext ctx, FeedId feedId, IFrameWriter writer,
FeedRuntimeType feedRuntimeType, int partition, FrameTupleAccessor fta, IFeedManager feedManager)
- throws IOException {
+ throws IOException {
this.feedId = feedId;
this.frameDistributor = new FrameDistributor(feedId, feedRuntimeType, partition, true,
feedManager.getFeedMemoryManager(), fta);
@@ -108,7 +108,7 @@
}
}
- public void notifyEndOfFeed() {
+ public void notifyEndOfFeed() throws InterruptedException {
frameDistributor.notifyEndOfFeed();
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedCollectRuntimeInputHandler.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedCollectRuntimeInputHandler.java
index 4a361fb..447c4bc 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedCollectRuntimeInputHandler.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedCollectRuntimeInputHandler.java
@@ -19,7 +19,6 @@
package org.apache.asterix.external.feed.dataflow;
import java.io.IOException;
-import java.nio.ByteBuffer;
import org.apache.asterix.external.feed.api.IFeedManager;
import org.apache.asterix.external.feed.management.FeedConnectionId;
@@ -35,8 +34,8 @@
private final FeedFrameCache feedFrameCache;
- public FeedCollectRuntimeInputHandler(IHyracksTaskContext ctx, FeedConnectionId connectionId, FeedRuntimeId runtimeId,
- IFrameWriter coreOperator, FeedPolicyAccessor fpa, boolean bufferingEnabled,
+ public FeedCollectRuntimeInputHandler(IHyracksTaskContext ctx, FeedConnectionId connectionId,
+ FeedRuntimeId runtimeId, IFrameWriter coreOperator, FeedPolicyAccessor fpa, boolean bufferingEnabled,
FrameTupleAccessor fta, RecordDescriptor recordDesc, IFeedManager feedManager, int nPartitions)
throws IOException {
super(ctx, connectionId, runtimeId, coreOperator, fpa, bufferingEnabled, fta, recordDesc, feedManager,
@@ -44,11 +43,6 @@
this.feedFrameCache = new FeedFrameCache(ctx, fta, coreOperator);
}
- public void process(ByteBuffer frame) throws HyracksDataException {
- feedFrameCache.sendMessage(frame);
- super.process(frame);
- }
-
public void replayFrom(int recordId) throws HyracksDataException {
feedFrameCache.replayRecords(recordId);
}
@@ -57,8 +51,7 @@
feedFrameCache.dropTillRecordId(recordId);
}
- public void replayCached() throws HyracksDataException{
+ public void replayCached() throws HyracksDataException {
feedFrameCache.replayAll();
}
-
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameHandlers.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameHandlers.java
index 3c45a20..2a6fd79 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameHandlers.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameHandlers.java
@@ -141,9 +141,9 @@
}
@Override
- public void handleDataBucket(DataBucket bucket) {
+ public void handleDataBucket(DataBucket bucket) throws InterruptedException {
for (FeedFrameCollector collector : frameCollectors) {
- collector.sendMessage(bucket); // asynchronous call
+ collector.sendMessage(bucket);
}
}
@@ -174,7 +174,7 @@
}
@Override
- public void handleFrame(ByteBuffer frame) throws HyracksDataException {
+ public void handleFrame(ByteBuffer frame) throws HyracksDataException, InterruptedException {
receiver.sendMessage(frame);
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedRuntimeInputHandler.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedRuntimeInputHandler.java
index 1ceb36b..dcd03a9 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedRuntimeInputHandler.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedRuntimeInputHandler.java
@@ -71,13 +71,13 @@
private final FrameEventCallback frameEventCallback;
private boolean bufferingEnabled;
- private IFrameWriter coreOperator;
private FrameCollection frameCollection;
private Mode mode;
private Mode lastMode;
private boolean finished;
private long nProcessed;
private boolean throttlingEnabled;
+ protected IFrameWriter coreOperator;
public FeedRuntimeInputHandler(IHyracksTaskContext ctx, FeedConnectionId connectionId, FeedRuntimeId runtimeId,
IFrameWriter coreOperator, FeedPolicyAccessor fpa, boolean bufferingEnabled, FrameTupleAccessor fta,
@@ -263,7 +263,6 @@
}
} else {
coreOperator.nextFrame(frame); // synchronous
- mBuffer.sendReport(frame);
}
} else {
DataBucket bucket = pool.getDataBucket();
@@ -374,8 +373,8 @@
}
mBuffer.close(false, disableMonitoring);
if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Closed input side handler for " + this.runtimeId + " disabled monitoring "
- + disableMonitoring + " Mode for runtime " + this.mode);
+ LOGGER.info("Closed input side handler for " + this.runtimeId + " disabled monitoring " + disableMonitoring
+ + " Mode for runtime " + this.mode);
}
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FrameDistributor.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FrameDistributor.java
index 0c9017b..802a791 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FrameDistributor.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FrameDistributor.java
@@ -74,7 +74,7 @@
public FrameDistributor(FeedId feedId, FeedRuntimeType feedRuntimeType, int partition,
boolean enableSynchronousTransfer, IFeedMemoryManager memoryManager, FrameTupleAccessor fta)
- throws HyracksDataException {
+ throws HyracksDataException {
this.feedId = feedId;
this.feedRuntimeType = feedRuntimeType;
this.partition = partition;
@@ -85,7 +85,7 @@
this.fta = fta;
}
- public void notifyEndOfFeed() {
+ public void notifyEndOfFeed() throws InterruptedException {
DataBucket bucket = getDataBucket();
if (bucket != null) {
sendEndOfFeedDataBucket(bucket);
@@ -104,7 +104,7 @@
}
}
- private void sendEndOfFeedDataBucket(DataBucket bucket) {
+ private void sendEndOfFeedDataBucket(DataBucket bucket) throws InterruptedException {
bucket.setContentType(DataBucket.ContentType.EOD);
nextBucket(bucket);
if (LOGGER.isLoggable(Level.INFO)) {
@@ -207,45 +207,50 @@
}
public synchronized void nextFrame(ByteBuffer frame) throws HyracksDataException {
- switch (distributionMode) {
- case INACTIVE:
- break;
- case SINGLE:
- FeedFrameCollector collector = registeredCollectors.values().iterator().next();
- switch (collector.getState()) {
- case HANDOVER:
- case ACTIVE:
- if (enableSynchronousTransfer) {
- collector.nextFrame(frame); // processing is synchronous
- } else {
+ try {
+ switch (distributionMode) {
+ case INACTIVE:
+ break;
+ case SINGLE:
+ FeedFrameCollector collector = registeredCollectors.values().iterator().next();
+ switch (collector.getState()) {
+ case HANDOVER:
+ case ACTIVE:
+ if (enableSynchronousTransfer) {
+ collector.nextFrame(frame); // processing is synchronous
+ } else {
+ handleDataBucket(frame);
+ }
+ break;
+ case TRANSITION:
handleDataBucket(frame);
- }
- break;
- case TRANSITION:
- handleDataBucket(frame);
- break;
- case FINISHED:
- if (LOGGER.isLoggable(Level.WARNING)) {
- LOGGER.warning("Discarding fetched tuples, feed has ended [" + registeredCollectors.get(0)
- + "]" + " Feed Id " + feedId + " frame distributor " + this.getFeedRuntimeType());
- }
- registeredCollectors.remove(0);
- break;
- }
- break;
- case SHARED:
- handleDataBucket(frame);
- break;
+ break;
+ case FINISHED:
+ if (LOGGER.isLoggable(Level.WARNING)) {
+ LOGGER.warning("Discarding fetched tuples, feed has ended ["
+ + registeredCollectors.get(0) + "]" + " Feed Id " + feedId
+ + " frame distributor " + this.getFeedRuntimeType());
+ }
+ registeredCollectors.remove(0);
+ break;
+ }
+ break;
+ case SHARED:
+ handleDataBucket(frame);
+ break;
+ }
+ } catch (Exception e) {
+ throw new HyracksDataException(e);
}
}
- private void nextBucket(DataBucket bucket) {
+ private void nextBucket(DataBucket bucket) throws InterruptedException {
for (FeedFrameCollector collector : registeredCollectors.values()) {
collector.sendMessage(bucket); // asynchronous call
}
}
- private void handleDataBucket(ByteBuffer frame) throws HyracksDataException {
+ private void handleDataBucket(ByteBuffer frame) throws HyracksDataException, InterruptedException {
DataBucket bucket = getDataBucket();
if (bucket == null) {
handleFrameDuringMemoryCongestion(frame);
@@ -281,32 +286,36 @@
return distributionMode;
}
- public void close() {
- switch (distributionMode) {
- case INACTIVE:
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("FrameDistributor is " + distributionMode);
- }
- break;
- case SINGLE:
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Disconnecting single frame reader in " + distributionMode + " mode " + " for feedId "
- + feedId + " " + this.feedRuntimeType);
- }
- setMode(DistributionMode.INACTIVE);
- if (!enableSynchronousTransfer) {
+ public void close() throws HyracksDataException {
+ try {
+ switch (distributionMode) {
+ case INACTIVE:
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("FrameDistributor is " + distributionMode);
+ }
+ break;
+ case SINGLE:
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Disconnecting single frame reader in " + distributionMode + " mode "
+ + " for feedId " + feedId + " " + this.feedRuntimeType);
+ }
+ setMode(DistributionMode.INACTIVE);
+ if (!enableSynchronousTransfer) {
+ notifyEndOfFeed(); // send EOD Data Bucket
+ waitForCollectorsToFinish();
+ }
+ registeredCollectors.values().iterator().next().disconnect();
+ break;
+ case SHARED:
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Signalling End Of Feed; currently operating in " + distributionMode + " mode");
+ }
notifyEndOfFeed(); // send EOD Data Bucket
waitForCollectorsToFinish();
- }
- registeredCollectors.values().iterator().next().disconnect();
- break;
- case SHARED:
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Signalling End Of Feed; currently operating in " + distributionMode + " mode");
- }
- notifyEndOfFeed(); // send EOD Data Bucket
- waitForCollectorsToFinish();
- break;
+ break;
+ }
+ } catch (Exception e) {
+ throw new HyracksDataException(e);
}
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedConnectionRequest.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedConnectionRequest.java
index 0175dcb..b1d3300 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedConnectionRequest.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedConnectionRequest.java
@@ -21,7 +21,7 @@
import java.util.List;
import java.util.Map;
-import org.apache.asterix.external.feed.api.IFeedLifecycleListener.ConnectionLocation;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
import org.apache.commons.lang3.StringUtils;
/**
@@ -47,7 +47,7 @@
private final FeedJointKey feedJointKey;
/** Location in the source feed pipeline from where feed tuples are received. **/
- private final ConnectionLocation connectionLocation;
+ private final FeedRuntimeType connectionLocation;
/** List of functions that need to be applied in sequence after the data hand-off at the source feedPointKey. **/
private final List<String> functionsToApply;
@@ -66,8 +66,7 @@
private final FeedId receivingFeedId;
-
- public FeedConnectionRequest(FeedJointKey feedPointKey, ConnectionLocation connectionLocation,
+ public FeedConnectionRequest(FeedJointKey feedPointKey, FeedRuntimeType connectionLocation,
List<String> functionsToApply, String targetDataset, String policy, Map<String, String> policyParameters,
FeedId receivingFeedId) {
this.feedJointKey = feedPointKey;
@@ -100,7 +99,7 @@
return targetDataset;
}
- public ConnectionLocation getSubscriptionLocation() {
+ public FeedRuntimeType getSubscriptionLocation() {
return connectionLocation;
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/MessageReceiver.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/MessageReceiver.java
index f69c552..6ed176a 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/MessageReceiver.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/MessageReceiver.java
@@ -18,9 +18,9 @@
*/
package org.apache.asterix.external.feed.message;
+import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
-import java.util.concurrent.LinkedBlockingQueue;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -31,11 +31,11 @@
protected static final Logger LOGGER = Logger.getLogger(MessageReceiver.class.getName());
- protected final LinkedBlockingQueue<T> inbox;
+ protected final ArrayBlockingQueue<T> inbox;
protected ExecutorService executor;
public MessageReceiver() {
- inbox = new LinkedBlockingQueue<T>();
+ inbox = new ArrayBlockingQueue<T>(2);
}
public abstract void processMessage(T message) throws Exception;
@@ -47,8 +47,8 @@
}
@Override
- public synchronized void sendMessage(T message) {
- inbox.add(message);
+ public synchronized void sendMessage(T message) throws InterruptedException {
+ inbox.put(message);
}
@Override
@@ -68,7 +68,7 @@
private static class MessageReceiverRunnable<T> implements Runnable {
- private final LinkedBlockingQueue<T> inbox;
+ private final ArrayBlockingQueue<T> inbox;
private final MessageReceiver<T> messageReceiver;
public MessageReceiverRunnable(MessageReceiver<T> messageReceiver) {
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/IngestionRuntime.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/IngestionRuntime.java
index 34cb575..021ad36 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/IngestionRuntime.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/IngestionRuntime.java
@@ -76,7 +76,7 @@
subscribers.remove(collectionRuntime);
}
- public void endOfFeed() {
+ public void endOfFeed() throws InterruptedException {
dWriter.notifyEndOfFeed();
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Notified End Of Feed [" + this + "]");
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FileOffsetIndexer.java b/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FileOffsetIndexer.java
index 873b420..d7a20e1 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FileOffsetIndexer.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FileOffsetIndexer.java
@@ -21,8 +21,7 @@
import java.io.IOException;
import org.apache.asterix.external.api.IExternalIndexer;
-import org.apache.asterix.external.api.IRecordReader;
-import org.apache.asterix.external.input.record.reader.hdfs.HDFSRecordReader;
+import org.apache.asterix.external.api.IIndexingDatasource;
import org.apache.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
import org.apache.asterix.om.base.AMutableInt32;
import org.apache.asterix.om.base.AMutableInt64;
@@ -40,7 +39,7 @@
public static final int NUM_OF_FIELDS = 2;
protected final AMutableInt32 fileNumber = new AMutableInt32(0);
protected final AMutableInt64 offset = new AMutableInt64(0);
- protected RecordReader<?, Writable> recordReader;
+ protected RecordReader<?, ? extends Writable> recordReader;
@SuppressWarnings("unchecked")
private ISerializerDeserializer<IAObject> intSerde = AqlSerializerDeserializerProvider.INSTANCE
@@ -50,13 +49,11 @@
.getSerializerDeserializer(BuiltinType.AINT64);
@Override
- public void reset(IRecordReader<?> reader) throws HyracksDataException {
+ public void reset(IIndexingDatasource dataSource) throws HyracksDataException {
try {
- //TODO: Make it more generic since we can't assume it is always going to be HDFS records.
- @SuppressWarnings("unchecked")
- HDFSRecordReader<?, Writable> hdfsReader = (HDFSRecordReader<?, Writable>) reader;
- fileNumber.setValue(hdfsReader.getSnapshot().get(hdfsReader.getCurrentSplitIndex()).getFileNumber());
- recordReader = hdfsReader.getReader();
+ //TODO: Make it more generic since we can't assume it is always going to be HDFS records
+ fileNumber.setValue(dataSource.getSnapshot().get(dataSource.getCurrentSplitIndex()).getFileNumber());
+ recordReader = dataSource.getReader();
offset.setValue(recordReader.getPos());
} catch (IOException e) {
throw new HyracksDataException(e);
@@ -79,5 +76,4 @@
public int getNumberOfFields() {
return NUM_OF_FIELDS;
}
-
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/RecordColumnarIndexer.java b/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/RecordColumnarIndexer.java
index a2641c8..c626be5 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/RecordColumnarIndexer.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/RecordColumnarIndexer.java
@@ -21,7 +21,7 @@
import java.io.IOException;
import org.apache.asterix.external.api.IExternalIndexer;
-import org.apache.asterix.external.api.IRecordReader;
+import org.apache.asterix.external.api.IIndexingDatasource;
import org.apache.asterix.external.input.record.reader.hdfs.HDFSRecordReader;
import org.apache.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
import org.apache.asterix.om.base.AMutableInt32;
@@ -52,7 +52,7 @@
.getSerializerDeserializer(BuiltinType.AINT64);
@Override
- public void reset(IRecordReader<?> reader) throws HyracksDataException {
+ public void reset(IIndexingDatasource reader) throws HyracksDataException {
try {
//TODO: Make this more generic. right now, it works because we only index hdfs files.
@SuppressWarnings("unchecked")
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
index f44d7bc..de185e0 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
@@ -24,15 +24,15 @@
import java.util.Map;
import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.api.IIndexibleExternalDataSource;
-import org.apache.asterix.external.api.IInputStreamProvider;
-import org.apache.asterix.external.api.IInputStreamProviderFactory;
+import org.apache.asterix.external.api.IInputStreamFactory;
import org.apache.asterix.external.api.IRecordReader;
import org.apache.asterix.external.api.IRecordReaderFactory;
import org.apache.asterix.external.indexing.ExternalFile;
import org.apache.asterix.external.indexing.IndexingScheduler;
import org.apache.asterix.external.input.record.reader.hdfs.HDFSRecordReader;
-import org.apache.asterix.external.input.stream.provider.HDFSInputStreamProvider;
+import org.apache.asterix.external.input.stream.HDFSInputStream;
import org.apache.asterix.external.provider.ExternalIndexerProvider;
import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.asterix.external.util.HDFSUtils;
@@ -49,7 +49,7 @@
import org.apache.hyracks.hdfs.scheduler.Scheduler;
public class HDFSDataSourceFactory
- implements IInputStreamProviderFactory, IRecordReaderFactory<Object>, IIndexibleExternalDataSource {
+ implements IInputStreamFactory, IRecordReaderFactory<Object>, IIndexibleExternalDataSource {
protected static final long serialVersionUID = 1L;
protected transient AlgebricksAbsolutePartitionConstraint clusterLocations;
@@ -116,10 +116,9 @@
* The method below was modified to take care of the following
* 1. when target files are not null, it generates a file aware input stream that validate
* against the files
- * 2. if the data is binary, it returns a generic reade */
+ * 2. if the data is binary, it returns a generic reader */
@Override
- public IInputStreamProvider createInputStreamProvider(IHyracksTaskContext ctx, int partition)
- throws HyracksDataException {
+ public AsterixInputStream createInputStream(IHyracksTaskContext ctx, int partition) throws HyracksDataException {
try {
if (!configured) {
conf = confFactory.getConf();
@@ -127,8 +126,7 @@
nodeName = ctx.getJobletContext().getApplicationContext().getNodeId();
configured = true;
}
- return new HDFSInputStreamProvider<Object>(read, inputSplits, readSchedule, nodeName, conf, configuration,
- files);
+ return new HDFSInputStream(read, inputSplits, readSchedule, nodeName, conf, configuration, files);
} catch (Exception e) {
throw new HyracksDataException(e);
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/CharArrayRecord.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/CharArrayRecord.java
index affdc84..6b8e10d 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/CharArrayRecord.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/CharArrayRecord.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.external.input.record;
+import java.io.IOException;
import java.nio.CharBuffer;
import java.util.Arrays;
@@ -29,6 +30,16 @@
private char[] value;
private int size;
+ public CharArrayRecord(int initialCapacity) {
+ value = new char[initialCapacity];
+ size = 0;
+ }
+
+ public CharArrayRecord() {
+ value = new char[ExternalDataConstants.DEFAULT_BUFFER_SIZE];
+ size = 0;
+ }
+
@Override
public byte[] getBytes() {
return new String(value).getBytes();
@@ -44,16 +55,6 @@
return size;
}
- public CharArrayRecord(int initialCapacity) {
- value = new char[initialCapacity];
- size = 0;
- }
-
- public CharArrayRecord() {
- value = new char[ExternalDataConstants.DEFAULT_BUFFER_SIZE];
- size = 0;
- }
-
public void setValue(char[] recordBuffer, int offset, int length) {
if (value.length < length) {
value = new char[length];
@@ -62,13 +63,21 @@
size = length;
}
- private void ensureCapacity(int len) {
+ private void ensureCapacity(int len) throws IOException {
if (value.length < len) {
- value = Arrays.copyOf(value, (int) (len * 1.25));
+ if (len > ExternalDataConstants.MAX_RECORD_SIZE) {
+ throw new IOException(
+ "Record is too large!. Maximum record size is " + ExternalDataConstants.MAX_RECORD_SIZE);
+ }
+ int newSize = len + ExternalDataConstants.DEFAULT_BUFFER_INCREMENT;
+ if (newSize > ExternalDataConstants.MAX_RECORD_SIZE) {
+ newSize = ExternalDataConstants.MAX_RECORD_SIZE;
+ }
+ value = Arrays.copyOf(value, newSize);
}
}
- public void append(char[] recordBuffer, int offset, int length) {
+ public void append(char[] recordBuffer, int offset, int length) throws IOException {
ensureCapacity(size + length);
System.arraycopy(recordBuffer, offset, value, size, length);
size += length;
@@ -84,25 +93,25 @@
return String.valueOf(value, 0, size);
}
- public void endRecord() {
+ public void endRecord() throws IOException {
if (value[size - 1] != ExternalDataConstants.LF) {
appendChar(ExternalDataConstants.LF);
}
}
- private void appendChar(char c) {
+ private void appendChar(char c) throws IOException {
ensureCapacity(size + 1);
value[size] = c;
size++;
}
- public void append(char[] recordBuffer) {
+ public void append(char[] recordBuffer) throws IOException {
ensureCapacity(size + recordBuffer.length);
System.arraycopy(recordBuffer, 0, value, size, recordBuffer.length);
size += recordBuffer.length;
}
- public void append(CharBuffer chars) {
+ public void append(CharBuffer chars) throws IOException {
ensureCapacity(size + chars.limit());
chars.get(value, size, chars.limit());
size += chars.limit();
@@ -114,7 +123,7 @@
this.size = value.length;
}
- public void set(StringBuilder builder) {
+ public void set(StringBuilder builder) throws IOException {
ensureCapacity(builder.length());
builder.getChars(0, builder.length(), value, 0);
this.size = builder.length();
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/DCPRequestToRecordWithMetadataAndPKConverter.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/DCPRequestToRecordWithMetadataAndPKConverter.java
index 1f82e85..379bbf2 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/DCPRequestToRecordWithMetadataAndPKConverter.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/DCPRequestToRecordWithMetadataAndPKConverter.java
@@ -100,7 +100,7 @@
}
public static void set(final ByteBuf content, final CharsetDecoder decoder, final ByteBuffer bytes,
- final CharBuffer chars, final CharArrayRecord record) {
+ final CharBuffer chars, final CharArrayRecord record) throws IOException {
int position = content.readerIndex();
final int limit = content.writerIndex();
final int contentSize = content.readableBytes();
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/TestAsterixMembersReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/TestAsterixMembersReader.java
index 916904f..665d50b 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/TestAsterixMembersReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/TestAsterixMembersReader.java
@@ -20,9 +20,9 @@
import java.io.IOException;
-import org.apache.asterix.external.api.IDataFlowController;
import org.apache.asterix.external.api.IRawRecord;
import org.apache.asterix.external.api.IRecordReader;
+import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
import org.apache.asterix.external.input.record.CharArrayRecord;
import org.apache.asterix.external.input.record.GenericRecord;
import org.apache.asterix.external.input.record.RecordWithPK;
@@ -79,10 +79,15 @@
}
@Override
- public void setController(final IDataFlowController controller) {
+ public void setController(final AbstractFeedDataFlowController controller) {
}
@Override
public void setFeedLogManager(final FeedLogManager feedLogManager) {
}
+
+ @Override
+ public boolean handleException(Throwable th) {
+ return false;
+ }
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/AbstractCharRecordLookupReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/AbstractCharRecordLookupReader.java
index 0627660..45d01e9 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/AbstractCharRecordLookupReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/AbstractCharRecordLookupReader.java
@@ -60,7 +60,7 @@
protected abstract void readRecord(RecordId rid) throws IOException;
- private void writeRecord() {
+ private void writeRecord() throws IOException {
reusableByteBuffer.clear();
if (reusableByteBuffer.remaining() < value.getLength()) {
reusableByteBuffer = ByteBuffer
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSRecordReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSRecordReader.java
index 5ed6dc5..3500d32 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSRecordReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSRecordReader.java
@@ -21,11 +21,11 @@
import java.io.IOException;
import java.util.List;
-import org.apache.asterix.external.api.IDataFlowController;
import org.apache.asterix.external.api.IExternalIndexer;
import org.apache.asterix.external.api.IIndexingDatasource;
import org.apache.asterix.external.api.IRawRecord;
import org.apache.asterix.external.api.IRecordReader;
+import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
import org.apache.asterix.external.indexing.ExternalFile;
import org.apache.asterix.external.input.record.GenericRecord;
import org.apache.asterix.external.util.FeedLogManager;
@@ -121,8 +121,9 @@
FileStatus fileStatus = hdfs.getFileStatus(new Path(fileName));
// Skip if not the same file stored in the files snapshot
if (fileStatus.getModificationTime() != snapshot.get(currentSplitIndex).getLastModefiedTime()
- .getTime())
+ .getTime()) {
continue;
+ }
}
reader.close();
@@ -160,10 +161,12 @@
return indexer;
}
+ @Override
public List<ExternalFile> getSnapshot() {
return snapshot;
}
+ @Override
public int getCurrentSplitIndex() {
return currentSplitIndex;
}
@@ -177,6 +180,11 @@
}
@Override
- public void setController(IDataFlowController controller) {
+ public void setController(AbstractFeedDataFlowController controller) {
+ }
+
+ @Override
+ public boolean handleException(Throwable th) {
+ return false;
}
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVReader.java
index 4e41357..185aea0 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVReader.java
@@ -21,7 +21,6 @@
import java.io.IOException;
import java.util.concurrent.ArrayBlockingQueue;
-import org.apache.asterix.external.api.IDataFlowController;
import org.apache.asterix.external.api.IRawRecord;
import org.apache.asterix.external.api.IRecordReader;
import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
@@ -92,10 +91,10 @@
}
private void connect() {
- core.send(new SeedNodesRequest(sourceNodes))
- .timeout(KVReaderFactory.TIMEOUT, KVReaderFactory.TIME_UNIT).toBlocking().single();
- core.send(new OpenBucketRequest(bucket, password))
- .timeout(KVReaderFactory.TIMEOUT, KVReaderFactory.TIME_UNIT).toBlocking().single();
+ core.send(new SeedNodesRequest(sourceNodes)).timeout(KVReaderFactory.TIMEOUT, KVReaderFactory.TIME_UNIT)
+ .toBlocking().single();
+ core.send(new OpenBucketRequest(bucket, password)).timeout(KVReaderFactory.TIMEOUT, KVReaderFactory.TIME_UNIT)
+ .toBlocking().single();
this.pushThread = new Thread(new Runnable() {
@Override
public void run() {
@@ -183,11 +182,16 @@
}
@Override
- public void setController(IDataFlowController controller) {
- this.controller = (AbstractFeedDataFlowController) controller;
+ public void setController(AbstractFeedDataFlowController controller) {
+ this.controller = controller;
}
@Override
public void setFeedLogManager(FeedLogManager feedLogManager) {
}
+
+ @Override
+ public boolean handleException(Throwable th) {
+ return false;
+ }
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReader.java
index f540f4e..b75f26c 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReader.java
@@ -22,9 +22,9 @@
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
-import org.apache.asterix.external.api.IDataFlowController;
import org.apache.asterix.external.api.IRawRecord;
import org.apache.asterix.external.api.IRecordReader;
+import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
import org.apache.asterix.external.input.record.GenericRecord;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.FeedLogManager;
@@ -169,11 +169,16 @@
}
@Override
- public void setController(final IDataFlowController controller) {
+ public void setController(final AbstractFeedDataFlowController controller) {
}
@Override
public void setFeedLogManager(final FeedLogManager feedLogManager) {
}
+ @Override
+ public boolean handleException(Throwable th) {
+ return false;
+ }
+
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReader.java
index a78f780..5760a7a 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReader.java
@@ -25,9 +25,9 @@
import java.util.List;
import java.util.Queue;
-import org.apache.asterix.external.api.IDataFlowController;
import org.apache.asterix.external.api.IRawRecord;
import org.apache.asterix.external.api.IRecordReader;
+import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
import org.apache.asterix.external.input.record.GenericRecord;
import org.apache.asterix.external.util.FeedLogManager;
import org.apache.log4j.Logger;
@@ -138,7 +138,12 @@
}
@Override
- public void setController(IDataFlowController controller) {
+ public void setController(AbstractFeedDataFlowController controller) {
+ }
+
+ @Override
+ public boolean handleException(Throwable th) {
+ return false;
}
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java
index 6a5d776..7d6c1f3 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java
@@ -19,21 +19,24 @@
package org.apache.asterix.external.input.record.reader.stream;
import java.io.IOException;
+import java.util.List;
-import org.apache.asterix.external.api.IDataFlowController;
+import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.api.IExternalIndexer;
import org.apache.asterix.external.api.IIndexingDatasource;
import org.apache.asterix.external.api.IRawRecord;
import org.apache.asterix.external.api.IRecordReader;
import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
+import org.apache.asterix.external.indexing.ExternalFile;
import org.apache.asterix.external.input.record.CharArrayRecord;
-import org.apache.asterix.external.input.stream.AInputStream;
-import org.apache.asterix.external.input.stream.AInputStreamReader;
+import org.apache.asterix.external.input.stream.AsterixInputStreamReader;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.FeedLogManager;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapred.RecordReader;
public abstract class AbstractStreamRecordReader implements IRecordReader<char[]>, IIndexingDatasource {
- protected final AInputStreamReader reader;
+ protected final AsterixInputStreamReader reader;
protected CharArrayRecord record;
protected char[] inputBuffer;
protected int bufferLength = 0;
@@ -42,8 +45,8 @@
protected boolean done = false;
protected FeedLogManager feedLogManager;
- public AbstractStreamRecordReader(AInputStream inputStream, IExternalIndexer indexer) {
- this.reader = new AInputStreamReader(inputStream);
+ public AbstractStreamRecordReader(AsterixInputStream inputStream, IExternalIndexer indexer) {
+ this.reader = new AsterixInputStreamReader(inputStream);
this.indexer = indexer;
record = new CharArrayRecord();
inputBuffer = new char[ExternalDataConstants.DEFAULT_BUFFER_SIZE];
@@ -79,8 +82,8 @@
}
@Override
- public void setController(IDataFlowController controller) {
- reader.setController((AbstractFeedDataFlowController) controller);
+ public void setController(AbstractFeedDataFlowController controller) {
+ reader.setController(controller);
}
@Override
@@ -88,4 +91,26 @@
this.feedLogManager = feedLogManager;
reader.setFeedLogManager(feedLogManager);
}
+
+ @Override
+ public boolean handleException(Throwable th) {
+ return reader.handleException(th);
+ }
+
+ //TODO: Fix the following method since they don't fit
+ //Already the fix is in another local branch
+ @Override
+ public List<ExternalFile> getSnapshot() {
+ return null;
+ }
+
+ @Override
+ public int getCurrentSplitIndex() {
+ return -1;
+ }
+
+ @Override
+ public RecordReader<?, Writable> getReader() {
+ return null;
+ }
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReaderFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReaderFactory.java
index 12c0229..2c82f47 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReaderFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReaderFactory.java
@@ -22,14 +22,13 @@
import java.util.Map;
import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.api.IExternalIndexer;
import org.apache.asterix.external.api.IIndexibleExternalDataSource;
import org.apache.asterix.external.api.IIndexingDatasource;
-import org.apache.asterix.external.api.IInputStreamProvider;
-import org.apache.asterix.external.api.IInputStreamProviderFactory;
+import org.apache.asterix.external.api.IInputStreamFactory;
import org.apache.asterix.external.api.IRecordReaderFactory;
import org.apache.asterix.external.indexing.ExternalFile;
-import org.apache.asterix.external.input.stream.AInputStream;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.api.context.IHyracksTaskContext;
@@ -39,11 +38,11 @@
implements IRecordReaderFactory<T>, IIndexibleExternalDataSource {
private static final long serialVersionUID = 1L;
- protected IInputStreamProviderFactory inputStreamFactory;
+ protected IInputStreamFactory inputStreamFactory;
protected Map<String, String> configuration;
public AbstractStreamRecordReaderFactory<T> setInputStreamFactoryProvider(
- IInputStreamProviderFactory inputStreamFactory) {
+ IInputStreamFactory inputStreamFactory) {
this.inputStreamFactory = inputStreamFactory;
return this;
}
@@ -82,15 +81,15 @@
return false;
}
- protected Pair<AInputStream, IExternalIndexer> getStreamAndIndexer(IHyracksTaskContext ctx, int partition)
+ protected Pair<AsterixInputStream, IExternalIndexer> getStreamAndIndexer(IHyracksTaskContext ctx, int partition)
throws HyracksDataException {
- IInputStreamProvider inputStreamProvider = inputStreamFactory.createInputStreamProvider(ctx, partition);
+ AsterixInputStream inputStream = inputStreamFactory.createInputStream(ctx, partition);
IExternalIndexer indexer = null;
if (inputStreamFactory.isIndexible()) {
if (((IIndexibleExternalDataSource) inputStreamFactory).isIndexingOp()) {
- indexer = ((IIndexingDatasource) inputStreamProvider).getIndexer();
+ indexer = ((IIndexingDatasource) inputStream).getIndexer();
}
}
- return new Pair<AInputStream, IExternalIndexer>(inputStreamProvider.getInputStream(), indexer);
+ return new Pair<AsterixInputStream, IExternalIndexer>(inputStream, indexer);
}
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReader.java
index fb56062..6964a82 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReader.java
@@ -20,13 +20,13 @@
import java.io.IOException;
+import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.api.IExternalIndexer;
-import org.apache.asterix.external.input.stream.AInputStream;
import org.apache.asterix.external.util.ExternalDataConstants;
public class EmptyLineSeparatedRecordReader extends AbstractStreamRecordReader {
- public EmptyLineSeparatedRecordReader(AInputStream inputStream, IExternalIndexer indexer) {
+ public EmptyLineSeparatedRecordReader(AsterixInputStream inputStream, IExternalIndexer indexer) {
super(inputStream, indexer);
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReaderFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReaderFactory.java
index 75d16c5..063ed11 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReaderFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReaderFactory.java
@@ -18,9 +18,9 @@
*/
package org.apache.asterix.external.input.record.reader.stream;
+import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.api.IExternalIndexer;
import org.apache.asterix.external.api.IRecordReader;
-import org.apache.asterix.external.input.stream.AInputStream;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -32,7 +32,7 @@
@Override
public IRecordReader<char[]> createRecordReader(IHyracksTaskContext ctx, int partition)
throws HyracksDataException {
- final Pair<AInputStream, IExternalIndexer> streamAndIndexer = getStreamAndIndexer(ctx, partition);
+ final Pair<AsterixInputStream, IExternalIndexer> streamAndIndexer = getStreamAndIndexer(ctx, partition);
return new EmptyLineSeparatedRecordReader(streamAndIndexer.first, streamAndIndexer.second);
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReader.java
index d55ac87..3089295 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReader.java
@@ -20,8 +20,8 @@
import java.io.IOException;
+import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.api.IExternalIndexer;
-import org.apache.asterix.external.input.stream.AInputStream;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -31,7 +31,7 @@
protected int newlineLength;
protected int recordNumber = 0;
- public LineRecordReader(final boolean hasHeader, final AInputStream stream, final IExternalIndexer indexer)
+ public LineRecordReader(final boolean hasHeader, final AsterixInputStream stream, final IExternalIndexer indexer)
throws HyracksDataException {
super(stream, indexer);
try {
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReaderFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReaderFactory.java
index 68f10f6..4d44001 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReaderFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReaderFactory.java
@@ -18,9 +18,9 @@
*/
package org.apache.asterix.external.input.record.reader.stream;
+import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.api.IExternalIndexer;
import org.apache.asterix.external.api.IRecordReader;
-import org.apache.asterix.external.input.stream.AInputStream;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.hyracks.algebricks.common.utils.Pair;
@@ -36,7 +36,7 @@
throws HyracksDataException {
String quoteString = configuration.get(ExternalDataConstants.KEY_QUOTE);
boolean hasHeader = ExternalDataUtils.hasHeader(configuration);
- Pair<AInputStream, IExternalIndexer> streamAndIndexer = getStreamAndIndexer(ctx, partition);
+ Pair<AsterixInputStream, IExternalIndexer> streamAndIndexer = getStreamAndIndexer(ctx, partition);
if (quoteString != null) {
return new QuotedLineRecordReader(hasHeader, streamAndIndexer.first, streamAndIndexer.second, quoteString);
} else {
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java
index 6266aa2..abd2952 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java
@@ -20,8 +20,8 @@
import java.io.IOException;
+import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.api.IExternalIndexer;
-import org.apache.asterix.external.input.stream.AInputStream;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.ExternalDataExceptionUtils;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -32,8 +32,8 @@
private boolean prevCharEscape;
private boolean inQuote;
- public QuotedLineRecordReader(final boolean hasHeader, final AInputStream stream, final IExternalIndexer indexer,
- final String quoteString) throws HyracksDataException {
+ public QuotedLineRecordReader(final boolean hasHeader, final AsterixInputStream stream,
+ final IExternalIndexer indexer, final String quoteString) throws HyracksDataException {
super(hasHeader, stream, indexer);
if ((quoteString == null) || (quoteString.length() != 1)) {
throw new HyracksDataException(ExternalDataExceptionUtils.incorrectParameterMessage(
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java
index 678dd03..7339bfd 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java
@@ -21,8 +21,8 @@
import java.io.IOException;
import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.api.IExternalIndexer;
-import org.apache.asterix.external.input.stream.AInputStream;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.ExternalDataExceptionUtils;
@@ -35,7 +35,7 @@
private char recordEnd;
private int recordNumber = 0;
- public SemiStructuredRecordReader(AInputStream stream, IExternalIndexer indexer, String recStartString,
+ public SemiStructuredRecordReader(AsterixInputStream stream, IExternalIndexer indexer, String recStartString,
String recEndString) throws AsterixException {
super(stream, indexer);
// set record opening char
@@ -100,9 +100,7 @@
&& inputBuffer[bufferPosn] != ExternalDataConstants.LF
&& inputBuffer[bufferPosn] != ExternalDataConstants.CR) {
// corrupted file. clear the buffer and stop reading
- if (!reader.skipError()) {
- reader.close();
- }
+ reader.reset();
bufferPosn = bufferLength = 0;
throw new IOException("Malformed input stream");
}
@@ -139,7 +137,13 @@
int appendLength = bufferPosn - startPosn;
if (appendLength > 0) {
- record.append(inputBuffer, startPosn, appendLength);
+ try {
+ record.append(inputBuffer, startPosn, appendLength);
+ } catch (IOException e) {
+ reader.reset();
+ bufferPosn = bufferLength = 0;
+ throw new IOException("Malformed input stream");
+ }
}
} while (!hasFinished);
record.endRecord();
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReaderFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReaderFactory.java
index 206ae50..0f50204 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReaderFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReaderFactory.java
@@ -19,9 +19,9 @@
package org.apache.asterix.external.input.record.reader.stream;
import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.api.IExternalIndexer;
import org.apache.asterix.external.api.IRecordReader;
-import org.apache.asterix.external.input.stream.AInputStream;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.api.context.IHyracksTaskContext;
@@ -34,7 +34,7 @@
@Override
public IRecordReader<? extends char[]> createRecordReader(IHyracksTaskContext ctx, int partition)
throws HyracksDataException {
- Pair<AInputStream, IExternalIndexer> streamAndIndexer = getStreamAndIndexer(ctx, partition);
+ Pair<AsterixInputStream, IExternalIndexer> streamAndIndexer = getStreamAndIndexer(ctx, partition);
try {
return new SemiStructuredRecordReader(streamAndIndexer.first, streamAndIndexer.second,
configuration.get(ExternalDataConstants.KEY_RECORD_START),
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java
index be9ce06..e31325a 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java
@@ -21,9 +21,9 @@
import java.io.IOException;
import java.util.List;
-import org.apache.asterix.external.api.IDataFlowController;
import org.apache.asterix.external.api.IRawRecord;
import org.apache.asterix.external.api.IRecordReader;
+import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
import org.apache.asterix.external.input.record.GenericRecord;
import org.apache.asterix.external.util.FeedLogManager;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -96,6 +96,11 @@
}
@Override
- public void setController(IDataFlowController controller) {
+ public void setController(AbstractFeedDataFlowController controller) {
+ }
+
+ @Override
+ public boolean handleException(Throwable th) {
+ return false;
}
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
index 64695b5..f04cdb9 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
@@ -21,9 +21,9 @@
import java.io.IOException;
import java.util.concurrent.LinkedBlockingQueue;
-import org.apache.asterix.external.api.IDataFlowController;
import org.apache.asterix.external.api.IRawRecord;
import org.apache.asterix.external.api.IRecordReader;
+import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
import org.apache.asterix.external.input.record.GenericRecord;
import org.apache.asterix.external.util.FeedLogManager;
@@ -131,6 +131,11 @@
}
@Override
- public void setController(IDataFlowController controller) {
+ public void setController(AbstractFeedDataFlowController controller) {
+ }
+
+ @Override
+ public boolean handleException(Throwable th) {
+ return false;
}
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AInputStreamReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
similarity index 90%
rename from asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AInputStreamReader.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
index bf85330..b1ef892 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AInputStreamReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
@@ -25,28 +25,25 @@
import java.nio.charset.CharsetDecoder;
import java.nio.charset.StandardCharsets;
+import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.FeedLogManager;
-public class AInputStreamReader extends Reader {
- private AInputStream in;
+public class AsterixInputStreamReader extends Reader {
+ private AsterixInputStream in;
private byte[] bytes = new byte[ExternalDataConstants.DEFAULT_BUFFER_SIZE];
private ByteBuffer byteBuffer = ByteBuffer.wrap(bytes);
private CharBuffer charBuffer = CharBuffer.allocate(ExternalDataConstants.DEFAULT_BUFFER_SIZE);
private CharsetDecoder decoder;
private boolean done = false;
- public AInputStreamReader(AInputStream in) {
+ public AsterixInputStreamReader(AsterixInputStream in) {
this.in = in;
this.decoder = StandardCharsets.UTF_8.newDecoder();
this.byteBuffer.flip();
}
- public boolean skipError() throws Exception {
- return in.skipError();
- }
-
public void stop() throws IOException {
try {
in.stop();
@@ -111,4 +108,13 @@
public void close() throws IOException {
in.close();
}
+
+ public boolean handleException(Throwable th) {
+ return in.handleException(th);
+ }
+
+ @Override
+ public void reset() throws IOException {
+ byteBuffer.clear();
+ }
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/BasicInputStream.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/BasicInputStream.java
index 176f5f4..e4eeba3 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/BasicInputStream.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/BasicInputStream.java
@@ -21,10 +21,9 @@
import java.io.IOException;
import java.io.InputStream;
-import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
-import org.apache.asterix.external.util.FeedLogManager;
+import org.apache.asterix.external.api.AsterixInputStream;
-public class BasicInputStream extends AInputStream {
+public class BasicInputStream extends AsterixInputStream {
private final InputStream in;
public BasicInputStream(InputStream in) {
@@ -78,20 +77,12 @@
}
@Override
- public boolean skipError() {
- return false;
- }
-
- @Override
public boolean stop() throws Exception {
return false;
}
@Override
- public void setFeedLogManager(FeedLogManager logManager) {
- }
-
- @Override
- public void setController(AbstractFeedDataFlowController controller) {
+ public boolean handleException(Throwable th) {
+ return false;
}
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/HDFSInputStream.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/HDFSInputStream.java
new file mode 100644
index 0000000..063b8fa
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/HDFSInputStream.java
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.input.stream;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.external.api.AsterixInputStream;
+import org.apache.asterix.external.api.IExternalIndexer;
+import org.apache.asterix.external.api.IIndexingDatasource;
+import org.apache.asterix.external.indexing.ExternalFile;
+import org.apache.asterix.external.input.record.reader.hdfs.EmptyRecordReader;
+import org.apache.asterix.external.provider.ExternalIndexerProvider;
+import org.apache.asterix.external.util.ExternalDataConstants;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapred.FileSplit;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class HDFSInputStream extends AsterixInputStream implements IIndexingDatasource {
+
+ private RecordReader<Object, Text> reader;
+ private Text value = null;
+ private Object key = null;
+ private int currentSplitIndex = 0;
+ private boolean read[];
+ private InputFormat<?, Text> inputFormat;
+ private InputSplit[] inputSplits;
+ private String[] readSchedule;
+ private String nodeName;
+ private JobConf conf;
+ // Indexing variables
+ private final IExternalIndexer indexer;
+ private final List<ExternalFile> snapshot;
+ private final FileSystem hdfs;
+ private int pos = 0;
+
+ @SuppressWarnings("unchecked")
+ public HDFSInputStream(boolean read[], InputSplit[] inputSplits, String[] readSchedule, String nodeName,
+ JobConf conf, Map<String, String> configuration, List<ExternalFile> snapshot)
+ throws IOException, AsterixException {
+ this.read = read;
+ this.inputSplits = inputSplits;
+ this.readSchedule = readSchedule;
+ this.nodeName = nodeName;
+ this.conf = conf;
+ this.inputFormat = conf.getInputFormat();
+ this.reader = new EmptyRecordReader<Object, Text>();
+ this.snapshot = snapshot;
+ this.hdfs = FileSystem.get(conf);
+ nextInputSplit();
+ this.value = new Text();
+ if (snapshot != null) {
+ this.indexer = ExternalIndexerProvider.getIndexer(configuration);
+ if (currentSplitIndex < snapshot.size()) {
+ indexer.reset(this);
+ }
+ } else {
+ this.indexer = null;
+ }
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (value.getLength() < pos) {
+ if (!readMore()) {
+ return -1;
+ }
+ } else if (value.getLength() == pos) {
+ pos++;
+ return ExternalDataConstants.BYTE_LF;
+ }
+ return value.getBytes()[pos++];
+ }
+
+ private int readRecord(byte[] buffer, int offset, int len) {
+ int actualLength = value.getLength() + 1;
+ if ((actualLength - pos) > len) {
+ //copy partial record
+ System.arraycopy(value.getBytes(), pos, buffer, offset, len);
+ pos += len;
+ return len;
+ } else {
+ int numBytes = value.getLength() - pos;
+ System.arraycopy(value.getBytes(), pos, buffer, offset, numBytes);
+ buffer[offset + numBytes] = ExternalDataConstants.LF;
+ pos += numBytes;
+ numBytes++;
+ return numBytes;
+ }
+ }
+
+ @Override
+ public int read(byte[] buffer, int offset, int len) throws IOException {
+ if (value.getLength() > pos) {
+ return readRecord(buffer, offset, len);
+ }
+ if (!readMore()) {
+ return -1;
+ }
+ return readRecord(buffer, offset, len);
+ }
+
+ private boolean readMore() throws IOException {
+ try {
+ pos = 0;
+ return HDFSInputStream.this.hasNext();
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public boolean stop() throws Exception {
+ return false;
+ }
+
+ @Override
+ public boolean handleException(Throwable th) {
+ return false;
+ }
+
+ @Override
+ public void close() throws IOException {
+ reader.close();
+ }
+
+ private boolean hasNext() throws Exception {
+ if (reader.next(key, value)) {
+ return true;
+ }
+ while (nextInputSplit()) {
+ if (reader.next(key, value)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean nextInputSplit() throws IOException {
+ for (; currentSplitIndex < inputSplits.length; currentSplitIndex++) {
+ /**
+ * read all the partitions scheduled to the current node
+ */
+ if (readSchedule[currentSplitIndex].equals(nodeName)) {
+ /**
+ * pick an unread split to read synchronize among
+ * simultaneous partitions in the same machine
+ */
+ synchronized (read) {
+ if (read[currentSplitIndex] == false) {
+ read[currentSplitIndex] = true;
+ } else {
+ continue;
+ }
+ }
+ if (snapshot != null) {
+ String fileName = ((FileSplit) (inputSplits[currentSplitIndex])).getPath().toUri().getPath();
+ FileStatus fileStatus = hdfs.getFileStatus(new Path(fileName));
+ // Skip if not the same file stored in the files snapshot
+ if (fileStatus.getModificationTime() != snapshot.get(currentSplitIndex).getLastModefiedTime()
+ .getTime()) {
+ continue;
+ }
+ }
+
+ reader.close();
+ reader = getRecordReader(currentSplitIndex);
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @SuppressWarnings("unchecked")
+ private RecordReader<Object, Text> getRecordReader(int splitIndex) throws IOException {
+ reader = (RecordReader<Object, Text>) inputFormat.getRecordReader(inputSplits[splitIndex], conf, Reporter.NULL);
+ if (key == null) {
+ key = reader.createKey();
+ value = reader.createValue();
+ }
+ if (indexer != null) {
+ try {
+ indexer.reset(this);
+ } catch (Exception e) {
+ throw new HyracksDataException(e);
+ }
+ }
+ return reader;
+ }
+
+ @Override
+ public IExternalIndexer getIndexer() {
+ return indexer;
+ }
+
+ @Override
+ public List<ExternalFile> getSnapshot() {
+ return snapshot;
+ }
+
+ @Override
+ public int getCurrentSplitIndex() {
+ return currentSplitIndex;
+ }
+
+ @Override
+ public RecordReader<?, ? extends Writable> getReader() {
+ return reader;
+ }
+}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFileSystemInputStream.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFSInputStream.java
similarity index 62%
rename from asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFileSystemInputStream.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFSInputStream.java
index dc6a130..00c1eb7 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFileSystemInputStream.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFSInputStream.java
@@ -18,34 +18,48 @@
*/
package org.apache.asterix.external.input.stream;
+import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.file.Path;
+import java.util.Map;
+import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.FeedLogManager;
import org.apache.asterix.external.util.FileSystemWatcher;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.std.file.FileSplit;
+import org.apache.log4j.Logger;
-public class LocalFileSystemInputStream extends AInputStream {
+public class LocalFSInputStream extends AsterixInputStream {
+
+ private static final Logger LOGGER = Logger.getLogger(LocalFSInputStream.class.getName());
+ private final Path path;
private final FileSystemWatcher watcher;
private FileInputStream in;
private byte lastByte;
+ private File currentFile;
- public LocalFileSystemInputStream(Path inputResource, String expression, boolean isFeed)
- throws HyracksDataException {
- this.watcher = new FileSystemWatcher(inputResource, expression, isFeed);
- watcher.init();
+ public LocalFSInputStream(final FileSplit[] fileSplits, final IHyracksTaskContext ctx,
+ final Map<String, String> configuration, final int partition, final String expression, final boolean isFeed)
+ throws IOException {
+ this.path = fileSplits[partition].getLocalFile().getFile().toPath();
+ this.watcher = new FileSystemWatcher(path, expression, isFeed);
+ this.watcher.init();
}
@Override
public void setFeedLogManager(FeedLogManager logManager) {
+ super.setFeedLogManager(logManager);
watcher.setFeedLogManager(logManager);
}
@Override
public void setController(AbstractFeedDataFlowController controller) {
+ super.setController(controller);
watcher.setController(controller);
}
@@ -76,6 +90,7 @@
in.close();
} finally {
in = null;
+ currentFile = null;
}
}
}
@@ -86,7 +101,8 @@
private boolean advance() throws IOException {
closeFile();
if (watcher.hasNext()) {
- in = new FileInputStream(watcher.next());
+ currentFile = watcher.next();
+ in = new FileInputStream(currentFile);
return true;
}
return false;
@@ -124,14 +140,43 @@
}
@Override
- public boolean skipError() throws Exception {
- advance();
+ public boolean stop() throws Exception {
+ watcher.close();
return true;
}
@Override
- public boolean stop() throws Exception {
- watcher.close();
- return true;
+ public boolean handleException(Throwable th) {
+ if (in == null) {
+ return false;
+ }
+ if (th instanceof IOException) {
+ // TODO: Change from string check to exception type
+ if (th.getCause().getMessage().contains("Malformed input stream")) {
+ if (currentFile != null) {
+ try {
+ logManager.logRecord(currentFile.getAbsolutePath(), "Corrupted input file");
+ } catch (IOException e) {
+ LOGGER.warn("Filed to write to feed log file", e);
+ }
+ LOGGER.warn("Corrupted input file: " + currentFile.getAbsolutePath());
+ }
+ try {
+ advance();
+ return true;
+ } catch (Exception e) {
+ return false;
+ }
+ } else {
+ try {
+ watcher.init();
+ } catch (IOException e) {
+ LOGGER.warn("Failed to initialize watcher during failure recovery", e);
+ return false;
+ }
+ }
+ return true;
+ }
+ return false;
}
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketClientInputStream.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketClientInputStream.java
new file mode 100644
index 0000000..4321bcd
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketClientInputStream.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.input.stream;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.Socket;
+import java.net.UnknownHostException;
+
+import org.apache.asterix.external.api.AsterixInputStream;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+
+public class SocketClientInputStream extends AsterixInputStream {
+
+ private final Socket socket;
+ private InputStream in;
+
+ public SocketClientInputStream(Pair<String, Integer> address) throws UnknownHostException, IOException {
+ this.socket = new Socket(address.first, address.second);
+ this.in = socket.getInputStream();
+ }
+
+ @Override
+ public int read() throws IOException {
+ return in.read();
+ }
+
+ @Override
+ public int read(byte[] buffer, int offset, int length) throws IOException {
+ return in.read(buffer, offset, length);
+ }
+
+ @Override
+ public boolean stop() throws Exception {
+ if (!socket.isClosed()) {
+ socket.close();
+ }
+ return true;
+ }
+
+ @Override
+ public boolean handleException(Throwable th) {
+ return false;
+ }
+}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketServerInputStream.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketServerInputStream.java
index 1c33709..ce1d893 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketServerInputStream.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketServerInputStream.java
@@ -23,16 +23,16 @@
import java.net.ServerSocket;
import java.net.Socket;
-import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
+import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.util.ExternalDataExceptionUtils;
-import org.apache.asterix.external.util.FeedLogManager;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.log4j.Logger;
-public class SocketServerInputStream extends AInputStream {
+public class SocketServerInputStream extends AsterixInputStream {
+ private static final Logger LOGGER = Logger.getLogger(SocketServerInputStream.class.getName());
private ServerSocket server;
private Socket socket;
private InputStream connectionStream;
- private AbstractFeedDataFlowController controller;
public SocketServerInputStream(ServerSocket server) {
this.server = server;
@@ -56,12 +56,6 @@
}
@Override
- public boolean skipError() throws Exception {
- accept();
- return true;
- }
-
- @Override
public int read(byte b[]) throws IOException {
return read(b, 0, b.length);
}
@@ -160,11 +154,13 @@
}
@Override
- public void setFeedLogManager(FeedLogManager logManager) {
- }
-
- @Override
- public void setController(AbstractFeedDataFlowController controller) {
- this.controller = controller;
+ public boolean handleException(Throwable th) {
+ try {
+ accept();
+ } catch (IOException e) {
+ LOGGER.warn("Failed accepting more connections", e);
+ return false;
+ }
+ return true;
}
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java
new file mode 100644
index 0000000..e2afd7b
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.input.stream;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PipedInputStream;
+import java.io.PipedOutputStream;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.asterix.external.api.AsterixInputStream;
+import org.apache.asterix.external.util.TweetGenerator;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+
+public class TwitterFirehoseInputStream extends AsterixInputStream {
+
+ private static final Logger LOGGER = Logger.getLogger(TwitterFirehoseInputStream.class.getName());
+ private final ExecutorService executorService;
+ private final PipedOutputStream outputStream;
+ private final PipedInputStream inputStream;
+ private final DataProvider dataProvider;
+ private boolean started;
+
+ public TwitterFirehoseInputStream(Map<String, String> configuration, IHyracksTaskContext ctx, int partition)
+ throws IOException {
+ executorService = Executors.newCachedThreadPool();
+ outputStream = new PipedOutputStream();
+ inputStream = new PipedInputStream(outputStream);
+ dataProvider = new DataProvider(configuration, partition, outputStream);
+ started = false;
+ }
+
+ @Override
+ public boolean stop() throws IOException {
+ dataProvider.stop();
+ return true;
+ }
+
+ public synchronized void start() {
+ if (!started) {
+ executorService.execute(dataProvider);
+ started = true;
+ }
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (!started) {
+ start();
+ }
+ return inputStream.read();
+ }
+
+ @Override
+ public int read(byte b[], int off, int len) throws IOException {
+ if (!started) {
+ start();
+ }
+ return inputStream.read(b, off, len);
+ }
+
+ @Override
+ public boolean handleException(Throwable th) {
+ return false;
+ }
+
+ private static class DataProvider implements Runnable {
+
+ public static final String KEY_MODE = "mode";
+
+ private final TweetGenerator tweetGenerator;
+ private boolean continuePush = true;
+ private int batchSize;
+ private final Mode mode;
+ private final OutputStream os;
+
+ public static enum Mode {
+ AGGRESSIVE,
+ CONTROLLED
+ }
+
+ public DataProvider(Map<String, String> configuration, int partition, OutputStream os) {
+ this.tweetGenerator = new TweetGenerator(configuration, partition);
+ this.tweetGenerator.registerSubscriber(os);
+ this.os = os;
+ mode = configuration.get(KEY_MODE) != null ? Mode.valueOf(configuration.get(KEY_MODE).toUpperCase())
+ : Mode.AGGRESSIVE;
+ switch (mode) {
+ case CONTROLLED:
+ String tpsValue = configuration.get(TweetGenerator.KEY_TPS);
+ if (tpsValue == null) {
+ throw new IllegalArgumentException("TPS value not configured. use tps=<value>");
+ }
+ batchSize = Integer.parseInt(tpsValue);
+ break;
+ case AGGRESSIVE:
+ batchSize = 5000;
+ break;
+ }
+ }
+
+ @Override
+ public void run() {
+ boolean moreData = true;
+ long startBatch;
+ long endBatch;
+ while (true) {
+ try {
+ while (moreData && continuePush) {
+ switch (mode) {
+ case AGGRESSIVE:
+ moreData = tweetGenerator.generateNextBatch(batchSize);
+ break;
+ case CONTROLLED:
+ startBatch = System.currentTimeMillis();
+ moreData = tweetGenerator.generateNextBatch(batchSize);
+ endBatch = System.currentTimeMillis();
+ if ((endBatch - startBatch) < 1000) {
+ Thread.sleep(1000 - (endBatch - startBatch));
+ }
+ break;
+ }
+ }
+ os.close();
+ break;
+ } catch (Exception e) {
+ if (LOGGER.isLoggable(Level.WARNING)) {
+ LOGGER.warning("Exception in adapter " + e.getMessage());
+ }
+ }
+ }
+ }
+
+ public void stop() {
+ continuePush = false;
+ }
+ }
+}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamProviderFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java
similarity index 89%
rename from asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamProviderFactory.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java
index 54ee780..85d0e41 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamProviderFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java
@@ -19,31 +19,33 @@
package org.apache.asterix.external.input.stream.factory;
import java.io.File;
+import java.io.IOException;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.external.api.IInputStreamProvider;
-import org.apache.asterix.external.api.IInputStreamProviderFactory;
+import org.apache.asterix.external.api.AsterixInputStream;
+import org.apache.asterix.external.api.IInputStreamFactory;
import org.apache.asterix.external.api.INodeResolver;
import org.apache.asterix.external.api.INodeResolverFactory;
-import org.apache.asterix.external.input.stream.provider.LocalFSInputStreamProvider;
+import org.apache.asterix.external.input.stream.LocalFSInputStream;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.asterix.external.util.FeedUtils;
import org.apache.asterix.external.util.NodeResolverFactory;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.dataflow.std.file.FileSplit;
-public class LocalFSInputStreamProviderFactory implements IInputStreamProviderFactory {
+public class LocalFSInputStreamFactory implements IInputStreamFactory {
private static final long serialVersionUID = 1L;
protected static final INodeResolver DEFAULT_NODE_RESOLVER = new NodeResolverFactory().createNodeResolver();
- protected static final Logger LOGGER = Logger.getLogger(LocalFSInputStreamProviderFactory.class.getName());
+ protected static final Logger LOGGER = Logger.getLogger(LocalFSInputStreamFactory.class.getName());
protected static INodeResolver nodeResolver;
protected Map<String, String> configuration;
protected FileSplit[] inputFileSplits;
@@ -54,8 +56,12 @@
private transient AlgebricksAbsolutePartitionConstraint constraints;
@Override
- public IInputStreamProvider createInputStreamProvider(IHyracksTaskContext ctx, int partition) {
- return new LocalFSInputStreamProvider(inputFileSplits, ctx, configuration, partition, expression, isFeed);
+ public AsterixInputStream createInputStream(IHyracksTaskContext ctx, int partition) throws HyracksDataException {
+ try {
+ return new LocalFSInputStream(inputFileSplits, ctx, configuration, partition, expression, isFeed);
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
}
@Override
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamProviderFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamFactory.java
similarity index 83%
rename from asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamProviderFactory.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamFactory.java
index 5e84123..1eb760e 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamProviderFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamFactory.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.external.input.stream.factory;
+import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
@@ -25,10 +26,10 @@
import java.util.Map;
import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.api.IExternalDataSourceFactory;
-import org.apache.asterix.external.api.IInputStreamProvider;
-import org.apache.asterix.external.api.IInputStreamProviderFactory;
-import org.apache.asterix.external.input.stream.provider.SocketClientInputStreamProvider;
+import org.apache.asterix.external.api.IInputStreamFactory;
+import org.apache.asterix.external.input.stream.SocketClientInputStream;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.http.impl.conn.SystemDefaultDnsResolver;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
@@ -36,7 +37,7 @@
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-public class SocketClientInputStreamProviderFactory implements IInputStreamProviderFactory {
+public class SocketClientInputStreamFactory implements IInputStreamFactory {
private static final long serialVersionUID = 1L;
private transient AlgebricksAbsolutePartitionConstraint clusterLocations;
@@ -73,8 +74,11 @@
}
@Override
- public IInputStreamProvider createInputStreamProvider(IHyracksTaskContext ctx, int partition)
- throws HyracksDataException {
- return new SocketClientInputStreamProvider(sockets.get(partition));
+ public AsterixInputStream createInputStream(IHyracksTaskContext ctx, int partition) throws HyracksDataException {
+ try {
+ return new SocketClientInputStream(sockets.get(partition));
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
}
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamProviderFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java
similarity index 91%
rename from asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamProviderFactory.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java
index a301c1a..f63b895 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamProviderFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java
@@ -28,9 +28,9 @@
import java.util.Set;
import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.external.api.IInputStreamProvider;
-import org.apache.asterix.external.api.IInputStreamProviderFactory;
-import org.apache.asterix.external.input.stream.provider.SocketServerInputStreamProvider;
+import org.apache.asterix.external.api.AsterixInputStream;
+import org.apache.asterix.external.api.IInputStreamFactory;
+import org.apache.asterix.external.input.stream.SocketServerInputStream;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.om.util.AsterixRuntimeUtil;
import org.apache.commons.lang3.StringUtils;
@@ -39,7 +39,7 @@
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-public class SocketServerInputStreamProviderFactory implements IInputStreamProviderFactory {
+public class SocketServerInputStreamFactory implements IInputStreamFactory {
private static final long serialVersionUID = 1L;
private List<Pair<String, Integer>> sockets;
@@ -104,13 +104,13 @@
}
@Override
- public synchronized IInputStreamProvider createInputStreamProvider(IHyracksTaskContext ctx, int partition)
+ public synchronized AsterixInputStream createInputStream(IHyracksTaskContext ctx, int partition)
throws HyracksDataException {
try {
Pair<String, Integer> socket = sockets.get(partition);
ServerSocket server;
server = new ServerSocket(socket.second);
- return new SocketServerInputStreamProvider(server);
+ return new SocketServerInputStream(server);
} catch (IOException e) {
throw new HyracksDataException(e);
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamProviderFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java
similarity index 85%
rename from asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamProviderFactory.java
rename to asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java
index 7b09ade..8052822 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamProviderFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java
@@ -18,13 +18,14 @@
*/
package org.apache.asterix.external.input.stream.factory;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-import org.apache.asterix.external.api.IInputStreamProvider;
-import org.apache.asterix.external.api.IInputStreamProviderFactory;
-import org.apache.asterix.external.input.stream.provider.TwitterFirehoseInputStreamProvider;
+import org.apache.asterix.external.api.AsterixInputStream;
+import org.apache.asterix.external.api.IInputStreamFactory;
+import org.apache.asterix.external.input.stream.TwitterFirehoseInputStream;
import org.apache.asterix.om.util.AsterixClusterProperties;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.api.context.IHyracksTaskContext;
@@ -36,7 +37,7 @@
* configurable rate measured in terms of TPS (tweets/second). The stream of
* tweets lasts for a configurable duration (measured in seconds).
*/
-public class TwitterFirehoseStreamProviderFactory implements IInputStreamProviderFactory {
+public class TwitterFirehoseStreamFactory implements IInputStreamFactory {
private static final long serialVersionUID = 1L;
@@ -91,8 +92,11 @@
}
@Override
- public IInputStreamProvider createInputStreamProvider(IHyracksTaskContext ctx, int partition)
- throws HyracksDataException {
- return new TwitterFirehoseInputStreamProvider(configuration, ctx, partition);
+ public AsterixInputStream createInputStream(IHyracksTaskContext ctx, int partition) throws HyracksDataException {
+ try {
+ return new TwitterFirehoseInputStream(configuration, ctx, partition);
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
}
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/HDFSInputStreamProvider.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/HDFSInputStreamProvider.java
deleted file mode 100644
index e1ab331..0000000
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/HDFSInputStreamProvider.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.input.stream.provider;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.asterix.external.api.IInputStreamProvider;
-import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
-import org.apache.asterix.external.indexing.ExternalFile;
-import org.apache.asterix.external.input.record.reader.hdfs.HDFSRecordReader;
-import org.apache.asterix.external.input.stream.AInputStream;
-import org.apache.asterix.external.provider.ExternalIndexerProvider;
-import org.apache.asterix.external.util.ExternalDataConstants;
-import org.apache.asterix.external.util.FeedLogManager;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hadoop.mapred.JobConf;
-
-public class HDFSInputStreamProvider<K> extends HDFSRecordReader<K, Text> implements IInputStreamProvider {
-
- public HDFSInputStreamProvider(boolean read[], InputSplit[] inputSplits, String[] readSchedule, String nodeName,
- JobConf conf, Map<String, String> configuration, List<ExternalFile> snapshot) throws Exception {
- super(read, inputSplits, readSchedule, nodeName, conf, snapshot,
- snapshot == null ? null : ExternalIndexerProvider.getIndexer(configuration));
- value = new Text();
- if (snapshot != null) {
- if (currentSplitIndex < snapshot.size()) {
- indexer.reset(this);
- }
- }
- }
-
- @Override
- public AInputStream getInputStream() {
- return new HDFSInputStream();
- }
-
- private class HDFSInputStream extends AInputStream {
- int pos = 0;
-
- @Override
- public int read() throws IOException {
- if (value.getLength() < pos) {
- if (!readMore()) {
- return -1;
- }
- } else if (value.getLength() == pos) {
- pos++;
- return ExternalDataConstants.BYTE_LF;
- }
- return value.getBytes()[pos++];
- }
-
- private int readRecord(byte[] buffer, int offset, int len) {
- int actualLength = value.getLength() + 1;
- if ((actualLength - pos) > len) {
- //copy partial record
- System.arraycopy(value.getBytes(), pos, buffer, offset, len);
- pos += len;
- return len;
- } else {
- int numBytes = value.getLength() - pos;
- System.arraycopy(value.getBytes(), pos, buffer, offset, numBytes);
- buffer[offset + numBytes] = ExternalDataConstants.LF;
- pos += numBytes;
- numBytes++;
- return numBytes;
- }
- }
-
- @Override
- public int read(byte[] buffer, int offset, int len) throws IOException {
- if (value.getLength() > pos) {
- return readRecord(buffer, offset, len);
- }
- if (!readMore()) {
- return -1;
- }
- return readRecord(buffer, offset, len);
- }
-
- private boolean readMore() throws IOException {
- try {
- pos = 0;
- return HDFSInputStreamProvider.this.hasNext();
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public boolean skipError() throws Exception {
- return true;
- }
-
- @Override
- public boolean stop() throws Exception {
- return false;
- }
-
- @Override
- public void setFeedLogManager(FeedLogManager logManager) {
- }
-
- @Override
- public void setController(AbstractFeedDataFlowController controller) {
- }
- }
-}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/LocalFSInputStreamProvider.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/LocalFSInputStreamProvider.java
deleted file mode 100644
index fbe6035..0000000
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/LocalFSInputStreamProvider.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.input.stream.provider;
-
-import java.nio.file.Path;
-import java.util.Map;
-
-import org.apache.asterix.external.api.IInputStreamProvider;
-import org.apache.asterix.external.input.stream.AInputStream;
-import org.apache.asterix.external.input.stream.LocalFileSystemInputStream;
-import org.apache.asterix.external.util.FeedLogManager;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
-
-public class LocalFSInputStreamProvider implements IInputStreamProvider {
-
- private final String expression;
- private final boolean isFeed;
- private final Path path;
- private FeedLogManager feedLogManager;
-
- public LocalFSInputStreamProvider(final FileSplit[] fileSplits, final IHyracksTaskContext ctx,
- final Map<String, String> configuration, final int partition, final String expression,
- final boolean isFeed) {
- this.expression = expression;
- this.isFeed = isFeed;
- this.path = fileSplits[partition].getLocalFile().getFile().toPath();
- }
-
- @Override
- public AInputStream getInputStream() throws HyracksDataException {
- final LocalFileSystemInputStream stream = new LocalFileSystemInputStream(path, expression, isFeed);
- stream.setFeedLogManager(feedLogManager);
- return stream;
- }
-
- @Override
- public void setFeedLogManager(final FeedLogManager feedLogManager) {
- this.feedLogManager = feedLogManager;
- }
-}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketClientInputStreamProvider.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketClientInputStreamProvider.java
deleted file mode 100644
index f842638..0000000
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketClientInputStreamProvider.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.input.stream.provider;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.Socket;
-
-import org.apache.asterix.external.api.IInputStreamProvider;
-import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
-import org.apache.asterix.external.input.stream.AInputStream;
-import org.apache.asterix.external.util.FeedLogManager;
-import org.apache.hyracks.algebricks.common.utils.Pair;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.log4j.Logger;
-
-public class SocketClientInputStreamProvider implements IInputStreamProvider {
-
- private static final Logger LOGGER = Logger.getLogger(SocketClientInputStreamProvider.class.getName());
- private final Socket socket;
-
- public SocketClientInputStreamProvider(Pair<String, Integer> ipAndPort) throws HyracksDataException {
- try {
- socket = new Socket(ipAndPort.first, ipAndPort.second);
- } catch (IOException e) {
- LOGGER.error(
- "Problem in creating socket against host " + ipAndPort.first + " on the port " + ipAndPort.second,
- e);
- throw new HyracksDataException(e);
- }
- }
-
- @Override
- public AInputStream getInputStream() throws HyracksDataException {
- InputStream in;
- try {
- in = socket.getInputStream();
- } catch (IOException e) {
- throw new HyracksDataException(e);
- }
- return new AInputStream() {
- @Override
- public int read() throws IOException {
- throw new IOException("method not supported. use read(byte[] buffer, int offset, int length) instead");
- }
-
- @Override
- public int read(byte[] buffer, int offset, int length) throws IOException {
- return in.read(buffer, offset, length);
- }
-
- @Override
- public boolean stop() throws Exception {
- if (!socket.isClosed()) {
- try {
- in.close();
- } finally {
- socket.close();
- }
- }
- return true;
- }
-
- @Override
- public boolean skipError() throws Exception {
- return false;
- }
-
- @Override
- public void setFeedLogManager(FeedLogManager logManager) {
- }
-
- @Override
- public void setController(AbstractFeedDataFlowController controller) {
- }
- };
- }
-
- @Override
- public void setFeedLogManager(FeedLogManager feedLogManager) {
- }
-}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketServerInputStreamProvider.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketServerInputStreamProvider.java
deleted file mode 100644
index 64f0342..0000000
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketServerInputStreamProvider.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.input.stream.provider;
-
-import java.net.ServerSocket;
-
-import org.apache.asterix.external.api.IInputStreamProvider;
-import org.apache.asterix.external.input.stream.AInputStream;
-import org.apache.asterix.external.input.stream.SocketServerInputStream;
-import org.apache.asterix.external.util.FeedLogManager;
-
-public class SocketServerInputStreamProvider implements IInputStreamProvider {
- private final ServerSocket server;
-
- public SocketServerInputStreamProvider(ServerSocket server) {
- this.server = server;
- }
-
- @Override
- public AInputStream getInputStream() {
- return new SocketServerInputStream(server);
- }
-
- @Override
- public void setFeedLogManager(FeedLogManager feedLogManager) {
- }
-}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/TwitterFirehoseInputStreamProvider.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/TwitterFirehoseInputStreamProvider.java
deleted file mode 100644
index a979262..0000000
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/TwitterFirehoseInputStreamProvider.java
+++ /dev/null
@@ -1,202 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.input.stream.provider;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.PipedInputStream;
-import java.io.PipedOutputStream;
-import java.util.Map;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import org.apache.asterix.external.api.IInputStreamProvider;
-import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
-import org.apache.asterix.external.input.stream.AInputStream;
-import org.apache.asterix.external.util.FeedLogManager;
-import org.apache.asterix.external.util.TweetGenerator;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public class TwitterFirehoseInputStreamProvider implements IInputStreamProvider {
-
- private static final Logger LOGGER = Logger.getLogger(TwitterFirehoseInputStreamProvider.class.getName());
-
- private final ExecutorService executorService;
-
- private final PipedOutputStream outputStream;
-
- private final PipedInputStream inputStream;
-
- private final TwitterServer twitterServer;
-
- public TwitterFirehoseInputStreamProvider(Map<String, String> configuration, IHyracksTaskContext ctx, int partition)
- throws HyracksDataException {
- try {
- executorService = Executors.newCachedThreadPool();
- outputStream = new PipedOutputStream();
- inputStream = new PipedInputStream(outputStream);
- twitterServer = new TwitterServer(configuration, partition, outputStream, executorService, inputStream);
- } catch (IOException e) {
- throw new HyracksDataException(e);
- }
- }
-
- @Override
- public AInputStream getInputStream() {
- return twitterServer;
- }
-
- private static class TwitterServer extends AInputStream {
- private final DataProvider dataProvider;
- private final ExecutorService executorService;
- private final InputStream in;
- private boolean started;
-
- public TwitterServer(Map<String, String> configuration, int partition, OutputStream os,
- ExecutorService executorService, InputStream in) {
- dataProvider = new DataProvider(configuration, partition, os);
- this.executorService = executorService;
- this.in = in;
- this.started = false;
- }
-
- @Override
- public boolean stop() throws IOException {
- dataProvider.stop();
- return true;
- }
-
- public synchronized void start() {
- if (!started) {
- executorService.execute(dataProvider);
- started = true;
- }
- }
-
- @Override
- public boolean skipError() throws Exception {
- return false;
- }
-
- @Override
- public int read() throws IOException {
- if (!started) {
- start();
- }
- return in.read();
- }
-
- @Override
- public int read(byte b[], int off, int len) throws IOException {
- if (!started) {
- start();
- started = true;
- }
- return in.read(b, off, len);
- }
-
- @Override
- public void setFeedLogManager(FeedLogManager logManager) {
- }
-
- @Override
- public void setController(AbstractFeedDataFlowController controller) {
- }
- }
-
- private static class DataProvider implements Runnable {
-
- public static final String KEY_MODE = "mode";
-
- private final TweetGenerator tweetGenerator;
- private boolean continuePush = true;
- private int batchSize;
- private final Mode mode;
- private final OutputStream os;
-
- public static enum Mode {
- AGGRESSIVE,
- CONTROLLED
- }
-
- public DataProvider(Map<String, String> configuration, int partition, OutputStream os) {
- this.tweetGenerator = new TweetGenerator(configuration, partition);
- this.tweetGenerator.registerSubscriber(os);
- this.os = os;
- mode = configuration.get(KEY_MODE) != null ? Mode.valueOf(configuration.get(KEY_MODE).toUpperCase())
- : Mode.AGGRESSIVE;
- switch (mode) {
- case CONTROLLED:
- String tpsValue = configuration.get(TweetGenerator.KEY_TPS);
- if (tpsValue == null) {
- throw new IllegalArgumentException("TPS value not configured. use tps=<value>");
- }
- batchSize = Integer.parseInt(tpsValue);
- break;
- case AGGRESSIVE:
- batchSize = 5000;
- break;
- }
- }
-
- @Override
- public void run() {
- boolean moreData = true;
- long startBatch;
- long endBatch;
- while (true) {
- try {
- while (moreData && continuePush) {
- switch (mode) {
- case AGGRESSIVE:
- moreData = tweetGenerator.generateNextBatch(batchSize);
- break;
- case CONTROLLED:
- startBatch = System.currentTimeMillis();
- moreData = tweetGenerator.generateNextBatch(batchSize);
- endBatch = System.currentTimeMillis();
- if ((endBatch - startBatch) < 1000) {
- Thread.sleep(1000 - (endBatch - startBatch));
- }
- break;
- }
- }
- os.close();
- break;
- } catch (Exception e) {
- if (LOGGER.isLoggable(Level.WARNING)) {
- LOGGER.warning("Exception in adapter " + e.getMessage());
- }
- }
- }
- }
-
- public void stop() {
- continuePush = false;
- }
- }
-
- @Override
- public void setFeedLogManager(FeedLogManager feedLogManager) {
- }
-}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorDescriptor.java b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorDescriptor.java
index 8475e45..9485b77 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorDescriptor.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorDescriptor.java
@@ -19,11 +19,9 @@
package org.apache.asterix.external.operators;
import java.util.Map;
-import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
-import org.apache.asterix.external.feed.api.IFeedLifecycleListener.ConnectionLocation;
import org.apache.asterix.external.feed.api.IFeedManager;
import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
import org.apache.asterix.external.feed.api.IFeedSubscriptionManager;
@@ -52,7 +50,7 @@
private static final long serialVersionUID = 1L;
private static final Logger LOGGER = Logger.getLogger(FeedCollectOperatorDescriptor.class.getName());
- /** The type associated with the ADM data output from the feed adaptor */
+ /** The type associated with the ADM data output from (the feed adapter OR the compute operator) */
private final IAType outputType;
/** unique identifier for a feed instance. */
@@ -67,12 +65,12 @@
/** The source feed from which the feed derives its data from. **/
private final FeedId sourceFeedId;
- /** The subscription location at which the recipient feed receives tuples from the source feed **/
- private final ConnectionLocation subscriptionLocation;
+ /** The subscription location at which the recipient feed receives tuples from the source feed {SOURCE_FEED_INTAKE_STAGE , SOURCE_FEED_COMPUTE_STAGE} **/
+ private final FeedRuntimeType subscriptionLocation;
public FeedCollectOperatorDescriptor(JobSpecification spec, FeedConnectionId feedConnectionId, FeedId sourceFeedId,
ARecordType atype, RecordDescriptor rDesc, Map<String, String> feedPolicyProperties,
- ConnectionLocation subscriptionLocation) {
+ FeedRuntimeType subscriptionLocation) {
super(spec, 0, 1);
recordDescriptors[0] = rDesc;
this.outputType = atype;
@@ -85,45 +83,25 @@
@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions)
- throws HyracksDataException {
+ throws HyracksDataException {
IAsterixAppRuntimeContext runtimeCtx = (IAsterixAppRuntimeContext) ctx.getJobletContext()
.getApplicationContext().getApplicationObject();
this.subscriptionManager = ((IFeedManager) runtimeCtx.getFeedManager()).getFeedSubscriptionManager();
ISubscribableRuntime sourceRuntime = null;
- IOperatorNodePushable nodePushable = null;
+ SubscribableFeedRuntimeId feedSubscribableRuntimeId = new SubscribableFeedRuntimeId(sourceFeedId,
+ subscriptionLocation, partition);
switch (subscriptionLocation) {
- case SOURCE_FEED_INTAKE_STAGE:
- try {
- SubscribableFeedRuntimeId feedSubscribableRuntimeId = new SubscribableFeedRuntimeId(sourceFeedId,
- FeedRuntimeType.INTAKE, partition);
- sourceRuntime = getIntakeRuntime(feedSubscribableRuntimeId);
- if (sourceRuntime == null) {
- throw new HyracksDataException(
- "Source intake task not found for source feed id " + sourceFeedId);
- }
- nodePushable = new FeedCollectOperatorNodePushable(ctx, sourceFeedId, connectionId,
- feedPolicyProperties, partition, nPartitions, sourceRuntime);
-
- } catch (Exception exception) {
- if (LOGGER.isLoggable(Level.SEVERE)) {
- LOGGER.severe("Initialization of the feed adaptor failed with exception " + exception);
- }
- throw new HyracksDataException("Initialization of the feed adapter failed", exception);
- }
+ case INTAKE:
+ sourceRuntime = getIntakeRuntime(feedSubscribableRuntimeId);
break;
- case SOURCE_FEED_COMPUTE_STAGE:
- SubscribableFeedRuntimeId feedSubscribableRuntimeId = new SubscribableFeedRuntimeId(sourceFeedId,
- FeedRuntimeType.COMPUTE, partition);
+ case COMPUTE:
sourceRuntime = subscriptionManager.getSubscribableRuntime(feedSubscribableRuntimeId);
- if (sourceRuntime == null) {
- throw new HyracksDataException("Source compute task not found for source feed id " + sourceFeedId
- + " " + FeedRuntimeType.COMPUTE + "[" + partition + "]");
- }
- nodePushable = new FeedCollectOperatorNodePushable(ctx, sourceFeedId, connectionId,
- feedPolicyProperties, partition, nPartitions, sourceRuntime);
break;
+ default:
+ throw new HyracksDataException("Can't subscirbe to FeedRuntime with Type: " + subscriptionLocation);
}
- return nodePushable;
+ return new FeedCollectOperatorNodePushable(ctx, sourceFeedId, connectionId, feedPolicyProperties, partition,
+ nPartitions, sourceRuntime);
}
public FeedConnectionId getFeedConnectionId() {
@@ -150,7 +128,7 @@
return (IngestionRuntime) subscriptionManager.getSubscribableRuntime(subscribableRuntimeId);
}
- public ConnectionLocation getSubscriptionLocation() {
+ public FeedRuntimeType getSubscriptionLocation() {
return subscriptionLocation;
}
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorNodePushable.java b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorNodePushable.java
index 7901f03..87e1edb 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorNodePushable.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorNodePushable.java
@@ -46,10 +46,9 @@
import org.apache.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable;
/**
- * The runtime for @see{FeedIntakeOperationDescriptor}
+ * The first operator in a collect job in a feed.
*/
public class FeedCollectOperatorNodePushable extends AbstractUnaryOutputSourceOperatorNodePushable {
-
private static Logger LOGGER = Logger.getLogger(FeedCollectOperatorNodePushable.class.getName());
private final int partition;
@@ -74,19 +73,16 @@
this.connectionId = feedConnectionId;
this.sourceRuntime = sourceRuntime;
this.feedPolicy = feedPolicy;
- policyAccessor = new FeedPolicyAccessor(feedPolicy);
- IAsterixAppRuntimeContext runtimeCtx = (IAsterixAppRuntimeContext) ctx.getJobletContext()
- .getApplicationContext().getApplicationObject();
- this.feedManager = (IFeedManager) runtimeCtx.getFeedManager();
+ this.policyAccessor = new FeedPolicyAccessor(feedPolicy);
+ this.feedManager = (IFeedManager) ((IAsterixAppRuntimeContext) ctx.getJobletContext().getApplicationContext()
+ .getApplicationObject()).getFeedManager();
}
@Override
public void initialize() throws HyracksDataException {
try {
outputRecordDescriptor = recordDesc;
- FeedRuntimeType sourceRuntimeType = ((SubscribableFeedRuntimeId) sourceRuntime.getRuntimeId())
- .getFeedRuntimeType();
- switch (sourceRuntimeType) {
+ switch (((SubscribableFeedRuntimeId) sourceRuntime.getRuntimeId()).getFeedRuntimeType()) {
case INTAKE:
handleCompleteConnection();
break;
@@ -94,7 +90,8 @@
handlePartialConnection();
break;
default:
- throw new IllegalStateException("Invalid source type " + sourceRuntimeType);
+ throw new IllegalStateException("Invalid source type "
+ + ((SubscribableFeedRuntimeId) sourceRuntime.getRuntimeId()).getFeedRuntimeType());
}
State state = collectRuntime.waitTillCollectionOver();
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractDataParser.java b/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractDataParser.java
index cfd1b9c..2517166 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractDataParser.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractDataParser.java
@@ -114,7 +114,7 @@
.getSerializerDeserializer(BuiltinType.ADOUBLE);
@SuppressWarnings("unchecked")
protected ISerializerDeserializer<AString> stringSerde = AqlSerializerDeserializerProvider.INSTANCE
- .getSerializerDeserializer(BuiltinType.ASTRING);
+ .getAStringSerializerDeserializer();
@SuppressWarnings("unchecked")
protected ISerializerDeserializer<ABinary> binarySerde = AqlSerializerDeserializerProvider.INSTANCE
.getSerializerDeserializer(BuiltinType.ABINARY);
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DataflowControllerProvider.java b/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DataflowControllerProvider.java
index 159ea73..d362201 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DataflowControllerProvider.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DataflowControllerProvider.java
@@ -22,12 +22,12 @@
import java.util.Map;
import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.api.IDataFlowController;
import org.apache.asterix.external.api.IDataParserFactory;
import org.apache.asterix.external.api.IExternalDataSourceFactory;
import org.apache.asterix.external.api.IIndexingDatasource;
-import org.apache.asterix.external.api.IInputStreamProvider;
-import org.apache.asterix.external.api.IInputStreamProviderFactory;
+import org.apache.asterix.external.api.IInputStreamFactory;
import org.apache.asterix.external.api.IRecordDataParser;
import org.apache.asterix.external.api.IRecordDataParserFactory;
import org.apache.asterix.external.api.IRecordReader;
@@ -44,7 +44,6 @@
import org.apache.asterix.external.dataflow.IndexingDataFlowController;
import org.apache.asterix.external.dataflow.RecordDataFlowController;
import org.apache.asterix.external.dataflow.StreamDataFlowController;
-import org.apache.asterix.external.input.stream.AInputStream;
import org.apache.asterix.external.parser.RecordWithMetadataParser;
import org.apache.asterix.external.util.DataflowUtils;
import org.apache.asterix.external.util.ExternalDataUtils;
@@ -106,17 +105,15 @@
recordReader, 1);
}
case STREAM:
- IInputStreamProviderFactory streamProviderFactory = (IInputStreamProviderFactory) dataSourceFactory;
- IInputStreamProvider streamProvider = streamProviderFactory.createInputStreamProvider(ctx,
- partition);
+ IInputStreamFactory streamFactory = (IInputStreamFactory) dataSourceFactory;
+ AsterixInputStream stream = streamFactory.createInputStream(ctx, partition);
IStreamDataParserFactory streamParserFactory = (IStreamDataParserFactory) dataParserFactory;
IStreamDataParser streamParser = streamParserFactory.createInputStreamParser(ctx, partition);
- AInputStream inputStream = streamProvider.getInputStream();
- streamParser.setInputStream(inputStream);
+ streamParser.setInputStream(stream);
if (isFeed) {
return new FeedStreamDataFlowController(ctx,
(FeedTupleForwarder) DataflowUtils.getTupleForwarder(configuration, feedLogManager),
- feedLogManager, FeedUtils.getNumOfFields(configuration), streamParser, inputStream);
+ feedLogManager, FeedUtils.getNumOfFields(configuration), streamParser, stream);
} else {
return new StreamDataFlowController(ctx, DataflowUtils.getTupleForwarder(configuration, null),
streamParser);
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java b/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
index e9307e5..f8d64e0 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
@@ -23,7 +23,7 @@
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.external.api.IExternalDataSourceFactory;
import org.apache.asterix.external.api.IExternalDataSourceFactory.DataSourceType;
-import org.apache.asterix.external.api.IInputStreamProviderFactory;
+import org.apache.asterix.external.api.IInputStreamFactory;
import org.apache.asterix.external.api.IRecordReaderFactory;
import org.apache.asterix.external.input.HDFSDataSourceFactory;
import org.apache.asterix.external.input.record.reader.RecordWithPKTestReaderFactory;
@@ -33,9 +33,9 @@
import org.apache.asterix.external.input.record.reader.stream.LineRecordReaderFactory;
import org.apache.asterix.external.input.record.reader.stream.SemiStructuredRecordReaderFactory;
import org.apache.asterix.external.input.record.reader.twitter.TwitterRecordReaderFactory;
-import org.apache.asterix.external.input.stream.factory.LocalFSInputStreamProviderFactory;
-import org.apache.asterix.external.input.stream.factory.SocketServerInputStreamProviderFactory;
-import org.apache.asterix.external.input.stream.factory.TwitterFirehoseStreamProviderFactory;
+import org.apache.asterix.external.input.stream.factory.LocalFSInputStreamFactory;
+import org.apache.asterix.external.input.stream.factory.SocketServerInputStreamFactory;
+import org.apache.asterix.external.input.stream.factory.TwitterFirehoseStreamFactory;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.ExternalDataUtils;
@@ -53,9 +53,9 @@
}
}
- public static IInputStreamProviderFactory getInputStreamFactory(String streamSource,
+ public static IInputStreamFactory getInputStreamFactory(String streamSource,
Map<String, String> configuration) throws AsterixException {
- IInputStreamProviderFactory streamSourceFactory;
+ IInputStreamFactory streamSourceFactory;
if (ExternalDataUtils.isExternal(streamSource)) {
String dataverse = ExternalDataUtils.getDataverse(configuration);
streamSourceFactory = ExternalDataUtils.createExternalInputStreamFactory(dataverse, streamSource);
@@ -65,17 +65,17 @@
streamSourceFactory = new HDFSDataSourceFactory();
break;
case ExternalDataConstants.STREAM_LOCAL_FILESYSTEM:
- streamSourceFactory = new LocalFSInputStreamProviderFactory();
+ streamSourceFactory = new LocalFSInputStreamFactory();
break;
case ExternalDataConstants.STREAM_SOCKET:
case ExternalDataConstants.ALIAS_SOCKET_ADAPTER:
- streamSourceFactory = new SocketServerInputStreamProviderFactory();
+ streamSourceFactory = new SocketServerInputStreamFactory();
break;
case ExternalDataConstants.STREAM_SOCKET_CLIENT:
- streamSourceFactory = new SocketServerInputStreamProviderFactory();
+ streamSourceFactory = new SocketServerInputStreamFactory();
break;
case ExternalDataConstants.ALIAS_TWITTER_FIREHOSE_ADAPTER:
- streamSourceFactory = new TwitterFirehoseStreamProviderFactory();
+ streamSourceFactory = new TwitterFirehoseStreamFactory();
break;
default:
throw new AsterixException("unknown input stream factory");
@@ -90,7 +90,7 @@
return ExternalDataUtils.createExternalRecordReaderFactory(configuration);
}
String parser = configuration.get(ExternalDataConstants.KEY_PARSER);
- IInputStreamProviderFactory inputStreamFactory;
+ IInputStreamFactory inputStreamFactory;
switch (parser) {
case ExternalDataConstants.FORMAT_ADM:
case ExternalDataConstants.FORMAT_JSON:
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index 0e68698..a02152b 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -206,8 +206,9 @@
* Size default values
*/
public static final int DEFAULT_BUFFER_SIZE = 4096;
- public static final int DEFAULT_BUFFER_INCREMENT = 4096;
+ public static final int DEFAULT_BUFFER_INCREMENT = 2048;
public static final int DEFAULT_QUEUE_SIZE = 64;
+ public static final int MAX_RECORD_SIZE = 32000000;
/**
* Expected parameter values
@@ -226,4 +227,6 @@
public static final String FORMAT_CSV = "csv";
public static final String TEST_RECORD_WITH_PK = "test-record-with-pk";
+ public static final String ERROR_LARGE_RECORD = "Record is too large";
+ public static final String ERROR_PARSE_RECORD = "Parser failed to parse record";
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 32139f1..42fe8bf 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -25,7 +25,7 @@
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.external.api.IDataParserFactory;
import org.apache.asterix.external.api.IExternalDataSourceFactory.DataSourceType;
-import org.apache.asterix.external.api.IInputStreamProviderFactory;
+import org.apache.asterix.external.api.IInputStreamFactory;
import org.apache.asterix.external.api.IRecordReaderFactory;
import org.apache.asterix.external.library.ExternalLibraryManager;
import org.apache.asterix.om.types.ARecordType;
@@ -116,13 +116,13 @@
return aString.trim().split(FeedConstants.NamingConstants.LIBRARY_NAME_SEPARATOR)[1];
}
- public static IInputStreamProviderFactory createExternalInputStreamFactory(String dataverse, String stream)
+ public static IInputStreamFactory createExternalInputStreamFactory(String dataverse, String stream)
throws AsterixException {
try {
String libraryName = getLibraryName(stream);
String className = getExternalClassName(stream);
ClassLoader classLoader = getClassLoader(dataverse, libraryName);
- return ((IInputStreamProviderFactory) (classLoader.loadClass(className).newInstance()));
+ return ((IInputStreamFactory) (classLoader.loadClass(className).newInstance()));
} catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
throw new AsterixException("Failed to create stream factory", e);
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java b/asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java
index 386a8cf..4eec348 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java
@@ -42,7 +42,7 @@
public class FileSystemWatcher {
private static final Logger LOGGER = Logger.getLogger(FileSystemWatcher.class.getName());
- private final WatchService watcher;
+ private WatchService watcher;
private final HashMap<WatchKey, Path> keys;
private final LinkedList<File> files = new LinkedList<File>();
private Iterator<File> it;
@@ -53,17 +53,14 @@
private boolean done;
private File current;
private AbstractFeedDataFlowController controller;
+ private final LinkedList<Path> dirs;
- public FileSystemWatcher(Path inputResource, String expression, boolean isFeed) throws HyracksDataException {
- try {
- this.watcher = isFeed ? FileSystems.getDefault().newWatchService() : null;
- this.keys = isFeed ? new HashMap<WatchKey, Path>() : null;
- this.expression = expression;
- this.path = inputResource;
- this.isFeed = isFeed;
- } catch (IOException e) {
- throw new HyracksDataException(e);
- }
+ public FileSystemWatcher(Path inputResource, String expression, boolean isFeed) {
+ this.keys = isFeed ? new HashMap<WatchKey, Path>() : null;
+ this.expression = expression;
+ this.path = inputResource;
+ this.isFeed = isFeed;
+ this.dirs = new LinkedList<Path>();
}
public void setFeedLogManager(FeedLogManager feedLogManager) {
@@ -72,11 +69,19 @@
public void init() throws HyracksDataException {
try {
- LinkedList<Path> dirs = null;
- dirs = new LinkedList<Path>();
+ dirs.clear();
LocalFileSystemUtils.traverse(files, path.toFile(), expression, dirs);
it = files.iterator();
if (isFeed) {
+ keys.clear();
+ if (watcher != null) {
+ try {
+ watcher.close();
+ } catch (IOException e) {
+ LOGGER.warn("Failed to close watcher service", e);
+ }
+ }
+ watcher = FileSystems.getDefault().newWatchService();
for (Path path : dirs) {
register(path);
}
@@ -125,7 +130,7 @@
return (WatchEvent<T>) event;
}
- private void handleEvents(WatchKey key) {
+ private void handleEvents(WatchKey key) throws IOException {
// get dir associated with the key
Path dir = keys.get(key);
if (dir == null) {
@@ -143,7 +148,10 @@
if (LOGGER.isEnabledFor(Level.WARN)) {
LOGGER.warn("Overflow event. Some events might have been missed");
}
- continue;
+ // need to read and validate all files.
+ //TODO: use btrees for all logs
+ init();
+ return;
}
// Context for directory entry event is the file name of entry
@@ -172,6 +180,7 @@
if (!done) {
if (watcher != null) {
watcher.close();
+ watcher = null;
}
if (logManager != null) {
if (current != null) {
@@ -205,7 +214,7 @@
return false;
}
- public boolean hasNext() throws HyracksDataException {
+ public boolean hasNext() throws IOException {
if (it.hasNext()) {
return true;
}
@@ -222,6 +231,7 @@
while (key != null) {
handleEvents(key);
if (endOfEvents(key)) {
+ close();
return false;
}
key = watcher.poll();
@@ -237,12 +247,18 @@
if (LOGGER.isEnabledFor(Level.WARN)) {
LOGGER.warn("Feed Closed");
}
- return false;
+ if (watcher == null) {
+ return false;
+ }
+ continue;
} catch (ClosedWatchServiceException e) {
if (LOGGER.isEnabledFor(Level.WARN)) {
LOGGER.warn("The watcher has exited");
}
- return false;
+ if (watcher == null) {
+ return false;
+ }
+ continue;
}
handleEvents(key);
if (endOfEvents(key)) {
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java b/asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java
index 84ccc35..936ada5 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java
@@ -28,7 +28,7 @@
import org.apache.asterix.external.indexing.ExternalFile;
import org.apache.asterix.external.indexing.IndexingScheduler;
import org.apache.asterix.external.indexing.RecordId.RecordIdType;
-import org.apache.asterix.external.input.stream.provider.HDFSInputStreamProvider;
+import org.apache.asterix.external.input.stream.HDFSInputStream;
import org.apache.asterix.om.util.AsterixAppContextInfo;
import org.apache.asterix.om.util.AsterixClusterProperties;
import org.apache.hadoop.fs.BlockLocation;
@@ -186,7 +186,7 @@
conf.set(ExternalDataConstants.KEY_HADOOP_FILESYSTEM_URI,
configuration.get(ExternalDataConstants.KEY_HDFS_URL).trim());
conf.set(ExternalDataConstants.KEY_HADOOP_FILESYSTEM_CLASS, ExternalDataConstants.CLASS_NAME_HDFS_FILESYSTEM);
- conf.setClassLoader(HDFSInputStreamProvider.class.getClassLoader());
+ conf.setClassLoader(HDFSInputStream.class.getClassLoader());
conf.set(ExternalDataConstants.KEY_HADOOP_INPUT_DIR, configuration.get(ExternalDataConstants.KEY_PATH).trim());
conf.set(ExternalDataConstants.KEY_HADOOP_INPUT_FORMAT, formatClassName);
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/AttributeReference.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/AttributeReference.java
index 04fe6ec..da2cebd 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/AttributeReference.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/AttributeReference.java
@@ -18,41 +18,42 @@
*/
package org.apache.asterix.external.classad;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.asterix.om.base.AMutableInt32;
import org.apache.commons.lang3.mutable.MutableBoolean;
import org.apache.hyracks.api.exceptions.HyracksDataException;
public class AttributeReference extends ExprTree {
- private ExprTree expr;
+ private final ExprTreeHolder expr;
private boolean absolute;
- private AMutableCharArrayString attributeStr;
- private ClassAd current = new ClassAd(false, false);
- private ExprList adList = new ExprList();
- private Value val = new Value();
- private MutableBoolean rVal = new MutableBoolean(false);
+ private final AMutableCharArrayString attributeStr;
+ private final ExprList adList;
+ private final Value val;
private AttributeReference tempAttrRef;
- private EvalState tstate = new EvalState();
+ private final EvalState tstate;
+ private MutableBoolean rVal = new MutableBoolean(false);
+ private ClassAd current;
public ExprTree getExpr() {
return expr;
}
public void setExpr(ExprTree expr) {
- this.expr = expr == null ? null : expr.self();
+ this.expr.setInnerTree(expr.self());
}
- public AttributeReference() {
- expr = null;
- attributeStr = null;
+ public AttributeReference(ClassAdObjectPool objectPool) {
+ super(objectPool);
+ this.val = new Value(objectPool);
+ this.current = new ClassAd(this.objectPool);
+ this.tstate = new EvalState(this.objectPool);
+ this.adList = new ExprList(this.objectPool);
+ this.attributeStr = new AMutableCharArrayString();
+ this.expr = new ExprTreeHolder(objectPool);
absolute = false;
}
- /// Copy Constructor
- public AttributeReference(AttributeReference ref) throws HyracksDataException {
- copyFrom(ref);
- }
-
/// Assignment operator
@Override
public boolean equals(Object o) {
@@ -69,8 +70,9 @@
return NodeKind.ATTRREF_NODE;
}
- public static AttributeReference createAttributeReference(ExprTree expr, AMutableCharArrayString attrName) {
- return createAttributeReference(expr, attrName, false);
+ public static AttributeReference createAttributeReference(ExprTree expr, AMutableCharArrayString attrName,
+ ClassAdObjectPool objectPool) {
+ return createAttributeReference(expr, attrName, false, objectPool);
}
/**
@@ -80,7 +82,7 @@
*/
@Override
public ExprTree copy() throws HyracksDataException {
- AttributeReference newTree = new AttributeReference();
+ AttributeReference newTree = objectPool.attrRefPool.get();
newTree.copyFrom(this);
return newTree;
}
@@ -94,14 +96,8 @@
* @throws HyracksDataException
*/
public boolean copyFrom(AttributeReference ref) throws HyracksDataException {
- if (attributeStr == null) {
- attributeStr = new AMutableCharArrayString(ref.attributeStr);
- } else {
- attributeStr.setValue(ref.attributeStr);
- }
- if (ref.expr != null) {
- expr = ref.expr.copy();
- }
+ attributeStr.setValue(ref.attributeStr);
+ expr.setInnerTree(ref.expr);
super.copyFrom(ref);
this.absolute = ref.absolute;
return true;
@@ -127,7 +123,7 @@
if (absolute != other_ref.absolute || !attributeStr.equals(other_ref.attributeStr)) {
is_same = false;
} else if ((expr == null && other_ref.expr == null) || (expr.equals(other_ref.expr))
- || (expr != null && other_ref.expr != null && ((AttributeReference) expr).sameAs(other_ref.expr))) {
+ || (expr != null && other_ref.expr != null && expr.sameAs(other_ref.expr))) {
// Will this check result in infinite recursion? How do I stop it?
is_same = true;
} else {
@@ -137,18 +133,9 @@
return is_same;
}
- // a private ctor for use in significant expr identification
- private AttributeReference(ExprTree tree, AMutableCharArrayString attrname, boolean absolut) {
- attributeStr = attrname;
- expr = tree == null ? null : tree.self();
- absolute = absolut;
- }
-
@Override
public void privateSetParentScope(ClassAd parent) {
- if (expr != null) {
- expr.setParentScope(parent);
- }
+ expr.setParentScope(parent);
}
public void getComponents(ExprTreeHolder tree, AMutableCharArrayString attr, MutableBoolean abs)
@@ -161,7 +148,7 @@
public EvalResult findExpr(EvalState state, ExprTreeHolder tree, ExprTreeHolder sig, boolean wantSig)
throws HyracksDataException {
// establish starting point for search
- if (expr == null) {
+ if (expr.getTree() == null) {
// "attr" and ".attr"
current = absolute ? state.getRootAd() : state.getCurAd();
if (absolute && (current == null)) { // NAC - circularity so no root
@@ -186,7 +173,7 @@
}
if (val.isListValue()) {
- ExprList eList = new ExprList();
+ ExprList eList = objectPool.exprListPool.get();
//
// iterate through exprList and apply attribute reference
// to each exprTree
@@ -195,12 +182,12 @@
return (EvalResult.EVAL_FAIL);
} else {
if (tempAttrRef == null) {
- tempAttrRef = new AttributeReference();
+ tempAttrRef = objectPool.attrRefPool.get();
} else {
tempAttrRef.reset();
}
createAttributeReference(currExpr.copy(), attributeStr, false, tempAttrRef);
- val.clear();
+ val.setUndefinedValue();
// Create new EvalState, within this scope, because
// attrRef is only temporary, so we do not want to
// cache the evaluated result in the outer state object.
@@ -212,8 +199,8 @@
return (EvalResult.EVAL_FAIL);
}
- ClassAd evaledAd = new ClassAd();
- ExprList evaledList = new ExprList();
+ ClassAd evaledAd = objectPool.classAdPool.get();
+ ExprList evaledList = objectPool.exprListPool.get();
if (val.isClassAdValue(evaledAd)) {
eList.add(evaledAd);
continue;
@@ -221,12 +208,13 @@
eList.add(evaledList.copy());
continue;
} else {
- eList.add(Literal.createLiteral(val));
+ eList.add(Literal.createLiteral(val, objectPool));
}
}
}
- tree.setInnerTree(ExprList.createExprList(eList));
- ClassAd newRoot = new ClassAd();
+
+ tree.setInnerTree(ExprList.createExprList(eList, objectPool));
+ ClassAd newRoot = objectPool.classAdPool.get();
tree.setParentScope(newRoot);
return EvalResult.EVAL_OK;
}
@@ -242,8 +230,15 @@
if (current == null) {
return EvalResult.EVAL_UNDEF;
}
- int rc = current.lookupInScope(attributeStr.toString(), tree, state);
- if (expr == null && !absolute && rc == EvalResult.EVAL_UNDEF.ordinal() && current.getAlternateScope() != null) {
+ int rc = 0;
+ try {
+ rc = current.lookupInScope(attributeStr.toString(), tree, state);
+ } catch (Throwable th) {
+ th.printStackTrace();
+ throw th;
+ }
+ if (expr.getTree() == null && !absolute && rc == EvalResult.EVAL_UNDEF.ordinal()
+ && current.getAlternateScope() != null) {
rc = current.getAlternateScope().lookupInScope(attributeStr.toString(), tree, state);
}
return EvalResult.values()[rc];
@@ -251,9 +246,10 @@
@Override
public boolean publicEvaluate(EvalState state, Value val) throws HyracksDataException {
- ExprTreeHolder tree = new ExprTreeHolder();
- ExprTreeHolder dummy = new ExprTreeHolder();
- ClassAd curAd = new ClassAd(state.getCurAd());
+ ExprTreeHolder tree = objectPool.mutableExprPool.get();
+ ExprTreeHolder dummy = objectPool.mutableExprPool.get();
+ ClassAd curAd = objectPool.classAdPool.get();
+ curAd.copyFrom(state.getCurAd());
boolean rval;
// find the expression and the evalstate
switch (findExpr(state, tree, dummy, false)) {
@@ -286,10 +282,13 @@
@Override
public boolean privateEvaluate(EvalState state, Value val, ExprTreeHolder sig) throws HyracksDataException {
- ExprTreeHolder tree = new ExprTreeHolder();
- ExprTreeHolder exprSig = new ExprTreeHolder();
- ClassAd curAd = new ClassAd(state.getCurAd());
- MutableBoolean rval = new MutableBoolean(true);
+ ExprTreeHolder tree = objectPool.mutableExprPool.get();
+ ExprTreeHolder exprSig = objectPool.mutableExprPool.get();
+ ClassAd curAd = objectPool.classAdPool.get();
+ curAd.copyFrom(state.getCurAd());
+ MutableBoolean rval = objectPool.boolPool.get();
+ rval.setValue(true);
+
switch (findExpr(state, tree, exprSig, true)) {
case EVAL_FAIL:
rval.setValue(false);
@@ -314,7 +313,9 @@
default:
throw new HyracksDataException("ClassAd: Should not reach here");
}
- sig.setInnerTree((new AttributeReference(exprSig, attributeStr, absolute)));
+ AttributeReference newAttrRef = objectPool.attrRefPool.get();
+ newAttrRef.setValue(exprSig, attributeStr, absolute);
+ sig.setInnerTree(newAttrRef);
state.getCurAd().setValue(curAd);
return rval.booleanValue();
}
@@ -322,8 +323,8 @@
@Override
public boolean privateFlatten(EvalState state, Value val, ExprTreeHolder ntree, AMutableInt32 op)
throws HyracksDataException {
- ExprTreeHolder tree = new ExprTreeHolder();
- ExprTreeHolder dummy = new ExprTreeHolder();
+ ExprTreeHolder tree = objectPool.mutableExprPool.get();
+ ExprTreeHolder dummy = objectPool.mutableExprPool.get();
ClassAd curAd;
boolean rval;
ntree.setInnerTree(null); // Just to be safe... wenger 2003-12-11.
@@ -338,8 +339,8 @@
return true;
case EVAL_UNDEF:
if (expr != null && state.isFlattenAndInline()) {
- ExprTreeHolder expr_ntree = new ExprTreeHolder();
- Value expr_val = new Value();
+ ExprTreeHolder expr_ntree = objectPool.mutableExprPool.get();
+ Value expr_val = objectPool.valuePool.get();
if (state.getDepthRemaining() <= 0) {
val.setErrorValue();
state.getCurAd().setValue(curAd);
@@ -349,7 +350,7 @@
rval = expr.publicFlatten(state, expr_val, expr_ntree);
state.incrementDepth();
if (rval && expr_ntree.getInnerTree() != null) {
- ntree.setInnerTree(createAttributeReference(expr_ntree, attributeStr));
+ ntree.setInnerTree(createAttributeReference(expr_ntree, attributeStr, objectPool));
if (ntree.getInnerTree() != null) {
state.getCurAd().setValue(curAd);
return true;
@@ -412,14 +413,16 @@
* expr is not NULL, default value is false;
*/
public static AttributeReference createAttributeReference(ExprTree tree, AMutableCharArrayString attrStr,
- boolean absolut) {
- return (new AttributeReference(tree, attrStr, absolut));
+ boolean absolut, ClassAdObjectPool objectPool) {
+ AttributeReference attrRef = objectPool.attrRefPool.get();
+ attrRef.setValue(tree, attrStr, absolut);
+ return attrRef;
}
public void setValue(ExprTree tree, AMutableCharArrayString attrStr, boolean absolut) {
this.absolute = absolut;
- this.attributeStr = attrStr;
- this.expr = tree == null ? null : tree.self();
+ this.attributeStr.copyValue(attrStr.getValue(), attrStr.size());
+ this.expr.setInnerTree(tree == null ? null : tree.self());
}
public static void createAttributeReference(ExprTree tree, AMutableCharArrayString attrStr, boolean absolut,
@@ -429,8 +432,8 @@
@Override
public boolean privateEvaluate(EvalState state, Value val) throws HyracksDataException {
- ExprTreeHolder tree = new ExprTreeHolder();
- ExprTreeHolder dummy = new ExprTreeHolder();
+ ExprTreeHolder tree = objectPool.mutableExprPool.get();
+ ExprTreeHolder dummy = objectPool.mutableExprPool.get();
ClassAd curAd;
boolean rval;
@@ -467,8 +470,12 @@
@Override
public void reset() {
- if (expr != null) {
- expr.reset();
- }
+ expr.reset();
+ val.reset();
+ current.reset();
+ tstate.reset();
+ adList.reset();
+ attributeStr.reset();
+ absolute = false;
}
}
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/BuiltinClassAdFunctions.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/BuiltinClassAdFunctions.java
index 1eca7e1..68bd220 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/BuiltinClassAdFunctions.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/BuiltinClassAdFunctions.java
@@ -21,7 +21,6 @@
import java.io.IOException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
-import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Random;
@@ -32,6 +31,7 @@
import org.apache.asterix.external.classad.ExprTree.NodeKind;
import org.apache.asterix.external.classad.Value.ValueType;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.asterix.external.library.ClassAdParser;
import org.apache.asterix.om.base.AMutableDouble;
import org.apache.asterix.om.base.AMutableInt32;
@@ -43,7 +43,8 @@
public static final ClassAdFunc IsType = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value val) throws HyracksDataException {
+ public boolean call(String name, ExprList argList, EvalState state, Value val, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
// need a single argument
if (argList.size() != 1) {
val.setErrorValue();
@@ -95,13 +96,14 @@
public static final ClassAdFunc TestMember = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value val) throws HyracksDataException {
- Value arg0 = new Value();
- Value arg1 = new Value();
- Value cArg = new Value();
+ public boolean call(String name, ExprList argList, EvalState state, Value val, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg0 = objectPool.valuePool.get();
+ Value arg1 = objectPool.valuePool.get();
+ Value cArg = objectPool.valuePool.get();
- ExprList el = new ExprList();
- MutableBoolean b = new MutableBoolean();
+ ExprList el = objectPool.exprListPool.get();
+ MutableBoolean b = objectPool.boolPool.get();
boolean useIS = name.equalsIgnoreCase("identicalmember");
// need two arguments
@@ -126,10 +128,10 @@
// Swap
if (arg0.isListValue() && !arg1.isListValue()) {
- Value swap = new Value();
- swap.copyFrom(arg0);
- arg0.copyFrom(arg1);
- arg1.copyFrom(swap);
+ Value swap = objectPool.valuePool.get();
+ swap.setValue(arg0);
+ arg0.setValue(arg1);
+ arg1.setValue(swap);
}
// arg1 must be a list; arg0 must be comparable
@@ -151,7 +153,8 @@
val.setErrorValue();
return (false);
}
- Operation.operate(useIS ? Operation.OpKind_IS_OP : Operation.OpKind_EQUAL_OP, cArg, arg0, val);
+ Operation.operate(useIS ? Operation.OpKind_IS_OP : Operation.OpKind_EQUAL_OP, cArg, arg0, val,
+ objectPool);
if (val.isBooleanValue(b) && b.booleanValue()) {
return true;
}
@@ -163,11 +166,13 @@
public static final ClassAdFunc Size = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value val) throws HyracksDataException {
- Value arg = new Value();
- ExprList listToSize = new ExprList();
- ClassAd classadToSize = new ClassAd();
- AMutableInt32 length = new AMutableInt32(0);
+ public boolean call(String name, ExprList argList, EvalState state, Value val, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg = objectPool.valuePool.get();
+ ExprList listToSize = objectPool.exprListPool.get();
+ ClassAd classadToSize = objectPool.classAdPool.get();
+ AMutableInt32 length = objectPool.int32Pool.get();
+ length.setValue(0);
// we accept only one argument
if (argList.size() != 1) {
val.setErrorValue();
@@ -197,15 +202,17 @@
public static final ClassAdFunc SumAvg = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value val) throws HyracksDataException {
+ public boolean call(String name, ExprList argList, EvalState state, Value val, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
- Value listElementValue = new Value();
- Value listVal = new Value();
- Value numElements = new Value();
- Value result = new Value();
- ExprList listToSum = new ExprList();
- MutableBoolean first = new MutableBoolean();
- AMutableInt64 len = new AMutableInt64(0);
+ Value listElementValue = objectPool.valuePool.get();
+ Value listVal = objectPool.valuePool.get();
+ Value numElements = objectPool.valuePool.get();
+ Value result = objectPool.valuePool.get();
+ ExprList listToSum = objectPool.exprListPool.get();
+ MutableBoolean first = objectPool.boolPool.get();
+ AMutableInt64 len = objectPool.int64Pool.get();
+ len.setValue(0);
boolean onlySum = name.equalsIgnoreCase("sum");
// we accept only one argument
@@ -245,28 +252,28 @@
// Either take the number if it's the first,
// or add to the running sum.
if (first.booleanValue()) {
- result.copyFrom(listElementValue);
+ result.setValue(listElementValue);
first.setValue(false);
} else {
- Operation.operate(Operation.OpKind_ADDITION_OP, result, listElementValue, result);
+ Operation.operate(Operation.OpKind_ADDITION_OP, result, listElementValue, result, objectPool);
}
}
// if the sum() function was called, we don't need to find the average
if (onlySum) {
- val.copyFrom(result);
+ val.setValue(result);
return true;
}
if (len.getLongValue() > 0) {
numElements.setRealValue(len.getLongValue());
- Operation.operate(Operation.OpKind_DIVISION_OP, result, numElements, result);
+ Operation.operate(Operation.OpKind_DIVISION_OP, result, numElements, result, objectPool);
} else {
val.setUndefinedValue();
}
- val.copyFrom(result);
+ val.setValue(result);
return true;
}
@@ -274,14 +281,16 @@
public static final ClassAdFunc MinMax = new ClassAdFunc() {
@Override
- public boolean call(String fn, ExprList argList, EvalState state, Value val) throws HyracksDataException {
- Value listElementValue = new Value();
- Value listVal = new Value();
- Value cmp = new Value();
- Value result = new Value();
- ExprList listToBound = new ExprList();
+ public boolean call(String fn, ExprList argList, EvalState state, Value val, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value listElementValue = objectPool.valuePool.get();
+ Value listVal = objectPool.valuePool.get();
+ Value cmp = objectPool.valuePool.get();
+ Value result = objectPool.valuePool.get();
+ ExprList listToBound = objectPool.exprListPool.get();
boolean first = true;
- MutableBoolean b = new MutableBoolean(false);
+ MutableBoolean b = objectPool.boolPool.get();
+ b.setValue(false);
int comparisonOperator;
// we accept only one argument
@@ -326,17 +335,17 @@
// If it's the first element, copy it to the bound,
// otherwise compare to decide what to do.
if (first) {
- result.copyFrom(listElementValue);
+ result.setValue(listElementValue);
first = false;
} else {
- Operation.operate(comparisonOperator, listElementValue, result, cmp);
+ Operation.operate(comparisonOperator, listElementValue, result, cmp, objectPool);
if (cmp.isBooleanValue(b) && b.booleanValue()) {
- result.copyFrom(listElementValue);
+ result.setValue(listElementValue);
}
}
}
- val.copyFrom(result);
+ val.setValue(result);
return true;
}
@@ -344,15 +353,16 @@
public static final ClassAdFunc ListCompare = new ClassAdFunc() {
@Override
- public boolean call(String fn, ExprList argList, EvalState state, Value val) throws HyracksDataException {
+ public boolean call(String fn, ExprList argList, EvalState state, Value val, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
- Value listElementValue = new Value();
- Value listVal = new Value();
- Value compareVal = new Value();
- Value stringValue = new Value();
- ExprList listToCompare = new ExprList();
+ Value listElementValue = objectPool.valuePool.get();
+ Value listVal = objectPool.valuePool.get();
+ Value compareVal = objectPool.valuePool.get();
+ Value stringValue = objectPool.valuePool.get();
+ ExprList listToCompare = objectPool.exprListPool.get();
boolean needAllMatch;
- AMutableCharArrayString comparison_string = new AMutableCharArrayString();
+ AMutableCharArrayString comparison_string = objectPool.strPool.get();
int comparisonOperator;
// We take three arguments:
@@ -445,10 +455,10 @@
val.setErrorValue();
return false;
} else {
- Value compareResult = new Value();
- MutableBoolean b = new MutableBoolean();
+ Value compareResult = objectPool.valuePool.get();
+ MutableBoolean b = objectPool.boolPool.get();
- Operation.operate(comparisonOperator, listElementValue, compareVal, compareResult);
+ Operation.operate(comparisonOperator, listElementValue, compareVal, compareResult, objectPool);
if (!compareResult.isBooleanValue(b)) {
if (compareResult.isUndefinedValue()) {
if (needAllMatch) {
@@ -491,32 +501,37 @@
};
public static final ClassAdFunc timeZoneOffset = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value val) throws HyracksDataException {
+ public boolean call(String name, ExprList argList, EvalState state, Value val, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
// no arguments
if (argList.size() > 0) {
val.setErrorValue();
return (true);
}
- val.setRelativeTimeValue(new ClassAdTime());
+ val.setRelativeTimeValue(objectPool.classAdTimePool.get());
return (true);
}
};
public static final ClassAdFunc debug = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
return false;
}
};
public static final ClassAdFunc formatTime = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- Value time_arg = new Value();
- Value format_arg = new Value();
- AMutableInt64 int64 = new AMutableInt64(0);
- ClassAdTime epoch_time = new ClassAdTime();
- ClassAdTime time_components = new ClassAdTime("GMT");
- ClassAd splitClassAd = new ClassAd();
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value time_arg = objectPool.valuePool.get();
+ Value format_arg = objectPool.valuePool.get();
+ AMutableInt64 int64 = objectPool.int64Pool.get();
+ int64.setValue(0);
+ ClassAdTime epoch_time = objectPool.classAdTimePool.get();
+ ClassAdTime time_components = objectPool.classAdTimePool.get();
+ time_components.setTimeZone("GMT");
+ ClassAd splitClassAd = objectPool.classAdPool.get();
String format;
int number_of_args;
boolean did_eval;
@@ -535,8 +550,7 @@
} else if (time_arg.isRelativeTimeValue()) {
result.setErrorValue();
} else if (time_arg.isAbsoluteTimeValue(time_components)) {
- } else if (!time_arg
- .isClassAdValue(splitClassAd) /* doSplitTime(time_arg, splitClassAd) */) {
+ } else if (!time_arg.isClassAdValue(splitClassAd) /* doSplitTime(time_arg, splitClassAd) */) {
result.setErrorValue();
} else {
if (!splitClassAd.evaluateAttrInt("Seconds", int64)) {
@@ -579,7 +593,7 @@
if (!argList.get(1).publicEvaluate(state, format_arg)) {
did_eval = false;
} else {
- AMutableCharArrayString formatString = new AMutableCharArrayString();
+ AMutableCharArrayString formatString = objectPool.strPool.get();
if (!format_arg.isStringValue(formatString)) {
result.setErrorValue();
} else {
@@ -618,12 +632,13 @@
public static final ClassAdFunc getField = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value val) throws HyracksDataException {
- Value arg = new Value();
- ClassAdTime asecs = new ClassAdTime();
- ClassAdTime rsecs = new ClassAdTime();
- ClassAdTime clock = new ClassAdTime();
- ClassAdTime tms = new ClassAdTime();
+ public boolean call(String name, ExprList argList, EvalState state, Value val, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg = objectPool.valuePool.get();
+ ClassAdTime asecs = objectPool.classAdTimePool.get();
+ ClassAdTime rsecs = objectPool.classAdTimePool.get();
+ ClassAdTime clock = objectPool.classAdTimePool.get();
+ ClassAdTime tms = objectPool.classAdTimePool.get();
if (argList.size() != 1) {
val.setErrorValue();
@@ -708,13 +723,14 @@
};
public static final ClassAdFunc currentTime = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value val) throws HyracksDataException {
+ public boolean call(String name, ExprList argList, EvalState state, Value val, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
// no arguments
if (argList.size() > 0) {
val.setErrorValue();
return (true);
}
- Literal time_literal = Literal.createAbsTime(new ClassAdTime());
+ Literal time_literal = Literal.createAbsTime(objectPool.classAdTimePool.get(), objectPool);
time_literal.GetValue(val);
return true;
}
@@ -722,9 +738,10 @@
public static final ClassAdFunc splitTime = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- Value arg = new Value();
- ClassAd split = new ClassAd();
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg = objectPool.valuePool.get();
+ ClassAd split = objectPool.classAdPool.get();
if (argList.size() != 1) {
result.setErrorValue();
@@ -736,7 +753,7 @@
return false;
}
- if (!arg.isClassAdValue() && doSplitTime(arg, split)) {
+ if (!arg.isClassAdValue() && doSplitTime(arg, split, objectPool)) {
result.setClassAdValue(split);
} else {
result.setErrorValue();
@@ -746,13 +763,16 @@
};
- public static boolean doSplitTime(Value time, ClassAd splitClassAd) throws HyracksDataException {
+ public static boolean doSplitTime(Value time, ClassAd splitClassAd, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
boolean did_conversion;
- AMutableInt64 integer = new AMutableInt64(0);
- AMutableDouble real = new AMutableDouble(0);
- ClassAdTime asecs = new ClassAdTime();
- ClassAdTime rsecs = new ClassAdTime();
- ClassAd classad = new ClassAd();
+ AMutableInt64 integer = objectPool.int64Pool.get();
+ integer.setValue(0);
+ AMutableDouble real = objectPool.doublePool.get();
+ real.setValue(0);
+ ClassAdTime asecs = objectPool.classAdTimePool.get();
+ ClassAdTime rsecs = objectPool.classAdTimePool.get();
+ ClassAd classad = objectPool.classAdPool.get();
did_conversion = true;
if (time.isIntegerValue(integer)) {
asecs.setValue(integer.getLongValue());
@@ -767,7 +787,7 @@
} else if (time.isRelativeTimeValue(rsecs)) {
relTimeToClassAd((rsecs.getRelativeTime() / 1000.0), splitClassAd);
} else if (time.isClassAdValue(classad)) {
- splitClassAd = new ClassAd();
+ splitClassAd = objectPool.classAdPool.get();
splitClassAd.copyFrom(classad);
} else {
did_conversion = false;
@@ -828,14 +848,16 @@
public static final ClassAdFunc dayTime = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value val) throws HyracksDataException {
- val.setRelativeTimeValue(new ClassAdTime());
+ public boolean call(String name, ExprList argList, EvalState state, Value val, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ val.setRelativeTimeValue(objectPool.classAdTimePool.get());
return (true);
}
};
public static final ClassAdFunc epochTime = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value val) throws HyracksDataException {
+ public boolean call(String name, ExprList argList, EvalState state, Value val, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
// no arguments
if (argList.size() > 0) {
val.setErrorValue();
@@ -847,15 +869,16 @@
};
public static final ClassAdFunc strCat = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- AMutableCharArrayString buf = new AMutableCharArrayString();
- AMutableCharArrayString s = new AMutableCharArrayString();
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ AMutableCharArrayString buf = objectPool.strPool.get();
+ AMutableCharArrayString s = objectPool.strPool.get();
boolean errorFlag = false;
boolean undefFlag = false;
boolean rval = false;
- Value val = new Value();
- Value stringVal = new Value();
+ Value val = objectPool.valuePool.get();
+ Value stringVal = objectPool.valuePool.get();
for (int i = 0; i < argList.size(); i++) {
@@ -867,7 +890,7 @@
if (val.isStringValue(s)) {
buf.appendString(s);
} else {
- Value.convertValueToStringValue(val, stringVal);
+ Value.convertValueToStringValue(val, stringVal, objectPool);
if (stringVal.isUndefinedValue()) {
undefFlag = true;
break;
@@ -906,10 +929,11 @@
};
public static final ClassAdFunc changeCase = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- Value val = new Value();
- Value stringVal = new Value();
- AMutableCharArrayString str = new AMutableCharArrayString();
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value val = objectPool.valuePool.get();
+ Value stringVal = objectPool.valuePool.get();
+ AMutableCharArrayString str = objectPool.strPool.get();
boolean lower = name.equalsIgnoreCase("tolower");
int len;
@@ -926,7 +950,7 @@
}
if (!val.isStringValue(str)) {
- Value.convertValueToStringValue(val, stringVal);
+ Value.convertValueToStringValue(val, stringVal, objectPool);
if (stringVal.isUndefinedValue()) {
result.setUndefinedValue();
return true;
@@ -949,14 +973,18 @@
public static final ClassAdFunc subString = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- Value arg0 = new Value();
- Value arg1 = new Value();
- Value arg2 = new Value();
- AMutableCharArrayString buf = new AMutableCharArrayString();
- AMutableInt64 offset = new AMutableInt64(0);
- AMutableInt64 len = new AMutableInt64(0);
- AMutableInt64 alen = new AMutableInt64(0);
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg0 = objectPool.valuePool.get();
+ Value arg1 = objectPool.valuePool.get();
+ Value arg2 = objectPool.valuePool.get();
+ AMutableCharArrayString buf = objectPool.strPool.get();
+ AMutableInt64 offset = objectPool.int64Pool.get();
+ offset.setValue(0);
+ AMutableInt64 len = objectPool.int64Pool.get();
+ len.setValue(0);
+ AMutableInt64 alen = objectPool.int64Pool.get();
+ len.setValue(0);
// two or three arguments
if (argList.size() < 2 || argList.size() > 3) {
@@ -1004,10 +1032,12 @@
// to make sure that if length is specified as 0 explicitly
// then, len is set to 0
if (argList.size() == 3) {
- AMutableInt64 templen = new AMutableInt64(0);
+ AMutableInt64 templen = objectPool.int64Pool.get();
+ templen.setValue(0);
arg2.isIntegerValue(templen);
- if (templen.getLongValue() == 0)
+ if (templen.getLongValue() == 0) {
len.setValue(0);
+ }
}
result.setStringValue(buf.substr((int) offset.getLongValue(), (int) len.getLongValue()));
return (true);
@@ -1016,8 +1046,9 @@
public static final ClassAdFunc convInt = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- Value arg = new Value();
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg = objectPool.valuePool.get();
// takes exactly one argument
if (argList.size() != 1) {
result.setErrorValue();
@@ -1027,18 +1058,19 @@
result.setErrorValue();
return (false);
}
- Value.convertValueToIntegerValue(arg, result);
+ Value.convertValueToIntegerValue(arg, result, objectPool);
return true;
}
};
public static final ClassAdFunc compareString = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- Value arg0 = new Value();
- Value arg1 = new Value();
- Value arg0_s = new Value();
- Value arg1_s = new Value();
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg0 = objectPool.valuePool.get();
+ Value arg1 = objectPool.valuePool.get();
+ Value arg0_s = objectPool.valuePool.get();
+ Value arg1_s = objectPool.valuePool.get();
// Must have two arguments
if (argList.size() != 2) {
@@ -1059,25 +1091,28 @@
return true;
}
- AMutableCharArrayString s0 = new AMutableCharArrayString();
- AMutableCharArrayString s1 = new AMutableCharArrayString();
- if (Value.convertValueToStringValue(arg0, arg0_s) && Value.convertValueToStringValue(arg1, arg1_s)
- && arg0_s.isStringValue(s0) && arg1_s.isStringValue(s1)) {
+ AMutableCharArrayString s0 = objectPool.strPool.get();
+ AMutableCharArrayString s1 = objectPool.strPool.get();
+ if (Value.convertValueToStringValue(arg0, arg0_s, objectPool)
+ && Value.convertValueToStringValue(arg1, arg1_s, objectPool) && arg0_s.isStringValue(s0)
+ && arg1_s.isStringValue(s1)) {
int order;
if (name.equalsIgnoreCase("strcmp")) {
order = s0.compareTo(s1);
- if (order < 0)
+ if (order < 0) {
order = -1;
- else if (order > 0)
+ } else if (order > 0) {
order = 1;
+ }
} else {
order = s0.compareToIgnoreCase(s1);
- if (order < 0)
+ if (order < 0) {
order = -1;
- else if (order > 0)
+ } else if (order > 0) {
order = 1;
+ }
}
result.setIntegerValue(order);
} else {
@@ -1090,15 +1125,16 @@
public static final ClassAdFunc matchPattern = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
boolean have_options;
- Value arg0 = new Value();
- Value arg1 = new Value();
- Value arg2 = new Value();
+ Value arg0 = objectPool.valuePool.get();
+ Value arg1 = objectPool.valuePool.get();
+ Value arg2 = objectPool.valuePool.get();
- AMutableCharArrayString pattern = new AMutableCharArrayString();
- AMutableCharArrayString target = new AMutableCharArrayString();
- AMutableCharArrayString options_string = new AMutableCharArrayString();
+ AMutableCharArrayString pattern = objectPool.strPool.get();
+ AMutableCharArrayString target = objectPool.strPool.get();
+ AMutableCharArrayString options_string = objectPool.strPool.get();
// need two or three arguments: pattern, string, optional settings
if (argList.size() != 2 && argList.size() != 3) {
@@ -1184,8 +1220,9 @@
public static final ClassAdFunc convReal = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- Value arg = new Value();
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg = objectPool.valuePool.get();
// takes exactly one argument
if (argList.size() != 1) {
@@ -1196,15 +1233,16 @@
result.setErrorValue();
return (false);
}
- Value.convertValueToRealValue(arg, result);
+ Value.convertValueToRealValue(arg, result, objectPool);
return true;
}
};
public static final ClassAdFunc convString = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- Value arg = new Value();
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg = objectPool.valuePool.get();
// takes exactly one argument
if (argList.size() != 1) {
@@ -1216,21 +1254,22 @@
return (false);
}
- Value.convertValueToStringValue(arg, result);
+ Value.convertValueToStringValue(arg, result, objectPool);
return true;
}
};
public static final ClassAdFunc unparse = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
if (argList.size() != 1 || argList.get(0).getKind() != NodeKind.ATTRREF_NODE) {
result.setErrorValue();
} else {
// use the printpretty on arg0 to spew out
- PrettyPrint unp = new PrettyPrint();
- AMutableCharArrayString szAttribute = new AMutableCharArrayString();
- AMutableCharArrayString szValue = new AMutableCharArrayString();
+ PrettyPrint unp = objectPool.prettyPrintPool.get();
+ AMutableCharArrayString szAttribute = objectPool.strPool.get();
+ AMutableCharArrayString szValue = objectPool.strPool.get();
ExprTree pTree;
unp.unparse(szAttribute, argList.get(0));
@@ -1247,8 +1286,9 @@
};
public static final ClassAdFunc convBool = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- Value arg = new Value();
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg = objectPool.valuePool.get();
// takes exactly one argument
if (argList.size() != 1) {
result.setErrorValue();
@@ -1273,25 +1313,26 @@
return (true);
case BOOLEAN_VALUE:
- result.copyFrom(arg);
+ result.setValue(arg);
return (true);
case INTEGER_VALUE: {
- AMutableInt64 ival = new AMutableInt64(0);
+ AMutableInt64 ival = objectPool.int64Pool.get();
+ ival.setValue(0);
arg.isIntegerValue(ival);
result.setBooleanValue(ival.getLongValue() != 0);
return (true);
}
case REAL_VALUE: {
- AMutableDouble rval = new AMutableDouble(0);
+ AMutableDouble rval = objectPool.doublePool.get();
arg.isRealValue(rval);
result.setBooleanValue(rval.getDoubleValue() != 0.0);
return (true);
}
case STRING_VALUE: {
- AMutableCharArrayString buf = new AMutableCharArrayString();
+ AMutableCharArrayString buf = objectPool.strPool.get();
arg.isStringValue(buf);
if (buf.equalsIgnoreCase("false") || buf.size() == 0) {
result.setBooleanValue(false);
@@ -1302,7 +1343,7 @@
}
case RELATIVE_TIME_VALUE: {
- ClassAdTime rsecs = new ClassAdTime();
+ ClassAdTime rsecs = objectPool.classAdTimePool.get();
arg.isRelativeTimeValue(rsecs);
result.setBooleanValue(rsecs.getTimeInMillis() != 0);
return (true);
@@ -1315,16 +1356,18 @@
};
public static final ClassAdFunc convTime = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- Value arg = new Value();
- Value arg2 = new Value();
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg = objectPool.valuePool.get();
+ Value arg2 = objectPool.valuePool.get();
boolean relative = name.equalsIgnoreCase("reltime");
boolean secondarg = false; // says whether a 2nd argument exists
- AMutableInt64 arg2num = new AMutableInt64(0);
+ AMutableInt64 arg2num = objectPool.int64Pool.get();
+ arg2num.setValue(0);
if (argList.size() == 0 && !relative) {
// absTime with no arguments returns the current time.
- return currentTime.call(name, argList, state, result);
+ return currentTime.call(name, argList, state, result, objectPool);
}
if ((argList.size() < 1) || (argList.size() > 2)) {
result.setErrorValue();
@@ -1340,9 +1383,10 @@
result.setErrorValue();
return (false);
}
- AMutableInt64 ivalue2 = new AMutableInt64(0);
- AMutableDouble rvalue2 = new AMutableDouble(0);
- ClassAdTime rsecs = new ClassAdTime();
+ AMutableInt64 ivalue2 = objectPool.int64Pool.get();
+ ivalue2.setValue(0);
+ AMutableDouble rvalue2 = objectPool.doublePool.get();
+ ClassAdTime rsecs = objectPool.classAdTimePool.get();
if (relative) {// 2nd argument is N/A for reltime
result.setErrorValue();
return (true);
@@ -1377,41 +1421,49 @@
return (true);
case INTEGER_VALUE: {
- AMutableInt64 ivalue = new AMutableInt64(0);
+ AMutableInt64 ivalue = objectPool.int64Pool.get();
+ ivalue.setValue(0);
arg.isIntegerValue(ivalue);
if (relative) {
- result.setRelativeTimeValue(new ClassAdTime(ivalue.getLongValue(), false));
+ ClassAdTime time = objectPool.classAdTimePool.get();
+ time.setRelativeTime(ivalue.getLongValue());
+ result.setRelativeTimeValue(time);
} else {
- ClassAdTime atvalue = new ClassAdTime(true);
+ ClassAdTime atvalue = objectPool.classAdTimePool.get();
atvalue.setValue(ivalue.getLongValue());
- if (secondarg) //2nd arg is the offset in secs
+ if (secondarg) {
atvalue.setTimeZone((int) arg2num.getLongValue());
- else
+ } else {
// the default offset is the current timezone
atvalue.setTimeZone(Literal.findOffset(atvalue));
+ }
if (atvalue.getOffset() == -1) {
result.setErrorValue();
return (false);
- } else
+ } else {
result.setAbsoluteTimeValue(atvalue);
+ }
}
return (true);
}
case REAL_VALUE: {
- AMutableDouble rvalue = new AMutableDouble(0);
+ AMutableDouble rvalue = objectPool.doublePool.get();
arg.isRealValue(rvalue);
if (relative) {
- result.setRelativeTimeValue(new ClassAdTime((long) (1000 * rvalue.getDoubleValue()), false));
+ ClassAdTime time = objectPool.classAdTimePool.get();
+ time.setRelativeTime((long) (1000 * rvalue.getDoubleValue()));
+ result.setRelativeTimeValue(time);
} else {
- ClassAdTime atvalue = new ClassAdTime();
+ ClassAdTime atvalue = objectPool.classAdTimePool.get();
atvalue.setValue((long) rvalue.getDoubleValue());
- if (secondarg) //2nd arg is the offset in secs
+ if (secondarg) {
atvalue.setTimeZone((int) arg2num.getLongValue());
- else
+ } else {
// the default offset is the current timezone
atvalue.setTimeZone(Literal.findOffset(atvalue));
+ }
result.setAbsoluteTimeValue(atvalue);
}
return (true);
@@ -1423,28 +1475,29 @@
}
case ABSOLUTE_TIME_VALUE: {
- ClassAdTime secs = new ClassAdTime();
+ ClassAdTime secs = objectPool.classAdTimePool.get();
arg.isAbsoluteTimeValue(secs);
if (relative) {
result.setRelativeTimeValue(secs);
} else {
- result.copyFrom(arg);
+ result.setValue(arg);
}
return (true);
}
case RELATIVE_TIME_VALUE: {
if (relative) {
- result.copyFrom(arg);
+ result.setValue(arg);
} else {
- ClassAdTime secs = new ClassAdTime();
+ ClassAdTime secs = objectPool.classAdTimePool.get();
arg.isRelativeTimeValue(secs);
- ClassAdTime atvalue = new ClassAdTime();
+ ClassAdTime atvalue = objectPool.classAdTimePool.get();
atvalue.setValue(secs);
- if (secondarg) //2nd arg is the offset in secs
+ if (secondarg) {
atvalue.setTimeZone((int) arg2num.getLongValue());
- else
+ } else {
// the default offset is the current timezone
atvalue.setTimeZone(Literal.findOffset(atvalue));
+ }
result.setAbsoluteTimeValue(atvalue);
}
return (true);
@@ -1458,9 +1511,10 @@
public static final ClassAdFunc doRound = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- Value arg = new Value();
- Value realValue = new Value();
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg = objectPool.valuePool.get();
+ Value realValue = objectPool.valuePool.get();
// takes exactly one argument
if (argList.size() != 1) {
result.setErrorValue();
@@ -1471,12 +1525,12 @@
return (false);
}
if (arg.getType() == ValueType.INTEGER_VALUE) {
- result.copyFrom(arg);
+ result.setValue(arg);
} else {
- if (!Value.convertValueToRealValue(arg, realValue)) {
+ if (!Value.convertValueToRealValue(arg, realValue, objectPool)) {
result.setErrorValue();
} else {
- AMutableDouble rvalue = new AMutableDouble(0);
+ AMutableDouble rvalue = objectPool.doublePool.get();
realValue.isRealValue(rvalue);
if (name.equalsIgnoreCase("floor")) {
result.setIntegerValue((long) Math.floor(rvalue.getDoubleValue()));
@@ -1495,9 +1549,10 @@
public static final ClassAdFunc doMath2 = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- Value arg = new Value();
- Value arg2 = new Value();
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg = objectPool.valuePool.get();
+ Value arg2 = objectPool.valuePool.get();
// takes 2 arguments pow(val,base)
if (argList.size() != 2) {
@@ -1511,20 +1566,24 @@
if (name.equalsIgnoreCase("pow")) {
// take arg2 to the power of arg2
- AMutableInt64 ival = new AMutableInt64(0);
- AMutableInt64 ibase = new AMutableInt64(0);
+ AMutableInt64 ival = objectPool.int64Pool.get();
+ ival.setValue(0);
+ AMutableInt64 ibase = objectPool.int64Pool.get();
+ ibase.setValue(0);
if (arg.isIntegerValue(ival) && arg2.isIntegerValue(ibase) && ibase.getLongValue() >= 0) {
ival.setValue((long) (Math.pow(ival.getLongValue(), ibase.getLongValue())));
result.setIntegerValue(ival.getLongValue());
} else {
- Value realValue = new Value();
- Value realBase = new Value();
- if (!Value.convertValueToRealValue(arg, realValue)
- || !Value.convertValueToRealValue(arg2, realBase)) {
+ Value realValue = objectPool.valuePool.get();
+ Value realBase = objectPool.valuePool.get();
+ if (!Value.convertValueToRealValue(arg, realValue, objectPool)
+ || !Value.convertValueToRealValue(arg2, realBase, objectPool)) {
result.setErrorValue();
} else {
- AMutableDouble rvalue = new AMutableDouble(0);
- AMutableDouble rbase = new AMutableDouble(1);
+ AMutableDouble rvalue = objectPool.doublePool.get();
+ rvalue.setValue(0);
+ AMutableDouble rbase = objectPool.doublePool.get();
+ rbase.setValue(1);
realValue.isRealValue(rvalue);
realBase.isRealValue(rbase);
result.setRealValue(Math.pow(rvalue.getDoubleValue(), rbase.getDoubleValue()));
@@ -1535,17 +1594,17 @@
// if arg2 is a list, choose the first item from the list that is larger than arg1
// if arg1 is larger than all of the items in the list, the result is an error.
- Value val = new Value();
- Value base = new Value();
- if (!Value.convertValueToRealValue(arg, val)) {
+ Value val = objectPool.valuePool.get();
+ Value base = objectPool.valuePool.get();
+ if (!Value.convertValueToRealValue(arg, val, objectPool)) {
result.setErrorValue();
} else {
// get the value to quantize into rval.
- AMutableDouble rval = new AMutableDouble(0);
- AMutableDouble rbase = new AMutableDouble(0);
+ AMutableDouble rval = objectPool.doublePool.get();
+ AMutableDouble rbase = objectPool.doublePool.get();
val.isRealValue(rval);
if (arg2.isListValue()) {
- ExprList list = new ExprList();
+ ExprList list = objectPool.exprListPool.get();
arg2.isListValue(list);
base.setRealValue(0.0);
rbase.setValue(0.0); // treat an empty list as 'don't quantize'
@@ -1554,7 +1613,7 @@
result.setErrorValue();
return false; // eval should not fail
}
- if (Value.convertValueToRealValue(base, val)) {
+ if (Value.convertValueToRealValue(base, val, objectPool)) {
val.isRealValue(rbase);
if (rbase.getDoubleValue() >= rval.getDoubleValue()) {
result.setValue(base);
@@ -1575,7 +1634,7 @@
// if arg2 is not a list, then it must evaluate to a real value
// or we can't use it. (note that if it's an int, we still want
// to return an int, but we assume that all ints can be converted to real)
- if (!Value.convertValueToRealValue(arg2, base)) {
+ if (!Value.convertValueToRealValue(arg2, base, objectPool)) {
result.setErrorValue();
return true;
}
@@ -1585,13 +1644,15 @@
// at this point rbase should contain the real value of either arg2 or the
// last entry in the list. and rval should contain the value to be quantized.
- AMutableInt64 ival = new AMutableInt64(0);
- AMutableInt64 ibase = new AMutableInt64(0);
+ AMutableInt64 ival = objectPool.int64Pool.get();
+ ival.setValue(0);
+ AMutableInt64 ibase = objectPool.int64Pool.get();
+ ibase.setValue(0);
if (arg2.isIntegerValue(ibase)) {
// quantize to an integer base,
- if (ibase.getLongValue() == 0L)
+ if (ibase.getLongValue() == 0L) {
result.setValue(arg);
- else if (arg.isIntegerValue(ival)) {
+ } else if (arg.isIntegerValue(ival)) {
ival.setValue(((ival.getLongValue() + ibase.getLongValue() - 1) / ibase.getLongValue())
* ibase.getLongValue());
result.setIntegerValue(ival.getLongValue());
@@ -1620,10 +1681,12 @@
}
};
public static final ClassAdFunc random = new ClassAdFunc() {
+ private Random randomGenerator = new Random(System.currentTimeMillis());
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- Value arg = new Value();;
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg = objectPool.valuePool.get();;
// takes exactly one argument
if (argList.size() > 1) {
result.setErrorValue();
@@ -1634,9 +1697,9 @@
result.setErrorValue();
return (false);
}
- AMutableInt64 int_max = new AMutableInt64(0);
- AMutableDouble double_max = new AMutableDouble(0);
- Random randomGenerator = new Random(System.currentTimeMillis());
+ AMutableInt64 int_max = objectPool.int64Pool.get();
+ int_max.setValue(0);
+ AMutableDouble double_max = objectPool.doublePool.get();
if (arg.isIntegerValue(int_max)) {
int random_int = randomGenerator.nextInt((int) int_max.getLongValue());
result.setIntegerValue(random_int);
@@ -1653,9 +1716,10 @@
public static final ClassAdFunc ifThenElse = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- Value arg1 = new Value();
- MutableBoolean arg1_bool = new MutableBoolean();
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg1 = objectPool.valuePool.get();
+ MutableBoolean arg1_bool = objectPool.boolPool.get();
// takes exactly three arguments
if (argList.size() != 3) {
result.setErrorValue();
@@ -1673,7 +1737,8 @@
}
break;
case INTEGER_VALUE: {
- AMutableInt64 intval = new AMutableInt64(0);
+ AMutableInt64 intval = objectPool.int64Pool.get();
+ intval.setValue(0);
if (!arg1.isIntegerValue(intval)) {
result.setErrorValue();
return (false);
@@ -1682,7 +1747,7 @@
break;
}
case REAL_VALUE: {
- AMutableDouble realval = new AMutableDouble(0);
+ AMutableDouble realval = objectPool.doublePool.get();
if (!arg1.isRealValue(realval)) {
result.setErrorValue();
return (false);
@@ -1721,14 +1786,15 @@
public static final ClassAdFunc stringListsIntersect = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- Value arg0 = new Value();
- Value arg1 = new Value();
- Value arg2 = new Value();
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg0 = objectPool.valuePool.get();
+ Value arg1 = objectPool.valuePool.get();
+ Value arg2 = objectPool.valuePool.get();
boolean have_delimiter;
- AMutableCharArrayString str0 = new AMutableCharArrayString();
- AMutableCharArrayString str1 = new AMutableCharArrayString();
- AMutableCharArrayString delimiter_string = new AMutableCharArrayString();
+ AMutableCharArrayString str0 = objectPool.strPool.get();
+ AMutableCharArrayString str1 = objectPool.strPool.get();
+ AMutableCharArrayString delimiter_string = objectPool.strPool.get();
// need two or three arguments: pattern, list, optional settings
if (argList.size() != 2 && argList.size() != 3) {
@@ -1782,7 +1848,7 @@
}
result.setBooleanValue(false);
- List<String> list0 = new ArrayList<String>();
+ List<String> list0 = objectPool.stringArrayListPool.get();
Set<String> set1 = new HashSet<String>();
split_string_list(str0, have_delimiter ? delimiter_string.charAt(0) : ',', list0);
@@ -1800,10 +1866,12 @@
};
public static final ClassAdFunc interval = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- Value arg = new Value();
- Value intarg = new Value();
- AMutableInt64 tot_secs = new AMutableInt64(0);
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg = objectPool.valuePool.get();
+ Value intarg = objectPool.valuePool.get();
+ AMutableInt64 tot_secs = objectPool.int64Pool.get();
+ tot_secs.setValue(0);
// takes exactly one argument
if (argList.size() != 1) {
result.setErrorValue();
@@ -1813,7 +1881,7 @@
result.setErrorValue();
return (false);
}
- if (!Value.convertValueToIntegerValue(arg, intarg)) {
+ if (!Value.convertValueToIntegerValue(arg, intarg, objectPool)) {
result.setErrorValue();
return (true);
}
@@ -1843,9 +1911,10 @@
};
public static final ClassAdFunc eval = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- Value arg = new Value();
- Value strarg = new Value();
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg = objectPool.valuePool.get();
+ Value strarg = objectPool.valuePool.get();
// takes exactly one argument
if (argList.size() != 1) {
result.setErrorValue();
@@ -1855,8 +1924,10 @@
result.setErrorValue();
return false;
}
- AMutableCharArrayString s = new AMutableCharArrayString();
- if (!Value.convertValueToStringValue(arg, strarg) || !strarg.isStringValue(s)) {
+ ClassAdParser parser = objectPool.classAdParserPool.get();
+ ExprTreeHolder expr = objectPool.mutableExprPool.get();
+ AMutableCharArrayString s = objectPool.strPool.get();
+ if (!Value.convertValueToStringValue(arg, strarg, objectPool) || !strarg.isStringValue(s)) {
result.setErrorValue();
return true;
}
@@ -1864,8 +1935,8 @@
result.setErrorValue();
return false;
}
- ClassAdParser parser = new ClassAdParser(null, false, true, false, null, null, null);
- ExprTreeHolder expr = new ExprTreeHolder();
+
+ expr.reset();
try {
if (!parser.parseExpression(s.toString(), expr, true) || (expr.getInnerTree() == null)) {
result.setErrorValue();
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/CaseInsensitiveString.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/CaseInsensitiveString.java
index ecc69f8..7a93eda 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/CaseInsensitiveString.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/CaseInsensitiveString.java
@@ -20,6 +20,7 @@
public class CaseInsensitiveString implements Comparable<CaseInsensitiveString> {
private String aString;
+ private String aLowerCaseString;
public String get() {
return aString;
@@ -32,30 +33,33 @@
public void set(String aString) {
this.aString = aString;
+ this.aLowerCaseString = aString == null ? null : aString.toLowerCase();
}
public CaseInsensitiveString(String aString) {
this.aString = aString;
+ this.aLowerCaseString = aString == null ? null : aString.toLowerCase();
}
public CaseInsensitiveString() {
this.aString = null;
+ this.aLowerCaseString = null;
}
@Override
public int compareTo(CaseInsensitiveString o) {
- return aString.compareToIgnoreCase(o.aString);
+ return aLowerCaseString.compareTo(o.aLowerCaseString);
}
@Override
public boolean equals(Object o) {
- return (o instanceof CaseInsensitiveString) ? aString.equalsIgnoreCase(((CaseInsensitiveString) o).aString)
- : false;
+ return (o instanceof CaseInsensitiveString)
+ ? aLowerCaseString.equals(((CaseInsensitiveString) o).aLowerCaseString) : false;
}
@Override
public int hashCode() {
- return aString.toLowerCase().hashCode();
+ return aLowerCaseString.hashCode();
}
}
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAd.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAd.java
index df6ae0d..4f27cd8 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAd.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAd.java
@@ -27,7 +27,7 @@
import java.util.TreeSet;
import org.apache.asterix.external.classad.Value.NumberFactor;
-import org.apache.asterix.external.classad.object.pool.CaseInsensitiveStringPool;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.asterix.external.library.ClassAdParser;
import org.apache.asterix.om.base.AMutableDouble;
import org.apache.asterix.om.base.AMutableInt32;
@@ -97,7 +97,6 @@
private static final int CLASSAD_VERSION_PATCH = 0;
private static final String CLASSAD_VERSION = "8.0.0";
public static final ArrayList<String> specialAttrNames = new ArrayList<String>();
- private final CaseInsensitiveStringPool StringPool = new CaseInsensitiveStringPool();
static {
specialAttrNames.add(ATTR_TOPLEVEL);
@@ -106,42 +105,22 @@
specialAttrNames.add(ATTR_PARENT);
}
- public static final FunctionCall curr_time_expr = FunctionCall.createFunctionCall("time", new ExprList());
+ public static final FunctionCall curr_time_expr = FunctionCall.createFunctionCall("time",
+ new ExprList(new ClassAdObjectPool()), new ClassAdObjectPool());
private ClassAd alternateScope;
- //private boolean doDirtyTracking;
- private Map<CaseInsensitiveString, ExprTree> attrList = new HashMap<CaseInsensitiveString, ExprTree>();
+ private final Map<CaseInsensitiveString, ExprTree> attrList;
private ClassAd chainedParentAd;
private ClassAdParser parser = null;
- private ClassAd newAd = null;
+ private ClassAd newAd;
/*
* Constructors
*/
- public ClassAd() {
- chainedParentAd = null;
- alternateScope = null;
- newAd = new ClassAd(false, false);
- parser = new ClassAdParser(null, false, true, false, null, null, null);
- }
-
- public ClassAd(boolean initializeParser, boolean initializeNewAd) {
- chainedParentAd = null;
- alternateScope = null;
- if (initializeNewAd) {
- newAd = new ClassAd(false, false);
- }
- if (initializeParser) {
- parser = new ClassAdParser(null, false, true, false, null, null, null);
- }
- }
-
- public ClassAd(ClassAd ad) throws HyracksDataException {
- if (ad == null) {
- clear();
- } else {
- copyFrom(ad);
- }
+ public ClassAd(ClassAdObjectPool objectPool) {
+ super(objectPool);
+ parser = new ClassAdParser(this.objectPool);
+ attrList = new HashMap<CaseInsensitiveString, ExprTree>();
}
@Override
@@ -165,10 +144,6 @@
return attrList;
}
- public void setAttrList(Map<CaseInsensitiveString, ExprTree> attrList) {
- this.attrList = attrList;
- }
-
public void classAdLibraryVersion(AMutableInt32 major, AMutableInt32 minor, AMutableInt32 patch) {
major.setValue(CLASSAD_VERSION_MAJOR);
minor.setValue(CLASSAD_VERSION_MINOR);
@@ -187,21 +162,6 @@
return curr_time_expr;
}
- //public TreeSet<CaseInsensitiveString> dirtyAttrList = new TreeSet<CaseInsensitiveString>();
-
- /*
- * Reference is an ordered set of Strings <The ordering uses less than ignore case>. Example
- * below
- * TreeSet<String> references = new TreeSet<String>(
- * new Comparator<String>(){
- * public int compare(String o1, String o2) {
- * return o1.compareToIgnoreCase(o2);
- * }
- * });
- *
- * // PortReferences is a Map<ClassAd,OrderedSet<Strings>>
- */
-
public boolean copyFrom(ClassAd ad) throws HyracksDataException {
boolean succeeded = true;
@@ -211,38 +171,26 @@
clear();
// copy scoping attributes
super.copyFrom(ad);
- if (ad.chainedParentAd != null) {
- if (chainedParentAd == null) {
- chainedParentAd = new ClassAd();
- }
- chainedParentAd.setValue(ad.chainedParentAd);
- }
- if (ad.alternateScope != null) {
- if (alternateScope == null) {
- alternateScope = new ClassAd();
- }
- alternateScope.setValue(ad.alternateScope);
- }
- //this.doDirtyTracking = false;
+ chainedParentAd = ad.chainedParentAd;
+ alternateScope = ad.alternateScope;
for (Entry<CaseInsensitiveString, ExprTree> attr : ad.attrList.entrySet()) {
- ExprTree tree = attr.getValue().copy();
- attrList.put(attr.getKey(), tree);
- // if (ad.doDirtyTracking && ad.IsAttributeDirty(attr.getKey())) {
- // dirtyAttrList.add(attr.getKey());
- //}
+ ExprTree tree = objectPool.mutableExprPool.get();
+ CaseInsensitiveString key = objectPool.caseInsensitiveStringPool.get();
+ tree.copyFrom(attr.getValue());
+ key.set(attr.getKey().get());
+ attrList.put(key, tree);
}
- //doDirtyTracking = ad.doDirtyTracking;
}
return succeeded;
}
public boolean update(ClassAd ad) throws HyracksDataException {
for (Entry<CaseInsensitiveString, ExprTree> attr : ad.attrList.entrySet()) {
- ExprTree tree = attr.getValue().copy();
- attrList.put(attr.getKey(), tree);
- // if (ad.doDirtyTracking && ad.IsAttributeDirty(attr.getKey())) {
- // dirtyAttrList.add(attr.getKey());
- //}
+ ExprTree tree = objectPool.mutableExprPool.get();
+ CaseInsensitiveString key = objectPool.caseInsensitiveStringPool.get();
+ tree.copyFrom(attr.getValue());
+ key.set(attr.getKey().get());
+ attrList.put(key, tree);
}
return true;
}
@@ -306,9 +254,6 @@
if (alternateScope != null) {
alternateScope.clear();
}
- if (parser != null) {
- parser.reset();
- }
}
public void unchain() {
@@ -317,18 +262,24 @@
}
}
- public void getComponents(Map<CaseInsensitiveString, ExprTree> attrs) {
+ public void getComponents(Map<CaseInsensitiveString, ExprTree> attrs, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
attrs.clear();
for (Entry<CaseInsensitiveString, ExprTree> attr : this.attrList.entrySet()) {
- attrs.put(attr.getKey(), attr.getValue());
+ ExprTree tree = objectPool.mutableExprPool.get();
+ CaseInsensitiveString key = objectPool.caseInsensitiveStringPool.get();
+ tree.copyFrom(attr.getValue());
+ key.set(attr.getKey().get());
+ attrs.put(key, tree);
}
}
public ClassAd privateGetDeepScope(ExprTree tree) throws HyracksDataException {
- ClassAd scope = new ClassAd();
- Value val = new Value();
- if (tree == null)
+ if (tree == null) {
return (null);
+ }
+ ClassAd scope = objectPool.classAdPool.get();
+ Value val = objectPool.valuePool.get();
tree.setParentScope(this);
if (!tree.publicEvaluate(val) || !val.isClassAdValue(scope)) {
return (null);
@@ -339,9 +290,9 @@
// --- begin integer attribute insertion ----
public boolean insertAttr(String name, int value, NumberFactor f) throws HyracksDataException {
ExprTree plit;
- Value val = new Value();
+ Value val = objectPool.valuePool.get();
val.setIntegerValue(value);
- plit = Literal.createLiteral(val, f);
+ plit = Literal.createLiteral(val, f, objectPool);
return insert(name, plit);
}
@@ -351,10 +302,10 @@
public boolean insertAttr(String name, long value, NumberFactor f) throws HyracksDataException {
ExprTree plit;
- Value val = new Value();
+ Value val = objectPool.valuePool.get();
val.setIntegerValue(value);
- plit = Literal.createLiteral(val, f);
+ plit = Literal.createLiteral(val, f, objectPool);
return (insert(name, plit));
}
@@ -365,16 +316,18 @@
public boolean deepInsertAttr(ExprTree scopeExpr, String name, int value, NumberFactor f)
throws HyracksDataException {
ClassAd ad = privateGetDeepScope(scopeExpr);
- if (ad == null)
+ if (ad == null) {
return (false);
+ }
return (ad.insertAttr(name, value, f));
}
public boolean deepInsertAttr(ExprTree scopeExpr, String name, long value, NumberFactor f)
throws HyracksDataException {
ClassAd ad = privateGetDeepScope(scopeExpr);
- if (ad == null)
+ if (ad == null) {
return (false);
+ }
return (ad.insertAttr(name, value, f));
}
@@ -383,17 +336,18 @@
// --- begin real attribute insertion ---
public boolean insertAttr(String name, double value, NumberFactor f) throws HyracksDataException {
ExprTree plit;
- Value val = new Value();
+ Value val = objectPool.valuePool.get();
val.setRealValue(value);
- plit = Literal.createLiteral(val, f);
+ plit = Literal.createLiteral(val, f, objectPool);
return (insert(name, plit));
}
public boolean deepInsertAttr(ExprTree scopeExpr, String name, double value, NumberFactor f)
throws HyracksDataException {
ClassAd ad = privateGetDeepScope(scopeExpr);
- if (ad == null)
+ if (ad == null) {
return (false);
+ }
return (ad.insertAttr(name, value, f));
}
@@ -402,16 +356,17 @@
// --- begin boolean attribute insertion
public boolean insertAttr(String name, boolean value) throws HyracksDataException {
ExprTree plit;
- Value val = new Value();
+ Value val = objectPool.valuePool.get();
val.setBooleanValue(value);
- plit = Literal.createLiteral(val);
+ plit = Literal.createLiteral(val, objectPool);
return (insert(name, plit));
}
public boolean deepInsertAttr(ExprTree scopeExpr, String name, boolean value) throws HyracksDataException {
ClassAd ad = privateGetDeepScope(scopeExpr);
- if (ad == null)
+ if (ad == null) {
return (false);
+ }
return (ad.insertAttr(name, value));
}
@@ -420,33 +375,35 @@
// --- begin string attribute insertion
public boolean insertAttr(String name, AMutableCharArrayString value) throws HyracksDataException {
ExprTree plit;
- Value val = new Value();
+ Value val = objectPool.valuePool.get();
val.setStringValue(value);
- plit = Literal.createLiteral(val);
+ plit = Literal.createLiteral(val, objectPool);
return (insert(name, plit));
}
public boolean deepInsertAttr(ExprTree scopeExpr, String name, AMutableCharArrayString value)
throws HyracksDataException {
ClassAd ad = privateGetDeepScope(scopeExpr);
- if (ad == null)
+ if (ad == null) {
return (false);
+ }
return (ad.insertAttr(name, value));
}
public boolean insertAttr(String name, String value) throws HyracksDataException {
ExprTree plit;
- Value val = new Value();
+ Value val = objectPool.valuePool.get();
val.setStringValue(value);
- plit = Literal.createLiteral(val);
+ plit = Literal.createLiteral(val, objectPool);
return (insert(name, plit));
}
public boolean deepInsertAttr(ExprTree scopeExpr, String name, String value) throws HyracksDataException {
ClassAd ad = privateGetDeepScope(scopeExpr);
- if (ad == null)
+ if (ad == null) {
return (false);
+ }
return (ad.insertAttr(name, value));
}
@@ -498,7 +455,7 @@
// cache doesn't already have an entry for this name:value, so add
// it to the cache now.
if (newTree.getKind() != NodeKind.LITERAL_NODE) {
- Literal lit = parser.getLiteral();
+ Literal lit = objectPool.literalPool.get();
lit.getValue().setStringValue(szValue);
bRet = insert(name, lit, false);
} else {
@@ -511,7 +468,8 @@
}
public boolean insert(String attrName, ExprTree expr) throws HyracksDataException {
- boolean result = insert(attrName, expr.isTreeHolder() ? ((ExprTreeHolder) expr).getInnerTree() : expr, false);
+ ExprTree tree = expr.copy();
+ boolean result = insert(attrName, tree.isTreeHolder() ? ((ExprTreeHolder) tree).getInnerTree() : tree, false);
return result;
}
@@ -522,13 +480,14 @@
if (attrName.isEmpty() || pRef == null) {
throw new HyracksDataException();
}
- CaseInsensitiveString pstrAttr = StringPool.get();
- pstrAttr.set(attrName);
-
if (tree != null) {
+ CaseInsensitiveString pstrAttr = objectPool.caseInsensitiveStringPool.get();
+ pstrAttr.set(attrName);
+ ExprTreeHolder mutableTree = objectPool.mutableExprPool.get();
+ mutableTree.copyFrom(tree);
// parent of the expression is this classad
tree.setParentScope(this);
- attrList.put(pstrAttr, tree);
+ attrList.put(pstrAttr, mutableTree);
bRet = true;
}
return (bRet);
@@ -536,8 +495,9 @@
public boolean deepInsert(ExprTree scopeExpr, String name, ExprTree tree) throws HyracksDataException {
ClassAd ad = privateGetDeepScope(scopeExpr);
- if (ad == null)
+ if (ad == null) {
return (false);
+ }
return (ad.insert(name, tree));
}
@@ -545,10 +505,9 @@
// --- begin lookup methods
public ExprTree lookup(String name) {
- CaseInsensitiveString aString = StringPool.get();
+ CaseInsensitiveString aString = objectPool.caseInsensitiveStringPool.get();
aString.set(name);
ExprTree expr = lookup(aString);
- StringPool.put(aString);
return expr;
}
@@ -569,9 +528,9 @@
}
}
- public ExprTree lookupInScope(AMutableCharArrayString name, ClassAd finalScope) {
- EvalState state = new EvalState();
- ExprTreeHolder tree = new ExprTreeHolder();
+ public ExprTree lookupInScope(AMutableCharArrayString name, ClassAd finalScope) throws HyracksDataException {
+ EvalState state = objectPool.evalStatePool.get();
+ ExprTreeHolder tree = objectPool.mutableExprPool.get();
int rval;
state.setScopes(this);
rval = lookupInScope(name.toString(), tree, state);
@@ -583,10 +542,10 @@
return null;
}
- public int lookupInScope(String name, ExprTreeHolder expr, EvalState state) {
+ public int lookupInScope(String name, ExprTreeHolder expr, EvalState state) throws HyracksDataException {
ClassAd current = this;
- ClassAd superScope = new ClassAd();
+ ClassAd superScope = objectPool.classAdPool.get();
expr.setInnerTree(null);
while (expr.getInnerTree() == null && current != null) {
@@ -598,11 +557,17 @@
if ((expr.getInnerTree() != null)) {
return EvalResult.EVAL_OK.ordinal();
}
-
- if (state.getRootAd().equals(current)) {
- superScope = null;
- } else {
- superScope = current.parentScope;
+ try {
+ if (state.getRootAd() == null) {
+ return (EvalResult.EVAL_UNDEF.ordinal());
+ } else if (state.getRootAd().equals(current)) {
+ superScope = null;
+ } else {
+ superScope = current.parentScope;
+ }
+ } catch (Throwable th) {
+ th.printStackTrace();
+ throw th;
}
if (!getSpecialAttrNames().contains(name)) {
// continue searching from the superScope ...
@@ -638,10 +603,9 @@
// --- begin deletion methods
public boolean delete(String name) throws HyracksDataException {
- CaseInsensitiveString aString = StringPool.get();
+ CaseInsensitiveString aString = objectPool.caseInsensitiveStringPool.get();
aString.set(name);
boolean success = delete(aString);
- StringPool.put(aString);
return success;
}
@@ -657,10 +621,10 @@
// behavior copied from old ClassAds. It's also one reason you
// probably don't want to use this feature in the future.
if (chainedParentAd != null && chainedParentAd.lookup(name) != null) {
- Value undefined_value = new Value();
+ Value undefined_value = objectPool.valuePool.get();
undefined_value.setUndefinedValue();
deleted_attribute = true;
- ExprTree plit = Literal.createLiteral(undefined_value);
+ ExprTree plit = Literal.createLiteral(undefined_value, objectPool);
insert(name.get(), plit);
}
return deleted_attribute;
@@ -668,12 +632,12 @@
public boolean deepDelete(ExprTree scopeExpr, String name) throws HyracksDataException {
ClassAd ad = privateGetDeepScope(scopeExpr);
- if (ad == null)
+ if (ad == null) {
return (false);
- CaseInsensitiveString aString = StringPool.get();
+ }
+ CaseInsensitiveString aString = objectPool.caseInsensitiveStringPool.get();
aString.set(name);;
boolean success = ad.delete(aString);
- StringPool.put(aString);
return success;
}
@@ -694,9 +658,9 @@
if (tree == null) {
tree = chainedParentAd.lookup(name);
}
- Value undefined_value = new Value();
+ Value undefined_value = objectPool.valuePool.get();
undefined_value.setUndefinedValue();
- ExprTree plit = Literal.createLiteral(undefined_value);
+ ExprTree plit = Literal.createLiteral(undefined_value, objectPool);
//why??
insert(name, plit);
}
@@ -705,8 +669,9 @@
public ExprTree deepRemove(ExprTree scopeExpr, String name) throws HyracksDataException {
ClassAd ad = privateGetDeepScope(scopeExpr);
- if (ad == null)
+ if (ad == null) {
return (null);
+ }
return (ad.remove(name));
}
@@ -721,7 +686,7 @@
public void modify(ClassAd mod) throws HyracksDataException {
ClassAd ctx;
ExprTree expr;
- Value val = new Value();
+ Value val = objectPool.valuePool.get();
// Step 0: Determine Context
if ((expr = mod.lookup(Common.ATTR_CONTEXT)) != null) {
@@ -734,7 +699,7 @@
// Step 1: Process Replace attribute
if ((expr = mod.lookup(Common.ATTR_REPLACE)) != null) {
- ClassAd ad = new ClassAd();
+ ClassAd ad = objectPool.classAdPool.get();
if (expr.publicEvaluate(val) && val.isClassAdValue(ad)) {
ctx.clear();
ctx.update(ad);
@@ -743,7 +708,7 @@
// Step 2: Process Updates attribute
if ((expr = mod.lookup(Common.ATTR_UPDATES)) != null) {
- ClassAd ad = new ClassAd();
+ ClassAd ad = objectPool.classAdPool.get();
if (expr.publicEvaluate(val) && val.isClassAdValue(ad)) {
ctx.update(ad);
}
@@ -751,8 +716,8 @@
// Step 3: Process Deletes attribute
if ((expr = mod.lookup(Common.ATTR_DELETES)) != null) {
- ExprList list = new ExprList();
- AMutableCharArrayString attrName = new AMutableCharArrayString();
+ ExprList list = objectPool.exprListPool.get();
+ AMutableCharArrayString attrName = objectPool.strPool.get();
// make a first pass to check that it is a list of strings ...
if (!expr.publicEvaluate(val) || !val.isListValue(list)) {
@@ -774,18 +739,18 @@
@Override
public ExprTree copy() throws HyracksDataException {
- ClassAd newAd = new ClassAd();
- newAd.parentScope = parentScope;
- newAd.chainedParentAd = chainedParentAd;
+ ClassAd newAd = objectPool.classAdPool.get();
+ newAd.parentScope = (parentScope == null) ? null : (ClassAd) parentScope.copy();
+ newAd.chainedParentAd = chainedParentAd == null ? null : (ClassAd) chainedParentAd.copy();
for (Entry<CaseInsensitiveString, ExprTree> entry : attrList.entrySet()) {
- newAd.insert(entry.getKey().get(), entry.getValue().copy(), false);
+ newAd.insert(entry.getKey().get(), entry.getValue(), false);
}
return newAd;
}
@Override
- public boolean publicEvaluate(EvalState state, Value val) {
+ public boolean publicEvaluate(EvalState state, Value val) throws HyracksDataException {
val.setClassAdValue(this);
return (true);
}
@@ -800,9 +765,9 @@
@Override
public boolean privateFlatten(EvalState state, Value val, ExprTreeHolder tree, AMutableInt32 i)
throws HyracksDataException {
- ClassAd newAd = new ClassAd();
- Value eval = new Value();
- ExprTreeHolder etree = new ExprTreeHolder();
+ ClassAd newAd = objectPool.classAdPool.get();
+ Value eval = objectPool.valuePool.get();
+ ExprTreeHolder etree = objectPool.mutableExprPool.get();;
ClassAd oldAd;
tree.setInnerTree(null); // Just to be safe... wenger 2003-12-11.
@@ -814,23 +779,27 @@
// flatten expression
if (!entry.getValue().publicFlatten(state, eval, etree)) {
tree.setInnerTree(null);;
- eval.clear();
+ eval.setUndefinedValue();
state.setCurAd(oldAd);
return false;
}
// if a value was obtained, convert it to a literal
if (etree.getInnerTree() == null) {
- etree.setInnerTree(Literal.createLiteral(eval));
+ etree.setInnerTree(Literal.createLiteral(eval, objectPool));
if (etree.getInnerTree() == null) {
tree.setInnerTree(null);
- eval.clear();
+ eval.setUndefinedValue();
state.setCurAd(oldAd);
return false;
}
}
- newAd.attrList.put(entry.getKey(), etree);
- eval.clear();
+ CaseInsensitiveString key = objectPool.caseInsensitiveStringPool.get();
+ ExprTreeHolder value = objectPool.mutableExprPool.get();
+ key.set(entry.getKey().get());
+ value.copyFrom(etree);
+ newAd.attrList.put(key, value);
+ eval.setUndefinedValue();
}
tree.setInnerTree(newAd);
@@ -839,8 +808,8 @@
}
public boolean evaluateAttr(String attr, Value val) throws HyracksDataException {
- EvalState state = new EvalState();
- ExprTreeHolder tree = new ExprTreeHolder();
+ EvalState state = objectPool.evalStatePool.get();
+ ExprTreeHolder tree = objectPool.mutableExprPool.get();
state.setScopes(this);
switch (lookupInScope(attr, tree, state)) {
case ExprTree.EVAL_FAIL_Int:
@@ -860,8 +829,8 @@
public boolean evaluateExpr(String buf, Value result) throws HyracksDataException {
boolean successfully_evaluated;
- ExprTreeHolder tree = new ExprTreeHolder();
- ClassAdParser parser = new ClassAdParser(null, false, true, false, null, null, null);
+ ExprTreeHolder tree = objectPool.mutableExprPool.get();
+ ClassAdParser parser = objectPool.classAdParserPool.get();
try {
if (parser.parseExpression(buf, tree)) {
@@ -876,54 +845,54 @@
}
public boolean evaluateExpr(ExprTreeHolder tree, Value val) throws HyracksDataException {
- EvalState state = new EvalState();
+ EvalState state = objectPool.evalStatePool.get();
state.setScopes(this);
return (tree.publicEvaluate(state, val));
}
public boolean evaluateExpr(ExprTreeHolder tree, Value val, ExprTreeHolder sig) throws HyracksDataException {
- EvalState state = new EvalState();
+ EvalState state = objectPool.evalStatePool.get();
state.setScopes(this);
return (tree.publicEvaluate(state, val, sig));
}
public boolean evaluateAttrInt(String attr, AMutableInt64 i) throws HyracksDataException {
- Value val = new Value();
+ Value val = objectPool.valuePool.get();
return (evaluateAttr(attr, val) && val.isIntegerValue(i));
}
public boolean evaluateAttrReal(String attr, AMutableDouble r) throws HyracksDataException {
- Value val = new Value();
+ Value val = objectPool.valuePool.get();
return (evaluateAttr(attr, val) && val.isRealValue(r));
}
public boolean evaluateAttrNumber(String attr, AMutableInt64 i) throws HyracksDataException {
- Value val = new Value();
+ Value val = objectPool.valuePool.get();
return (evaluateAttr(attr, val) && val.isNumber(i));
}
public boolean evaluateAttrNumber(String attr, AMutableDouble r) throws HyracksDataException {
- Value val = new Value();
+ Value val = objectPool.valuePool.get();
return (evaluateAttr(attr, val) && val.isNumber(r));
}
public boolean evaluateAttrString(String attr, AMutableCharArrayString buf, int len) throws HyracksDataException {
- Value val = new Value();
+ Value val = objectPool.valuePool.get();
return (evaluateAttr(attr, val) && val.isStringValue(buf, len));
}
public boolean evaluateAttrString(String attr, AMutableCharArrayString buf) throws HyracksDataException {
- Value val = new Value();
+ Value val = objectPool.valuePool.get();
return (evaluateAttr(attr, val) && val.isStringValue(buf));
}
public boolean evaluateAttrBool(String attr, MutableBoolean b) throws HyracksDataException {
- Value val = new Value();
+ Value val = objectPool.valuePool.get();
return (evaluateAttr(attr, val) && val.isBooleanValue(b));
}
public boolean evaluateAttrBoolEquiv(String attr, MutableBoolean b) throws HyracksDataException {
- Value val = new Value();
+ Value val = objectPool.valuePool.get();
return (evaluateAttr(attr, val) && val.isBooleanValueEquiv(b));
}
@@ -942,7 +911,7 @@
public boolean getExternalReferences(ExprTree tree, TreeSet<String> refs, boolean fullNames)
throws HyracksDataException {
- EvalState state = new EvalState();
+ EvalState state = objectPool.evalStatePool.get();
// Treat this ad as the root of the tree for reference tracking.
// If an attribute is only present in a parent scope of this ad,
// then we want to treat it as an external reference.
@@ -962,12 +931,12 @@
return (true);
case ATTRREF_NODE: {
- ClassAd start = new ClassAd();
- ExprTreeHolder tree = new ExprTreeHolder();
- ExprTreeHolder result = new ExprTreeHolder();
- AMutableCharArrayString attr = new AMutableCharArrayString();
- Value val = new Value();
- MutableBoolean abs = new MutableBoolean();
+ ClassAd start = objectPool.classAdPool.get();
+ ExprTreeHolder tree = objectPool.mutableExprPool.get();
+ ExprTreeHolder result = objectPool.mutableExprPool.get();
+ AMutableCharArrayString attr = objectPool.strPool.get();
+ Value val = objectPool.valuePool.get();
+ MutableBoolean abs = objectPool.boolPool.get();
((AttributeReference) expr).getComponents(tree, attr, abs);
// establish starting point for attribute search
@@ -984,9 +953,9 @@
// are in the tree part
if (val.isUndefinedValue()) {
if (fullNames) {
- AMutableCharArrayString fullName = new AMutableCharArrayString();
+ AMutableCharArrayString fullName = objectPool.strPool.get();
if (tree.getInnerTree() != null) {
- ClassAdUnParser unparser = new PrettyPrint();
+ ClassAdUnParser unparser = objectPool.prettyPrintPool.get();
unparser.unparse(fullName, tree);
fullName.appendChar('.');
}
@@ -1041,10 +1010,10 @@
}
case OP_NODE: {
// recurse on subtrees
- AMutableInt32 opKind = new AMutableInt32(0);
- ExprTreeHolder t1 = new ExprTreeHolder();
- ExprTreeHolder t2 = new ExprTreeHolder();
- ExprTreeHolder t3 = new ExprTreeHolder();
+ AMutableInt32 opKind = objectPool.int32Pool.get();
+ ExprTreeHolder t1 = objectPool.mutableExprPool.get();
+ ExprTreeHolder t2 = objectPool.mutableExprPool.get();
+ ExprTreeHolder t3 = objectPool.mutableExprPool.get();
((Operation) expr).getComponents(opKind, t1, t2, t3);
if (t1.getInnerTree() != null && !privateGetExternalReferences(t1, ad, state, refs, fullNames)) {
@@ -1060,8 +1029,8 @@
}
case FN_CALL_NODE: {
// recurse on subtrees
- AMutableCharArrayString fnName = new AMutableCharArrayString();
- ExprList args = new ExprList();
+ AMutableCharArrayString fnName = objectPool.strPool.get();
+ ExprList args = objectPool.exprListPool.get();
((FunctionCall) expr).getComponents(fnName, args);
for (ExprTree tree : args.getExprList()) {
if (!privateGetExternalReferences(tree, ad, state, refs, fullNames)) {
@@ -1072,8 +1041,8 @@
}
case CLASSAD_NODE: {
// recurse on subtrees
- Map<CaseInsensitiveString, ExprTree> attrs = new HashMap<CaseInsensitiveString, ExprTree>();
- ((ClassAd) expr).getComponents(attrs);
+ Map<CaseInsensitiveString, ExprTree> attrs = objectPool.strToExprPool.get();
+ ((ClassAd) expr).getComponents(attrs, objectPool);
for (Entry<CaseInsensitiveString, ExprTree> entry : attrs.entrySet()) {
if (state.getDepthRemaining() <= 0) {
return false;
@@ -1089,7 +1058,7 @@
}
case EXPR_LIST_NODE: {
// recurse on subtrees
- ExprList exprs = new ExprList();
+ ExprList exprs = objectPool.exprListPool.get();
((ExprList) expr).getComponents(exprs);
for (ExprTree exprTree : exprs.getExprList()) {
@@ -1115,7 +1084,7 @@
// PortReferences is a Map<ClassAd,TreeSet<Strings>>
public boolean getExternalReferences(ExprTree tree, Map<ClassAd, TreeSet<String>> refs)
throws HyracksDataException {
- EvalState state = new EvalState();
+ EvalState state = objectPool.evalStatePool.get();
// Treat this ad as the root of the tree for reference tracking.
// If an attribute is only present in a parent scope of this ad,
// then we want to treat it as an external reference.
@@ -1133,12 +1102,12 @@
return (true);
case ATTRREF_NODE: {
- ClassAd start = new ClassAd();
- ExprTreeHolder tree = new ExprTreeHolder();
- ExprTreeHolder result = new ExprTreeHolder();
- AMutableCharArrayString attr = new AMutableCharArrayString();
- Value val = new Value();
- MutableBoolean abs = new MutableBoolean();
+ ClassAd start = objectPool.classAdPool.get();
+ ExprTreeHolder tree = objectPool.mutableExprPool.get();
+ ExprTreeHolder result = objectPool.mutableExprPool.get();
+ AMutableCharArrayString attr = objectPool.strPool.get();
+ Value val = objectPool.valuePool.get();
+ MutableBoolean abs = objectPool.boolPool.get();
((AttributeReference) expr).getComponents(tree, attr, abs);
// establish starting point for attribute search
@@ -1148,8 +1117,9 @@
return false; // NAC
} // NAC
} else {
- if (!tree.publicEvaluate(state, val))
+ if (!tree.publicEvaluate(state, val)) {
return (false);
+ }
// if the tree evals to undefined, the external references
// are in the tree part
if (val.isUndefinedValue()) {
@@ -1157,8 +1127,9 @@
}
// otherwise, if the tree didn't evaluate to a classad,
// we have a problem
- if (!val.isClassAdValue(start))
+ if (!val.isClassAdValue(start)) {
return (false);
+ }
// make sure that we are starting from a "valid" scope
if (!refs.containsKey(start) && start != this) {
@@ -1169,7 +1140,7 @@
ClassAd curAd = state.getCurAd();
TreeSet<String> pitr = refs.get(start);
if (pitr == null) {
- pitr = new TreeSet<String>();
+ pitr = objectPool.strSetPool.get();
refs.put(start, pitr);
}
switch (start.lookupInScope(attr.toString(), result, state)) {
@@ -1198,10 +1169,10 @@
case OP_NODE: {
// recurse on subtrees
- AMutableInt32 opKind = new AMutableInt32(0);
- ExprTreeHolder t1 = new ExprTreeHolder();
- ExprTreeHolder t2 = new ExprTreeHolder();
- ExprTreeHolder t3 = new ExprTreeHolder();
+ AMutableInt32 opKind = objectPool.int32Pool.get();
+ ExprTreeHolder t1 = objectPool.mutableExprPool.get();
+ ExprTreeHolder t2 = objectPool.mutableExprPool.get();
+ ExprTreeHolder t3 = objectPool.mutableExprPool.get();
((Operation) expr).getComponents(opKind, t1, t2, t3);
if (t1.getInnerTree() != null && !privateGetExternalReferences(t1, ad, state, refs)) {
return (false);
@@ -1217,8 +1188,8 @@
case FN_CALL_NODE: {
// recurse on subtrees
- AMutableCharArrayString fnName = new AMutableCharArrayString();
- ExprList args = new ExprList();
+ AMutableCharArrayString fnName = objectPool.strPool.get();
+ ExprList args = objectPool.exprListPool.get();
((FunctionCall) expr).getComponents(fnName, args);
for (ExprTree exprTree : args.getExprList()) {
@@ -1231,9 +1202,9 @@
case CLASSAD_NODE: {
// recurse on subtrees
- HashMap<CaseInsensitiveString, ExprTree> attrs = new HashMap<CaseInsensitiveString, ExprTree>();
+ HashMap<CaseInsensitiveString, ExprTree> attrs = objectPool.strToExprPool.get();
- ((ClassAd) expr).getComponents(attrs);
+ ((ClassAd) expr).getComponents(attrs, objectPool);
for (Entry<CaseInsensitiveString, ExprTree> entry : attrs.entrySet()) {
if (!privateGetExternalReferences(entry.getValue(), ad, state, refs)) {
return (false);
@@ -1244,7 +1215,7 @@
case EXPR_LIST_NODE: {
// recurse on subtrees
- ExprList exprs = new ExprList();
+ ExprList exprs = objectPool.exprListPool.get();
((ExprList) expr).getComponents(exprs);
for (ExprTree exprTree : exprs.getExprList()) {
if (!privateGetExternalReferences(exprTree, ad, state, refs)) {
@@ -1273,7 +1244,7 @@
*/
public boolean getInternalReferences(ExprTree tree, TreeSet<String> refs, boolean fullNames)
throws HyracksDataException {
- EvalState state = new EvalState();
+ EvalState state = objectPool.evalStatePool.get();
// Treat this ad as the root of the tree for reference tracking.
// If an attribute is only present in a parent scope of this ad,
@@ -1295,12 +1266,12 @@
}
case ATTRREF_NODE: {
- ClassAd start = new ClassAd();
- ExprTreeHolder tree = new ExprTreeHolder();
- ExprTreeHolder result = new ExprTreeHolder();
- AMutableCharArrayString attr = new AMutableCharArrayString();
- Value val = new Value();
- MutableBoolean abs = new MutableBoolean();
+ ClassAd start = objectPool.classAdPool.get();;
+ ExprTreeHolder tree = objectPool.mutableExprPool.get();
+ ExprTreeHolder result = objectPool.mutableExprPool.get();
+ AMutableCharArrayString attr = objectPool.strPool.get();
+ Value val = objectPool.valuePool.get();
+ MutableBoolean abs = objectPool.boolPool.get();
((AttributeReference) expr).getComponents(tree, attr, abs);
@@ -1393,10 +1364,10 @@
case OP_NODE: {
//recurse on subtrees
- AMutableInt32 op = new AMutableInt32(0);
- ExprTreeHolder t1 = new ExprTreeHolder();
- ExprTreeHolder t2 = new ExprTreeHolder();
- ExprTreeHolder t3 = new ExprTreeHolder();
+ AMutableInt32 op = objectPool.int32Pool.get();
+ ExprTreeHolder t1 = objectPool.mutableExprPool.get();
+ ExprTreeHolder t2 = objectPool.mutableExprPool.get();
+ ExprTreeHolder t3 = objectPool.mutableExprPool.get();
((Operation) expr).getComponents(op, t1, t2, t3);
if (t1.getInnerTree() != null && !privateGetInternalReferences(t1, ad, state, refs, fullNames)) {
return false;
@@ -1414,8 +1385,8 @@
case FN_CALL_NODE: {
//recurse on the subtrees!
- AMutableCharArrayString fnName = new AMutableCharArrayString();
- ExprList args = new ExprList();
+ AMutableCharArrayString fnName = objectPool.strPool.get();
+ ExprList args = objectPool.exprListPool.get();
((FunctionCall) expr).getComponents(fnName, args);
for (ExprTree exprTree : args.getExprList()) {
@@ -1429,7 +1400,7 @@
case CLASSAD_NODE: {
//also recurse on subtrees...
- HashMap<CaseInsensitiveString, ExprTree> attrs = new HashMap<CaseInsensitiveString, ExprTree>();
+ HashMap<CaseInsensitiveString, ExprTree> attrs = objectPool.strToExprPool.get();
// If this ClassAd is only being used here as the scoping
// for an attribute reference, don't recurse into all of
@@ -1438,7 +1409,7 @@
return true;
}
- ((ClassAd) expr).getComponents(attrs);
+ ((ClassAd) expr).getComponents(attrs, objectPool);
for (Entry<CaseInsensitiveString, ExprTree> entry : attrs.entrySet()) {
if (state.getDepthRemaining() <= 0) {
return false;
@@ -1457,7 +1428,7 @@
}
case EXPR_LIST_NODE: {
- ExprList exprs = new ExprList();
+ ExprList exprs = objectPool.exprListPool.get();
((ExprList) expr).getComponents(exprs);
for (ExprTree exprTree : exprs.getExprList()) {
@@ -1484,14 +1455,14 @@
}
public boolean publicFlatten(ExprTree tree, Value val, ExprTreeHolder fexpr) throws HyracksDataException {
- EvalState state = new EvalState();
+ EvalState state = objectPool.evalStatePool.get();
state.setScopes(this);
return (tree.publicFlatten(state, val, fexpr));
}
public boolean flattenAndInline(ExprTree tree, Value val, ExprTreeHolder fexpr) throws HyracksDataException {
- EvalState state = new EvalState();
+ EvalState state = objectPool.evalStatePool.get();
state.setScopes(this);
state.setFlattenAndInline(true);
@@ -1529,12 +1500,8 @@
return chainedParentAd;
}
- public void setValue(ClassAd value) {
- this.attrList = value.attrList;
- this.alternateScope = value.alternateScope;
- this.chainedParentAd = value.chainedParentAd;
- this.parentScope = value.parentScope;
- this.size = value.size;
+ public void setValue(ClassAd value) throws HyracksDataException {
+ copyFrom(value);
}
@Override
@@ -1566,6 +1533,6 @@
}
public void createParser() {
- parser = new ClassAdParser(null, false, true, false, null, null, null);
+ parser = objectPool.classAdParserPool.get();
}
}
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdFunc.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdFunc.java
index 4e77bc0..8a951ad 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdFunc.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdFunc.java
@@ -18,8 +18,10 @@
*/
package org.apache.asterix.external.classad;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.hyracks.api.exceptions.HyracksDataException;
public interface ClassAdFunc {
- public boolean call(String name, ExprList argList, EvalState state, Value val) throws HyracksDataException;
+ public boolean call(String name, ExprList argList, EvalState state, Value val, ClassAdObjectPool objectPool)
+ throws HyracksDataException;
}
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdTime.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdTime.java
index 66c5f56..359511b 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdTime.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdTime.java
@@ -92,6 +92,7 @@
}
public void setRelativeTime(long ms) {
+ this.isAbsolute = false;
timeZoneCalendar.setTimeInMillis(ms);
}
@@ -141,6 +142,17 @@
this.timeZoneCalendar.setTimeInMillis(0);
}
+ public void setCurrentAbsolute() {
+ this.isAbsolute = true;
+ this.timeZoneCalendar = Calendar.getInstance();
+ this.timeZoneCalendar.setTimeInMillis(0);
+ }
+
+ public void setTimeZone(String timeZoneId) {
+ this.timeZoneCalendar = Calendar.getInstance(TimeZone.getTimeZone(timeZoneId));
+ this.timeZoneCalendar.setTimeInMillis(0);
+ }
+
public ClassAdTime(String timeZoneId) {
this.isAbsolute = true;
this.timeZoneCalendar = Calendar.getInstance(TimeZone.getTimeZone(timeZoneId));
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdUnParser.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdUnParser.java
index 4689612..41333c5 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdUnParser.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdUnParser.java
@@ -18,12 +18,12 @@
*/
package org.apache.asterix.external.classad;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import org.apache.asterix.external.classad.Value.NumberFactor;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.asterix.om.base.AMutableDouble;
import org.apache.asterix.om.base.AMutableInt32;
import org.apache.asterix.om.base.AMutableInt64;
@@ -38,8 +38,11 @@
" >>> ", " () ", " [] ", " ?: " };
protected static char delimiter = '\"';
+ protected final ClassAdObjectPool objectPool;
+
/// Constructor
- public ClassAdUnParser() {
+ public ClassAdUnParser(ClassAdObjectPool objectPool) {
+ this.objectPool = objectPool;
}
// The default delimiter for strings is '\"'
@@ -64,7 +67,7 @@
break;
case STRING_VALUE: {
- AMutableCharArrayString s = new AMutableCharArrayString();
+ AMutableCharArrayString s = objectPool.strPool.get();
val.isStringValue(s);
buffer.appendChar('"');
for (int i = 0; i < s.getLength(); i++) {
@@ -118,13 +121,13 @@
return;
}
case INTEGER_VALUE: {
- AMutableInt64 i = new AMutableInt64(0);
+ AMutableInt64 i = objectPool.int64Pool.get();
val.isIntegerValue(i);
buffer.appendString(String.valueOf(i.getLongValue()));
return;
}
case REAL_VALUE: {
- AMutableDouble real = new AMutableDouble(0);
+ AMutableDouble real = objectPool.doublePool.get();
val.isRealValue(real);
if (real.getDoubleValue() == 0.0) {
// It might be positive or negative and it's
@@ -145,7 +148,7 @@
return;
}
case BOOLEAN_VALUE: {
- MutableBoolean b = new MutableBoolean();
+ MutableBoolean b = objectPool.boolPool.get();
val.isBooleanValue(b);
buffer.appendString(b.booleanValue() ? "true" : "false");
return;
@@ -159,7 +162,7 @@
return;
}
case ABSOLUTE_TIME_VALUE: {
- ClassAdTime asecs = new ClassAdTime();
+ ClassAdTime asecs = objectPool.classAdTimePool.get();
val.isAbsoluteTimeValue(asecs);
buffer.appendString("absTime(\"");
@@ -168,7 +171,7 @@
return;
}
case RELATIVE_TIME_VALUE: {
- ClassAdTime rsecs = new ClassAdTime();
+ ClassAdTime rsecs = objectPool.classAdTimePool.get();
val.isRelativeTimeValue(rsecs);
buffer.appendString("relTime(\"");
Util.relTimeToString(rsecs.getRelativeTime(), buffer);
@@ -177,16 +180,16 @@
return;
}
case CLASSAD_VALUE: {
- ClassAd ad = new ClassAd();
- Map<CaseInsensitiveString, ExprTree> attrs = new HashMap<CaseInsensitiveString, ExprTree>();
+ ClassAd ad = objectPool.classAdPool.get();
+ Map<CaseInsensitiveString, ExprTree> attrs = objectPool.strToExprPool.get();
val.isClassAdValue(ad);
- ad.getComponents(attrs);
+ ad.getComponents(attrs, objectPool);
unparseAux(buffer, attrs);
return;
}
case SLIST_VALUE:
case LIST_VALUE: {
- ExprList el = new ExprList();
+ ExprList el = objectPool.exprListPool.get();
val.isListValue(el);
unparseAux(buffer, el);
return;
@@ -211,49 +214,49 @@
switch (tree.getKind()) {
case LITERAL_NODE: { // value
- Value val = new Value();
- AMutableNumberFactor factor = new AMutableNumberFactor();
- ((Literal) tree.self()).getComponents(val, factor);
+ Value val = objectPool.valuePool.get();
+ AMutableNumberFactor factor = objectPool.numFactorPool.get();
+ ((Literal) tree.getTree()).getComponents(val, factor);
unparseAux(buffer, val, factor.getFactor());
return;
}
case ATTRREF_NODE: { // string
- ExprTreeHolder expr = new ExprTreeHolder(); //needs initialization
- AMutableCharArrayString ref = new AMutableCharArrayString();
- MutableBoolean absolute = new MutableBoolean();
- ((AttributeReference) tree.self()).getComponents(expr, ref, absolute);
+ ExprTreeHolder expr = objectPool.mutableExprPool.get(); //needs initialization
+ AMutableCharArrayString ref = objectPool.strPool.get();
+ MutableBoolean absolute = objectPool.boolPool.get();
+ ((AttributeReference) tree.getTree()).getComponents(expr, ref, absolute);
unparseAux(buffer, expr, ref, absolute.booleanValue());
return;
}
case OP_NODE: { //string
- AMutableInt32 op = new AMutableInt32(0);
- ExprTreeHolder t1 = new ExprTreeHolder();
- ExprTreeHolder t2 = new ExprTreeHolder();
- ExprTreeHolder t3 = new ExprTreeHolder();
- ((Operation) tree.self()).getComponents(op, t1, t2, t3);
+ AMutableInt32 op = objectPool.int32Pool.get();
+ ExprTreeHolder t1 = objectPool.mutableExprPool.get();
+ ExprTreeHolder t2 = objectPool.mutableExprPool.get();
+ ExprTreeHolder t3 = objectPool.mutableExprPool.get();
+ ((Operation) tree.getTree()).getComponents(op, t1, t2, t3);
unparseAux(buffer, op.getIntegerValue().intValue(), t1, t2, t3);
return;
}
case FN_CALL_NODE: { // string
- AMutableCharArrayString fnName = new AMutableCharArrayString();
- ExprList args = new ExprList();
- ((FunctionCall) tree.self()).getComponents(fnName, args);
+ AMutableCharArrayString fnName = objectPool.strPool.get();
+ ExprList args = objectPool.exprListPool.get();
+ ((FunctionCall) tree.getTree()).getComponents(fnName, args);
unparseAux(buffer, fnName, args);
return;
}
case CLASSAD_NODE: { // nested record
- Map<CaseInsensitiveString, ExprTree> attrs = new HashMap<CaseInsensitiveString, ExprTree>();
- ((ClassAd) tree.self()).getComponents(attrs);
+ Map<CaseInsensitiveString, ExprTree> attrs = objectPool.strToExprPool.get();
+ ((ClassAd) tree.getTree()).getComponents(attrs, objectPool);
unparseAux(buffer, attrs);
return;
}
case EXPR_LIST_NODE: { // list
- ExprList exprs = new ExprList();
- ((ExprList) tree.self()).getComponents(exprs);
+ ExprList exprs = objectPool.exprListPool.get();
+ ((ExprList) tree.getTree()).getComponents(exprs);
unparseAux(buffer, exprs);
return;
}
@@ -345,8 +348,8 @@
// to unparse attribute names (quoted & unquoted attributes)
public void unparseAux(AMutableCharArrayString buffer, AMutableCharArrayString identifier)
throws HyracksDataException {
- Value val = new Value();
- AMutableCharArrayString idstr = new AMutableCharArrayString();
+ Value val = objectPool.valuePool.get();
+ AMutableCharArrayString idstr = objectPool.strPool.get();
val.setStringValue(identifier);
setDelimiter('\''); // change the delimiter from string-literal mode to quoted attribute mode
@@ -386,8 +389,9 @@
buffer.appendString("." + attrName);
return;
}
- if (absolute)
+ if (absolute) {
buffer.appendChar('.');
+ }
unparseAux(buffer, attrName);
}
@@ -474,8 +478,8 @@
* it's unparsed either as a quoted attribute or non-quoted attribute
*/
public void unparseAux(AMutableCharArrayString buffer, String identifier) throws HyracksDataException {
- Value val = new Value();
- AMutableCharArrayString idstr = new AMutableCharArrayString();
+ Value val = objectPool.valuePool.get();
+ AMutableCharArrayString idstr = objectPool.strPool.get();
val.setStringValue(identifier);
setDelimiter('\''); // change the delimiter from string-literal mode to quoted attribute mode
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/EvalState.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/EvalState.java
index 0719fd8..41f34bc 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/EvalState.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/EvalState.java
@@ -18,6 +18,9 @@
*/
package org.apache.asterix.external.classad;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
public class EvalState {
private int depthRemaining; // max recursion depth - current depth
@@ -27,10 +30,13 @@
// It can be set to a closer parent scope. Then that ClassAd is
// treated like it has no parent scope for LookupInScope() and
// Evaluate().
+ private final ClassAd rootAdTemp;
+ private final ClassAd curAdTemp;
private ClassAd rootAd;
private ClassAd curAd;
private boolean flattenAndInline; // NAC
private boolean inAttrRefScope;
+ private final ClassAdObjectPool objectPool;
public boolean isInAttrRefScope() {
return inAttrRefScope;
@@ -44,12 +50,15 @@
this.inAttrRefScope = inAttrRefScope;
}
- public EvalState() {
- rootAd = new ClassAd();
- curAd = new ClassAd();
+ public EvalState(ClassAdObjectPool objectPool) {
+ this.objectPool = objectPool;
+ rootAd = new ClassAd(this.objectPool);
+ curAd = new ClassAd(this.objectPool);
depthRemaining = ExprTree.MAX_CLASSAD_RECURSION;
flattenAndInline = false; // NAC
inAttrRefScope = false;
+ rootAdTemp = rootAd;
+ curAdTemp = curAd;
}
public boolean isFlattenAndInline() {
@@ -76,13 +85,14 @@
prevScope = curScope;
curScope = curScope.getParentScope();
}
-
rootAd = prevScope;
}
return;
}
public void reset() {
+ rootAd = rootAdTemp;
+ curAd = curAdTemp;
rootAd.reset();
curAd.reset();
depthRemaining = ExprTree.MAX_CLASSAD_RECURSION;
@@ -117,4 +127,16 @@
public void setRootAd(ClassAd classAd) {
this.rootAd = classAd;
}
+
+ public void set(EvalState state) throws HyracksDataException {
+ rootAd = rootAdTemp;
+ curAd = curAdTemp;
+ rootAd.reset();
+ curAd.reset();
+ rootAd.copyFrom(state.rootAd);
+ curAd.copyFrom(state.curAd);
+ depthRemaining = state.depthRemaining;
+ flattenAndInline = state.flattenAndInline;
+ inAttrRefScope = state.inAttrRefScope;
+ }
}
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprList.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprList.java
index e9376e2..3c8405f 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprList.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprList.java
@@ -22,20 +22,41 @@
import java.util.Iterator;
import java.util.List;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.asterix.om.base.AMutableInt32;
import org.apache.hyracks.api.exceptions.HyracksDataException;
public class ExprList extends ExprTree {
- private List<ExprTree> exprList;
- private EvalState state = new EvalState();
+ private final List<ExprTree> exprList;
+ private final EvalState state;
public boolean isShared = false;
+ public ExprList(List<ExprTree> exprs, ClassAdObjectPool objectPool) {
+ super(objectPool);
+ exprList = new ArrayList<ExprTree>();
+ this.state = new EvalState(this.objectPool);
+ copyList(exprs);
+ return;
+ }
+
+ public ExprList(ClassAdObjectPool objectPool) {
+ super(objectPool);
+ this.state = new EvalState(this.objectPool);
+ this.exprList = new ArrayList<ExprTree>();
+ }
+
+ public ExprList(boolean b, ClassAdObjectPool objectPool) {
+ super(objectPool);
+ this.state = new EvalState(this.objectPool);
+ this.exprList = new ArrayList<ExprTree>();
+ this.isShared = b;
+ }
+
public boolean copyFrom(ExprList exprList) throws HyracksDataException {
this.exprList.clear();
- for (ExprTree expr : exprList.exprList) {
- this.exprList.add(expr.copy());
- }
+ copyList(exprList.getExprList());
+ this.state.set(exprList.state);
return true;
}
@@ -86,28 +107,8 @@
}
public void setExprList(List<ExprTree> exprList) {
- this.exprList = exprList;
- }
-
- public ExprList(List<ExprTree> exprs) {
- exprList = new ArrayList<ExprTree>();
- copyList(exprs);
- return;
- }
-
- public ExprList(ExprList other_list) throws HyracksDataException {
- exprList = new ArrayList<ExprTree>();
- copyFrom(other_list);
- return;
- }
-
- public ExprList() {
- exprList = new ArrayList<ExprTree>();
- }
-
- public ExprList(boolean b) {
- this.exprList = new ArrayList<ExprTree>();
- this.isShared = b;
+ this.exprList.clear();
+ this.exprList.addAll(exprList);
}
public void clear() {
@@ -116,7 +117,7 @@
@Override
public ExprTree copy() throws HyracksDataException {
- ExprList newList = new ExprList();
+ ExprList newList = objectPool.exprListPool.get();
newList.copyFrom(this);
return newList;
}
@@ -129,7 +130,7 @@
} else if (tree.getKind() != NodeKind.EXPR_LIST_NODE) {
is_same = false;
} else {
- ExprList other_list = (ExprList) tree;
+ ExprList other_list = (ExprList) tree.getTree();
if (exprList.size() != other_list.size()) {
is_same = false;
} else {
@@ -145,14 +146,14 @@
return is_same;
}
- public static ExprList createExprList(List<ExprTree> exprs) {
- ExprList el = new ExprList();
+ public static ExprList createExprList(List<ExprTree> exprs, ClassAdObjectPool objectPool) {
+ ExprList el = objectPool.exprListPool.get();
el.copyList(exprs);
return el;
}
- public static ExprList createExprList(ExprList exprs) {
- ExprList el = new ExprList();
+ public static ExprList createExprList(ExprList exprs, ClassAdObjectPool objectPool) {
+ ExprList el = objectPool.exprListPool.get();
el.copyList(exprs.exprList);
return el;
}
@@ -217,9 +218,9 @@
@Override
public boolean privateFlatten(EvalState state, Value val, ExprTreeHolder tree, AMutableInt32 aInt)
throws HyracksDataException {
- ExprTreeHolder nexpr = new ExprTreeHolder();
- Value tempVal = new Value();
- ExprList newList = new ExprList();
+ ExprTreeHolder nexpr = objectPool.mutableExprPool.get();
+ Value tempVal = objectPool.valuePool.get();
+ ExprList newList = objectPool.exprListPool.get();
tree.setInnerTree(null);; // Just to be safe... wenger 2003-12-11.
@@ -230,7 +231,7 @@
}
// if only a value was obtained, convert to an expression
if (nexpr.getInnerTree() == null) {
- nexpr.setInnerTree(Literal.createLiteral(tempVal));
+ nexpr.setInnerTree(Literal.createLiteral(tempVal, objectPool));
if (nexpr.getInnerTree() == null) {
return false;
}
@@ -249,10 +250,11 @@
}
public boolean getValue(Value val, ExprTree tree, EvalState es) throws HyracksDataException {
- EvalState currentState = new EvalState();
+ EvalState currentState = objectPool.evalStatePool.get();
- if (tree == null)
+ if (tree == null) {
return false;
+ }
// if called from user code, es == NULL so we use &state instead
currentState = (es != null) ? es : state;
@@ -276,5 +278,6 @@
@Override
public void reset() {
exprList.clear();
+ state.reset();
}
}
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprTree.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprTree.java
index ccbfd8b..2b5af38 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprTree.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprTree.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.external.classad;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.asterix.om.base.AMutableDouble;
import org.apache.asterix.om.base.AMutableInt32;
import org.apache.asterix.om.base.AMutableInt64;
@@ -71,15 +72,24 @@
public ClassAd parentScope;
private CallableDebugFunction userDebugFunction;
+ protected final ClassAdObjectPool objectPool;
public abstract void reset();
- public ExprTree() {
+ public ExprTree(ClassAdObjectPool objectPool) {
+ if (objectPool == null) {
+ System.out.println();
+ }
+ this.objectPool = objectPool;
this.parentScope = null;
this.size = 0;
}
- public ExprTree(ExprTree expr) {
+ public ExprTree(ExprTree expr, ClassAdObjectPool objectPool) {
+ if (objectPool == null) {
+ System.out.println();
+ }
+ this.objectPool = objectPool;
this.size = expr.size;
}
@@ -112,14 +122,7 @@
* into a ClassAd.
*/
public void setParentScope(ClassAd scope) {
- if (scope == null) {
- parentScope = null;
- return;
- }
- if (parentScope == null) {
- parentScope = new ClassAd();
- }
- parentScope.setValue(scope);
+ parentScope = scope;
privateSetParentScope(scope);
}
@@ -169,8 +172,8 @@
/// A debugging method; send expression to stdout
public void puke() throws HyracksDataException {
- PrettyPrint unp = new PrettyPrint();
- AMutableCharArrayString buffer = new AMutableCharArrayString();
+ PrettyPrint unp = objectPool.prettyPrintPool.get();
+ AMutableCharArrayString buffer = objectPool.strPool.get();
unp.unparse(buffer, this);
System.out.println(buffer.toString());
}
@@ -193,16 +196,17 @@
}
public void debugFormatValue(Value value, double time) throws HyracksDataException {
- MutableBoolean boolValue = new MutableBoolean(false);
- AMutableInt64 intValue = new AMutableInt64(0);
- AMutableDouble doubleValue = new AMutableDouble(0.0);
- AMutableCharArrayString stringValue = new AMutableCharArrayString();
+ MutableBoolean boolValue = objectPool.boolPool.get();
+ AMutableInt64 intValue = objectPool.int64Pool.get();
+ AMutableDouble doubleValue = objectPool.doublePool.get();
+ AMutableCharArrayString stringValue = objectPool.strPool.get();
- if (NodeKind.CLASSAD_NODE == getKind())
+ if (NodeKind.CLASSAD_NODE == getKind()) {
return;
+ }
- PrettyPrint unp = new PrettyPrint();
- AMutableCharArrayString buffer = new AMutableCharArrayString();
+ PrettyPrint unp = objectPool.prettyPrintPool.get();
+ AMutableCharArrayString buffer = objectPool.strPool.get();
unp.unparse(buffer, this);
String result = "Classad debug: ";
@@ -230,8 +234,9 @@
result += "UNDEFINED\n";
break;
case BOOLEAN_VALUE:
- if (value.isBooleanValue(boolValue))
+ if (value.isBooleanValue(boolValue)) {
result += boolValue.booleanValue() ? "TRUE\n" : "FALSE\n";
+ }
break;
case INTEGER_VALUE:
if (value.isIntegerValue(intValue)) {
@@ -305,7 +310,7 @@
* @throws HyracksDataException
*/
public boolean publicEvaluate(Value val) throws HyracksDataException {
- EvalState state = new EvalState();
+ EvalState state = objectPool.evalStatePool.get();
if (parentScope == null) {
val.setErrorValue();
return false;
@@ -340,13 +345,13 @@
}
public boolean publicEvaluate(Value val, ExprTreeHolder sig) throws HyracksDataException {
- EvalState state = new EvalState();
+ EvalState state = objectPool.evalStatePool.get();
state.setScopes(parentScope);
return (publicEvaluate(state, val, sig));
}
public boolean publicFlatten(Value val, ExprTreeHolder tree) throws HyracksDataException {
- EvalState state = new EvalState();
+ EvalState state = objectPool.evalStatePool.get();
state.setScopes(parentScope);
return (publicFlatten(state, val, tree));
}
@@ -375,8 +380,9 @@
@Override
public String toString() {
- ClassAdUnParser unparser = new PrettyPrint();
- AMutableCharArrayString string_representation = new AMutableCharArrayString();
+ ClassAdObjectPool objectPool = new ClassAdObjectPool();
+ ClassAdUnParser unparser = new ClassAdUnParser(objectPool);
+ AMutableCharArrayString string_representation = objectPool.strPool.get();
try {
unparser.unparse(string_representation, this);
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprTreeHolder.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprTreeHolder.java
index 89c5c0b..0e2f894 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprTreeHolder.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprTreeHolder.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.external.classad;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.asterix.om.base.AMutableInt32;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -34,9 +35,6 @@
if (tree == null) {
innerTree = null;
} else {
- if (tree.isTreeHolder()) {
- tree = ((ExprTreeHolder) tree).innerTree;
- }
if (innerTree == null) {
innerTree = tree.copy();
} else {
@@ -52,8 +50,8 @@
@Override
public void puke() throws HyracksDataException {
- PrettyPrint unp = new PrettyPrint();
- AMutableCharArrayString buffer = new AMutableCharArrayString();
+ PrettyPrint unp = objectPool.prettyPrintPool.get();
+ AMutableCharArrayString buffer = objectPool.strPool.get();
unp.unparse(buffer, innerTree);
System.out.println(buffer.toString());
}
@@ -78,11 +76,13 @@
return true;
}
- public ExprTreeHolder() {
+ public ExprTreeHolder(ClassAdObjectPool objectPool) {
+ super(objectPool);
innerTree = null;
}
- public ExprTreeHolder(ExprTree tree) {
+ public ExprTreeHolder(ExprTree tree, ClassAdObjectPool objectPool) {
+ super(objectPool);
setInnerTree(tree);
}
@@ -100,7 +100,10 @@
@Override
public ExprTree copy() throws HyracksDataException {
- return innerTree.copy();
+ if (innerTree != null) {
+ return innerTree.copy();
+ }
+ return null;
}
@Override
@@ -118,18 +121,18 @@
@Override
public boolean privateEvaluate(EvalState state, Value val) throws HyracksDataException {
- return innerTree.privateEvaluate(state, val);
+ return innerTree == null ? false : innerTree.privateEvaluate(state, val);
}
@Override
public boolean privateEvaluate(EvalState state, Value val, ExprTreeHolder tree) throws HyracksDataException {
- return innerTree.privateEvaluate(state, val, tree);
+ return innerTree == null ? false : innerTree.privateEvaluate(state, val, tree);
}
@Override
public boolean privateFlatten(EvalState state, Value val, ExprTreeHolder tree, AMutableInt32 op)
throws HyracksDataException {
- return innerTree.privateFlatten(state, val, tree, op);
+ return innerTree == null ? false : innerTree.privateFlatten(state, val, tree, op);
}
@Override
@@ -139,6 +142,8 @@
@Override
protected void privateSetParentScope(ClassAd scope) {
- innerTree.privateSetParentScope(scope);
+ if (innerTree != null) {
+ innerTree.privateSetParentScope(scope);
+ }
}
}
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/FunctionCall.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/FunctionCall.java
index bbc0e7a..234ba20 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/FunctionCall.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/FunctionCall.java
@@ -20,6 +20,7 @@
import java.util.HashMap;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.asterix.om.base.AMutableInt32;
import org.apache.asterix.om.base.AMutableString;
import org.apache.commons.lang3.mutable.MutableBoolean;
@@ -28,7 +29,7 @@
public class FunctionCall extends ExprTree {
public static boolean initialized = false;
-
+ public static final HashMap<String, ClassAdFunc> funcTable = new HashMap<String, ClassAdFunc>();
public static final ClassAdFunc[] ClassAdBuiltinFunc = { BuiltinClassAdFunctions.IsType,
BuiltinClassAdFunctions.TestMember, BuiltinClassAdFunctions.Size, BuiltinClassAdFunctions.SumAvg,
BuiltinClassAdFunctions.MinMax, BuiltinClassAdFunctions.ListCompare, BuiltinClassAdFunctions.debug,
@@ -43,12 +44,6 @@
BuiltinClassAdFunctions.ifThenElse, BuiltinClassAdFunctions.stringListsIntersect,
BuiltinClassAdFunctions.interval, BuiltinClassAdFunctions.eval };
- // function call specific information
- private String functionName;
- private ClassAdFunc function;
- private ExprList arguments;
- public static final HashMap<String, ClassAdFunc> funcTable = new HashMap<String, ClassAdFunc>();
-
static {
// load up the function dispatch table
// type predicates
@@ -144,6 +139,36 @@
initialized = true;
}
+ // function call specific information
+ private final CaseInsensitiveString functionName;
+ private ClassAdFunc function;
+ private final ExprList arguments;
+
+ public FunctionCall(ClassAdObjectPool objectPool) {
+ super(objectPool);
+ functionName = new CaseInsensitiveString();
+ arguments = new ExprList(objectPool);
+ function = null;
+ }
+
+ public static FunctionCall createFunctionCall(String functionName, ExprList args, ClassAdObjectPool objectPool) {
+ FunctionCall fc = objectPool != null ? objectPool.funcPool.get() : new FunctionCall(null);
+ fc.function = funcTable.get(functionName.toLowerCase());
+ fc.functionName.set(functionName);
+ fc.arguments.setExprList(args.getExprList());
+ return fc;
+ }
+
+ // start up with an argument list of size 4
+
+ public FunctionCall(FunctionCall functioncall, ClassAdObjectPool objectPool) throws HyracksDataException {
+ super(objectPool);
+ functionName = new CaseInsensitiveString();
+ arguments = new ExprList(objectPool);
+ function = null;
+ copyFrom(functioncall);
+ }
+
/**
* Returns true if the function expression points to a valid
* function in the ClassAd library.
@@ -154,37 +179,13 @@
public void copyFrom(FunctionCall copiedFrom) throws HyracksDataException {
this.function = copiedFrom.function;
- this.functionName = copiedFrom.functionName;
- if (this.arguments == null) {
- this.arguments = (ExprList) copiedFrom.arguments.copy();
- } else {
- this.arguments.copyFrom(copiedFrom.arguments);
- }
- }
-
- public FunctionCall() {
- functionName = null;
- function = null;
- arguments = null;
- }
-
- public static FunctionCall createFunctionCall(String functionName, ExprList args) {
- FunctionCall fc = new FunctionCall();
- fc.function = funcTable.get(functionName.toLowerCase());
- fc.functionName = functionName;
- fc.arguments = args;
- return fc;
- }
-
- // start up with an argument list of size 4
-
- public FunctionCall(FunctionCall functioncall) throws HyracksDataException {
- copyFrom(functioncall);
+ this.functionName.set(copiedFrom.functionName.get());
+ this.arguments.setExprList(copiedFrom.arguments.getExprList());
}
@Override
public ExprTree copy() throws HyracksDataException {
- FunctionCall newTree = new FunctionCall();
+ FunctionCall newTree = objectPool.funcPool.get();
newTree.copyFrom(this);
return newTree;
}
@@ -192,7 +193,7 @@
@Override
public void copyFrom(ExprTree tree) throws HyracksDataException {
FunctionCall functioncall = (FunctionCall) tree;
- functionName = functioncall.functionName;
+ functionName.set(functioncall.functionName.get());
function = functioncall.function;
arguments.copyFrom(arguments);
super.copyFrom(functioncall);
@@ -209,13 +210,17 @@
} else if (pSelfTree.getKind() != NodeKind.FN_CALL_NODE) {
is_same = false;
} else {
- other_fn = (FunctionCall) pSelfTree;
- if (functionName == other_fn.functionName && function.equals(other_fn.function)
- && arguments.equals(other_fn.arguments)) {
- is_same = true;
+ try {
+ other_fn = (FunctionCall) pSelfTree;
+ if (functionName == other_fn.functionName && function.equals(other_fn.function)
+ && arguments.equals(other_fn.arguments)) {
+ is_same = true;
- } else {
- is_same = false;
+ } else {
+ is_same = false;
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
}
}
return is_same;
@@ -242,14 +247,14 @@
//This will move pointers to objects (not create clones)
public void getComponents(AMutableString fn, ExprList exprList) {
- fn.setValue(functionName);
+ fn.setValue(functionName.get());
for (ExprTree tree : arguments.getExprList()) {
exprList.add(tree);
}
}
public void getComponents(AMutableCharArrayString fn, ExprList exprList) {
- fn.setValue(functionName);
+ fn.setValue(functionName.get());
for (ExprTree tree : arguments.getExprList()) {
exprList.add(tree);
}
@@ -258,7 +263,7 @@
@Override
public boolean privateEvaluate(EvalState state, Value value) throws HyracksDataException {
if (function != null) {
- return function.call(functionName, arguments, state, value);
+ return function.call(functionName.get(), arguments, state, value, objectPool);
} else {
value.setErrorValue();
return (true);
@@ -267,19 +272,20 @@
@Override
public boolean privateEvaluate(EvalState state, Value value, ExprTreeHolder tree) throws HyracksDataException {
- FunctionCall tmpSig = new FunctionCall();
- Value tmpVal = new Value();
- ExprTreeHolder argSig = new ExprTreeHolder();
- MutableBoolean rval = new MutableBoolean();
+ FunctionCall tmpSig = objectPool.funcPool.get();
+ Value tmpVal = objectPool.valuePool.get();
+ ExprTreeHolder argSig = objectPool.mutableExprPool.get();
+ MutableBoolean rval = objectPool.boolPool.get();
if (!privateEvaluate(state, value)) {
return false;
}
- tmpSig.functionName = functionName;
+ tmpSig.functionName.set(functionName.get());
rval.setValue(true);
for (ExprTree i : arguments.getExprList()) {
rval.setValue(i.publicEvaluate(state, tmpVal, argSig));
- if (rval.booleanValue())
+ if (rval.booleanValue()) {
tmpSig.arguments.add(argSig.getInnerTree());
+ }
}
tree.setInnerTree(tmpSig);
return rval.booleanValue();
@@ -288,11 +294,10 @@
@Override
public boolean privateFlatten(EvalState state, Value value, ExprTreeHolder tree, AMutableInt32 i)
throws HyracksDataException {
- FunctionCall newCall = new FunctionCall();
- ExprTreeHolder argTree = new ExprTreeHolder();
- Value argValue = new Value();
+ FunctionCall newCall = objectPool.funcPool.get();
+ ExprTreeHolder argTree = objectPool.mutableExprPool.get();
+ Value argValue = objectPool.valuePool.get();
boolean fold = true;
-
tree.setInnerTree(null); // Just to be safe... wenger 2003-12-11.
// if the function cannot be resolved, the value is "error"
@@ -301,7 +306,7 @@
return true;
}
- newCall.functionName = functionName;
+ newCall.functionName.set(functionName.get());
newCall.function = function;
// flatten the arguments
@@ -313,7 +318,7 @@
continue;
} else {
// Assert: argTree == NULL
- argTree.setInnerTree(Literal.createLiteral(argValue));
+ argTree.setInnerTree(Literal.createLiteral(argValue, objectPool));
if (argTree.getInnerTree() != null) {
newCall.arguments.add(argTree.getInnerTree());
continue;
@@ -330,7 +335,7 @@
// assume all functions are "pure" (i.e., side-affect free)
if (fold) {
// flattened to a value
- if (!function.call(functionName, arguments, state, value)) {
+ if (!function.call(functionName.get(), arguments, state, value, objectPool)) {
return false;
}
tree.setInnerTree(null);
@@ -349,6 +354,6 @@
public void reset() {
this.arguments.clear();
this.function = null;
- this.functionName = "";
+ this.functionName.set("");;
}
}
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Lexer.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Lexer.java
index 97b7ea8..2ef9620 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Lexer.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Lexer.java
@@ -433,8 +433,9 @@
if (och == '.' || ch == '.') {
// fraction part of real or selection operator
- if (ch == '.')
+ if (ch == '.') {
wind();
+ }
if (Character.isDigit(ch)) {
// real; get digits after decimal point
numberType = NumberType.REAL;
@@ -459,8 +460,9 @@
// i.e., [eE][+-]?[0-9]+
if (numberType == NumberType.REAL && Character.toLowerCase(ch) == 'e') {
wind();
- if (ch == '+' || ch == '-')
+ if (ch == '+' || ch == '-') {
wind();
+ }
if (!Character.isDigit(ch)) {
cut();
tokenType = TokenType.LEX_TOKEN_ERROR;
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Literal.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Literal.java
index 5254a37..f04eb1d 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Literal.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Literal.java
@@ -23,6 +23,7 @@
import org.apache.asterix.external.classad.Value.NumberFactor;
import org.apache.asterix.external.classad.Value.ValueType;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.asterix.om.base.AMutableDouble;
import org.apache.asterix.om.base.AMutableInt32;
import org.apache.asterix.om.base.AMutableInt64;
@@ -34,12 +35,13 @@
* reals, booleans, strings, undefined and real.
*/
// literal specific information
- private Value value;
+ private final Value value;
private NumberFactor factor;
- public Literal() {
+ public Literal(ClassAdObjectPool objectPool) {
+ super(objectPool);
factor = Value.NumberFactor.NO_FACTOR;
- value = new Value();
+ value = new Value(objectPool);
}
@Override
@@ -69,47 +71,44 @@
}
}
- public Literal(Literal literal) throws HyracksDataException {
- copyFrom(literal);
- }
-
@Override
public ExprTree copy() throws HyracksDataException {
- Literal newTree = new Literal();
+ Literal newTree = objectPool.literalPool.get();
newTree.copyFrom(this);
return newTree;
}
public void copyFrom(Literal literal) throws HyracksDataException {
super.copyFrom(literal);
- value.copyFrom(literal.value);
+ value.setValue(literal.value);
factor = literal.factor;
return;
}
- public static Literal createReal(AMutableCharArrayString aString) throws HyracksDataException {
- Value val = new Value();
+ public static Literal createReal(AMutableCharArrayString aString, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value val = objectPool.valuePool.get();
double real;
real = Double.parseDouble(aString.toString());
val.setRealValue(real);
- return createLiteral(val);
+ return createLiteral(val, objectPool);
}
- public static Literal createReal(String aString) throws HyracksDataException {
- Value val = new Value();
+ public static Literal createReal(String aString, ClassAdObjectPool objectPool) throws HyracksDataException {
+ Value val = objectPool.valuePool.get();
double real;
real = Double.parseDouble(aString.toString());
val.setRealValue(real);
- return createLiteral(val);
+ return createLiteral(val, objectPool);
}
- public static Literal createAbsTime(ClassAdTime tim) throws HyracksDataException {
- Value val = new Value();
+ public static Literal createAbsTime(ClassAdTime tim, ClassAdObjectPool objectPool) throws HyracksDataException {
+ Value val = objectPool.valuePool.get();
if (tim == null) { // => current time/offset
- tim = new ClassAdTime();
+ tim = objectPool.classAdTimePool.get();
}
val.setAbsoluteTimeValue(tim);
- return (createLiteral(val));
+ return (createLiteral(val, objectPool));
}
/* Creates an absolute time literal, from the string timestr,
@@ -118,17 +117,21 @@
D => non-digit, d=> digit
Ex - 2003-01-25T09:00:00-06:00
*/
- public static Literal createAbsTime(AMutableCharArrayString timeStr) throws HyracksDataException {
- Value val = new Value();
+ public static Literal createAbsTime(AMutableCharArrayString timeStr, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value val = objectPool.valuePool.get();
boolean offset = false; // to check if the argument conatins a timezone offset parameter
- AMutableInt32 tzhr = new AMutableInt32(0); // corresponds to 1st "dd" in -|+dd:dd
- AMutableInt32 tzmin = new AMutableInt32(0); // corresponds to 2nd "dd" in -|+dd:dd
+ AMutableInt32 tzhr = objectPool.int32Pool.get(); // corresponds to 1st "dd" in -|+dd:dd
+ AMutableInt32 tzmin = objectPool.int32Pool.get(); // corresponds to 2nd "dd" in -|+dd:dd
int len = timeStr.getLength();
- AMutableInt32 index = new AMutableInt32(len - 1);
+ AMutableInt32 index = objectPool.int32Pool.get();
+ index.setValue(len - 1);
prevNonSpaceChar(timeStr, index);
- AMutableInt32 i = new AMutableInt32(index.getIntegerValue());
+ AMutableInt32 i = objectPool.int32Pool.get();
+ i.setValue(index.getIntegerValue());
+
if ((timeStr.charAt(i.getIntegerValue()) == 'z') || (timeStr.charAt(i.getIntegerValue()) == 'Z')) { // z|Z corresponds to a timezone offset of 0
offset = true;
timeStr.erase(i.getIntegerValue()); // remove the offset section from the string
@@ -145,7 +148,7 @@
nextDigitChar(timeStr, i);
if (i.getIntegerValue() > len - 4) { // string has to contain dddd (year)
val.setErrorValue();
- return (createLiteral(val));
+ return (createLiteral(val, objectPool));
}
int tm_year, tm_mon = 0, tm_mday = 0, tm_hour = 0, tm_min = 0, tm_sec = 0;
tm_year = Integer.parseInt((timeStr.substr(i.getIntegerValue(), 4)));// - 1900;
@@ -183,12 +186,12 @@
if ((i.getIntegerValue() <= len - 1) && (Character.isDigit(timeStr.charAt(i.getIntegerValue())))) { // there should be no more digit characters once the required
val.setErrorValue(); // parameteres are parsed
- return (createLiteral(val));
+ return (createLiteral(val, objectPool));
}
Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("GMT"));
cal.clear();
cal.set(tm_year, tm_mon, tm_mday, tm_hour, tm_min, tm_sec);
- ClassAdTime time = new ClassAdTime();
+ ClassAdTime time = objectPool.classAdTimePool.get();
time.setValue(cal.getTimeInMillis());
if (offset) {
time.setTimeZone((tzhr.getIntegerValue() * 3600000) + (tzmin.getIntegerValue() * 60000));
@@ -197,28 +200,29 @@
time.setDefaultTimeZone();
}
val.setAbsoluteTimeValue(time);
- return (createLiteral(val));
+ return (createLiteral(val, objectPool));
}
public Literal createRelTime(ClassAdTime t1, ClassAdTime t2) throws HyracksDataException {
- Value val = new Value();
+ Value val = objectPool.valuePool.get();
val.setRelativeTimeValue(t1.subtract(t2));
- return (createLiteral(val));
+ return (createLiteral(val, objectPool));
}
Literal createRelTime(ClassAdTime secs) throws HyracksDataException {
- Value val = new Value();
+ Value val = objectPool.valuePool.get();
val.setRelativeTimeValue(secs);
- return (createLiteral(val));
+ return (createLiteral(val, objectPool));
}
/* Creates a relative time literal, from the string timestr,
*parsing it as [[[days+]hh:]mm:]ss
* Ex - 1+00:02:00
*/
- public static Literal createRelTime(AMutableCharArrayString timeStr) throws HyracksDataException {
- Value val = new Value();
- ClassAdTime rsecs = new ClassAdTime();
+ public static Literal createRelTime(AMutableCharArrayString timeStr, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value val = objectPool.valuePool.get();
+ ClassAdTime rsecs = objectPool.classAdTimePool.get();
int len = timeStr.getLength();
double secs = 0;
@@ -226,8 +230,8 @@
int hrs = 0;
int days = 0;
boolean negative = false;
-
- AMutableInt32 i = new AMutableInt32(len - 1);
+ AMutableInt32 i = objectPool.int32Pool.get();
+ i.setValue(len - 1);
prevNonSpaceChar(timeStr, i);
// checking for 'sec' parameter & collecting it if present (ss.sss)
if ((i.getIntegerValue() >= 0)
@@ -237,7 +241,7 @@
i.setValue(i.getIntegerValue() - 1);
}
prevNonSpaceChar(timeStr, i);
- AMutableCharArrayString revSecStr = new AMutableCharArrayString();
+ AMutableCharArrayString revSecStr = objectPool.strPool.get();
while ((i.getIntegerValue() >= 0) && (Character.isDigit(timeStr.charAt(i.getIntegerValue())))) {
revSecStr.appendChar(timeStr.charAt(i.getIntegerValue()));
i.setValue(i.getIntegerValue() - 1);
@@ -250,7 +254,7 @@
i.setValue(i.getIntegerValue() - 1);
}
}
- secs = revDouble(revSecStr);
+ secs = revDouble(revSecStr, objectPool);
}
prevNonSpaceChar(timeStr, i);
@@ -258,13 +262,13 @@
if ((i.getIntegerValue() >= 0) && ((timeStr.charAt(i.getIntegerValue()) == 'm')
|| (timeStr.charAt(i.getIntegerValue()) == 'M') || (timeStr.charAt(i.getIntegerValue()) == ':'))) {
i.setValue(i.getIntegerValue() - 1);
- AMutableCharArrayString revMinStr = new AMutableCharArrayString();
+ AMutableCharArrayString revMinStr = objectPool.strPool.get();
prevNonSpaceChar(timeStr, i);
while ((i.getIntegerValue() >= 0) && (Character.isDigit(timeStr.charAt(i.getIntegerValue())))) {
revMinStr.appendChar(timeStr.charAt(i.getIntegerValue()));
i.setValue(i.getIntegerValue() - 1);
}
- mins = revInt(revMinStr);
+ mins = revInt(revMinStr, objectPool);
}
prevNonSpaceChar(timeStr, i);
@@ -272,13 +276,13 @@
if ((i.getIntegerValue() >= 0) && ((timeStr.charAt(i.getIntegerValue()) == 'h')
|| (timeStr.charAt(i.getIntegerValue()) == 'H') || (timeStr.charAt(i.getIntegerValue()) == ':'))) {
i.setValue(i.getIntegerValue() - 1);
- AMutableCharArrayString revHrStr = new AMutableCharArrayString();
+ AMutableCharArrayString revHrStr = objectPool.strPool.get();
prevNonSpaceChar(timeStr, i);
while ((i.getIntegerValue() >= 0) && (Character.isDigit(timeStr.charAt(i.getIntegerValue())))) {
revHrStr.appendChar(timeStr.charAt(i.getIntegerValue()));
i.setValue(i.getIntegerValue() - 1);
}
- hrs = revInt(revHrStr);
+ hrs = revInt(revHrStr, objectPool);
}
prevNonSpaceChar(timeStr, i);
@@ -286,13 +290,13 @@
if ((i.getIntegerValue() >= 0) && ((timeStr.charAt(i.getIntegerValue()) == 'd')
|| (timeStr.charAt(i.getIntegerValue()) == 'D') || (timeStr.charAt(i.getIntegerValue()) == '+'))) {
i.setValue(i.getIntegerValue() - 1);
- AMutableCharArrayString revDayStr = new AMutableCharArrayString();
+ AMutableCharArrayString revDayStr = objectPool.strPool.get();
prevNonSpaceChar(timeStr, i);
while ((i.getIntegerValue() >= 0) && (Character.isDigit(timeStr.charAt(i.getIntegerValue())))) {
revDayStr.appendChar(timeStr.charAt(i.getIntegerValue()));
i.setValue(i.getIntegerValue() - 1);
}
- days = revInt(revDayStr);
+ days = revInt(revDayStr, objectPool);
}
prevNonSpaceChar(timeStr, i);
@@ -306,14 +310,14 @@
if ((i.getIntegerValue() >= 0) && (!(Character.isWhitespace(timeStr.charAt(i.getIntegerValue()))))) { // should not conatin any non-space char beyond -,d,h,m,s
val.setErrorValue();
- return (createLiteral(val));
+ return (createLiteral(val, objectPool));
}
rsecs.setRelativeTime(
(long) ((negative ? -1 : +1) * (days * 86400000 + hrs * 3600000 + mins * 60000 + secs * 1000.0)));
val.setRelativeTimeValue(rsecs);
- return (createLiteral(val));
+ return (createLiteral(val, objectPool));
}
/* Function which iterates through the string Str from the location 'index',
@@ -343,8 +347,8 @@
* order of the digits & returns the corresponding number as an
* integer.
*/
- public static int revInt(AMutableCharArrayString revNumStr) {
- AMutableCharArrayString numStr = new AMutableCharArrayString(revNumStr.getLength());
+ public static int revInt(AMutableCharArrayString revNumStr, ClassAdObjectPool objectPool) {
+ AMutableCharArrayString numStr = objectPool.strPool.get();
for (int i = revNumStr.getLength() - 1; i >= 0; i--) {
numStr.appendChar(revNumStr.charAt(i));
}
@@ -354,8 +358,8 @@
/* Function which takes a number in string format, and reverses the
* order of the digits & returns the corresponding number as a double.
*/
- public static double revDouble(AMutableCharArrayString revNumStr) {
- AMutableCharArrayString numStr = new AMutableCharArrayString(revNumStr.getLength());
+ public static double revDouble(AMutableCharArrayString revNumStr, ClassAdObjectPool objectPool) {
+ AMutableCharArrayString numStr = objectPool.strPool.get();
for (int i = revNumStr.getLength() - 1; i >= 0; i--) {
numStr.appendChar(revNumStr.charAt(i));
}
@@ -369,15 +373,17 @@
return Util.timezoneOffset(epochsecs);
}
- public static Literal createLiteral(Value val, NumberFactor f) throws HyracksDataException {
+ public static Literal createLiteral(Value val, NumberFactor f, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
if (val.getType() == ValueType.CLASSAD_VALUE || val.getType() == ValueType.LIST_VALUE
|| val.getType() == ValueType.SLIST_VALUE) {
throw new HyracksDataException("list and classad values are not literals");
}
- Literal lit = new Literal();
- lit.value.copyFrom(val);
- if (!val.isIntegerValue() && !val.isRealValue())
+ Literal lit = objectPool.literalPool.get();
+ lit.value.setValue(val);
+ if (!val.isIntegerValue() && !val.isRealValue()) {
f = NumberFactor.NO_FACTOR;
+ }
lit.factor = f;
return lit;
}
@@ -387,20 +393,21 @@
|| val.getType() == ValueType.SLIST_VALUE) {
throw new HyracksDataException("list and classad values are not literals");
}
- lit.value.copyFrom(val);
- if (!val.isIntegerValue() && !val.isRealValue())
+ lit.value.setValue(val);
+ if (!val.isIntegerValue() && !val.isRealValue()) {
f = NumberFactor.NO_FACTOR;
+ }
lit.factor = f;
}
- public static Literal createLiteral(Value val) throws HyracksDataException {
- return createLiteral(val, NumberFactor.NO_FACTOR);
+ public static Literal createLiteral(Value val, ClassAdObjectPool objectPool) throws HyracksDataException {
+ return createLiteral(val, NumberFactor.NO_FACTOR, objectPool);
}
public void GetValue(Value val) throws HyracksDataException {
- AMutableInt64 i = new AMutableInt64(0);
- AMutableDouble r = new AMutableDouble(0);
- val.copyFrom(value);
+ AMutableInt64 i = objectPool.int64Pool.get();
+ AMutableDouble r = objectPool.doublePool.get();
+ val.setValue(value);
// if integer or real, multiply by the factor
if (val.isIntegerValue(i)) {
@@ -415,7 +422,7 @@
}
public void getComponents(Value val, AMutableNumberFactor factor) throws HyracksDataException {
- val.copyFrom(value);
+ val.setValue(value);
factor.setFactor(this.factor);
}
@@ -446,10 +453,10 @@
@Override
public boolean privateEvaluate(EvalState eval, Value val) throws HyracksDataException {
- AMutableInt64 i = new AMutableInt64(0);
- AMutableDouble r = new AMutableDouble(0);
+ AMutableInt64 i = objectPool.int64Pool.get();
+ AMutableDouble r = objectPool.doublePool.get();
- val.copyFrom(value);
+ val.setValue(value);
// if integer or real, multiply by the factor
if (val.isIntegerValue(i)) {
@@ -511,7 +518,7 @@
@Override
public void reset() {
- value.clear();
+ value.reset();
factor = NumberFactor.NO_FACTOR;
}
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Operation.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Operation.java
index ae3c5ef..b2ed5b1 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Operation.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Operation.java
@@ -19,6 +19,7 @@
package org.apache.asterix.external.classad;
import org.apache.asterix.external.classad.Value.ValueType;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.asterix.om.base.AMutableDouble;
import org.apache.asterix.om.base.AMutableInt32;
import org.apache.asterix.om.base.AMutableInt64;
@@ -91,9 +92,9 @@
public static final int OpKind_LAST_OP = OpKind_MISC_END;
private int opKind;
- private ExprTree child1;
- private ExprTree child2;
- private ExprTree child3;
+ private final ExprTreeHolder child1;
+ private final ExprTreeHolder child2;
+ private final ExprTreeHolder child3;
/// node type
@Override
@@ -117,14 +118,17 @@
* @param e3
* The third sub-expression child of the node (if any).
* @return The constructed operation
+ * @throws HyracksDataException
*/
- public static Operation createOperation(int opkind, ExprTree e1, ExprTree e2) {
- return createOperation(opkind, e1, e2, null);
+ public static Operation createOperation(int opkind, ExprTree e1, ExprTree e2, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ return createOperation(opkind, e1, e2, null, objectPool);
}
- public static Operation createOperation(int opkind, ExprTree e1) {
- return createOperation(opkind, e1, null, null);
+ public static Operation createOperation(int opkind, ExprTree e1, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ return createOperation(opkind, e1, null, null, objectPool);
}
// public access to operation function
@@ -166,57 +170,36 @@
* @return true if the operator is strict, false otherwise.
*/
- public Operation() {
+ public Operation(ClassAdObjectPool objectPool) {
+ super(objectPool);
opKind = OpKind_NO_OP;
- child1 = null;
- child2 = null;
- child3 = null;
+ child1 = new ExprTreeHolder(objectPool);
+ child2 = new ExprTreeHolder(objectPool);
+ child3 = new ExprTreeHolder(objectPool);
}
- public Operation(Operation op) throws HyracksDataException {
+ public Operation(Operation op, ClassAdObjectPool objectPool) throws HyracksDataException {
+ super(objectPool);
+ child1 = new ExprTreeHolder(objectPool);
+ child2 = new ExprTreeHolder(objectPool);
+ child3 = new ExprTreeHolder(objectPool);
copyFrom(op);
- return;
}
@Override
public ExprTree copy() throws HyracksDataException {
- Operation newTree = new Operation();
+ Operation newTree = objectPool.operationPool.get();
newTree.copyFrom(this);
return newTree;
}
public boolean copyFrom(Operation op) throws HyracksDataException {
- boolean success = true;
- if (op.child1 == null) {
- child1 = null;
- } else {
- if (child1 == null) {
- child1 = new ExprTreeHolder();
- }
- child1.copyFrom(op.child1);
- child1 = child1.self();
- }
- if (op.child2 == null) {
- child2 = null;
- } else {
- if (child2 == null) {
- child2 = new ExprTreeHolder();
- }
- child2.copyFrom(op.child2);
- child2 = child2.self();
- }
- if (op.child3 == null) {
- child3 = null;
- } else {
- if (child3 == null) {
- child3 = new ExprTreeHolder();
- }
- child3.copyFrom(op.child3);
- child3 = child3.self();
- }
+ child1.copyFrom(op.child1);
+ child2.copyFrom(op.child2);
+ child3.copyFrom(op.child3);
this.opKind = op.opKind;
super.copyFrom(op);
- return success;
+ return true;
}
@Override
@@ -257,33 +240,28 @@
@Override
public void privateSetParentScope(ClassAd parent) {
- if (child1 != null) {
- child1.setParentScope(parent);
- }
- if (child2 != null) {
- child2.setParentScope(parent);
- }
- if (child3 != null) {
- child3.setParentScope(parent);
- }
+ child1.setParentScope(parent);
+ child2.setParentScope(parent);
+ child3.setParentScope(parent);
}
- public static void operate(int opKind, Value op1, Value op2, Value result) throws HyracksDataException {
- Value dummy = new Value();
- privateDoOperation(opKind, op1, op2, dummy, true, true, false, result, null);
+ public static void operate(int opKind, Value op1, Value op2, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value dummy = objectPool.valuePool.get();
+ privateDoOperation(opKind, op1, op2, dummy, true, true, false, result, null, objectPool);
}
public void operate(int op, Value op1, Value op2, Value op3, Value result) throws HyracksDataException {
- privateDoOperation(op, op1, op2, op3, true, true, true, result, null);
+ privateDoOperation(op, op1, op2, op3, true, true, true, result, null, objectPool);
}
public static int privateDoOperation(int op, Value val1, Value val2, Value val3, boolean valid1, boolean valid2,
- boolean valid3, Value result) throws HyracksDataException {
- return privateDoOperation(op, val1, val2, val3, valid1, valid2, valid3, result, null);
+ boolean valid3, Value result, ClassAdObjectPool objectPool) throws HyracksDataException {
+ return privateDoOperation(op, val1, val2, val3, valid1, valid2, valid3, result, null, objectPool);
}
public static int privateDoOperation(int op, Value val1, Value val2, Value val3, boolean valid1, boolean valid2,
- boolean valid3, Value result, EvalState es) throws HyracksDataException {
+ boolean valid3, Value result, EvalState es, ClassAdObjectPool objectPool) throws HyracksDataException {
ValueType vt1;
ValueType vt2;
ValueType vt3;
@@ -295,7 +273,7 @@
// take care of the easy cases
if (op == OpKind_NO_OP || op == OpKind_PARENTHESES_OP) {
- result.copyFrom(val1);
+ result.setValue(val1);
return SigValues.SIG_CHLD1.ordinal();
} else if (op == OpKind_UNARY_PLUS_OP) {
if (vt1 == ValueType.BOOLEAN_VALUE || vt1 == ValueType.STRING_VALUE || val1.isListValue()
@@ -303,7 +281,7 @@
result.setErrorValue();
} else {
// applies for ERROR, UNDEFINED and Numbers
- result.copyFrom(val1);
+ result.setValue(val1);
}
return SigValues.SIG_CHLD1.ordinal();
}
@@ -342,28 +320,28 @@
// comparison operations (binary, one unary)
if (op >= OpKind_COMPARISON_START && op <= OpKind_COMPARISON_END) {
- return (doComparison(op, val1, val2, result));
+ return (doComparison(op, val1, val2, result, objectPool));
}
// arithmetic operations (binary)
if (op >= OpKind_ARITHMETIC_START && op <= OpKind_ARITHMETIC_END) {
- return (doArithmetic(op, val1, val2, result));
+ return (doArithmetic(op, val1, val2, result, objectPool));
}
// logical operators (binary, one unary)
if (op >= OpKind_LOGIC_START && op <= OpKind_LOGIC_END) {
- return (doLogical(op, val1, val2, result));
+ return (doLogical(op, val1, val2, result, objectPool));
}
// bitwise operators (binary, one unary)
if (op >= OpKind_BITWISE_START && op <= OpKind_BITWISE_END) {
- return (doBitwise(op, val1, val2, result));
+ return (doBitwise(op, val1, val2, result, objectPool));
}
// misc.
if (op == OpKind_TERNARY_OP) {
// ternary (if-operator)
- MutableBoolean b = new MutableBoolean(false);
+ MutableBoolean b = objectPool.boolPool.get();
// if the selector is UNDEFINED, the result is undefined
if (vt1 == ValueType.UNDEFINED_VALUE) {
@@ -375,18 +353,18 @@
result.setErrorValue();
return SigValues.SIG_CHLD1.ordinal();
} else if (b.booleanValue()) {
- result.copyFrom(val2);
+ result.setValue(val2);
return (SigValues.SIG_CHLD2.ordinal());
} else {
- result.copyFrom(val3);
+ result.setValue(val3);
return (SigValues.SIG_CHLD3.ordinal());
}
} else if (op == OpKind_SUBSCRIPT_OP) {
// subscripting from a list (strict)
if (vt1 == ValueType.CLASSAD_VALUE && vt2 == ValueType.STRING_VALUE) {
- ClassAd classad = new ClassAd();
- AMutableCharArrayString index = new AMutableCharArrayString();
+ ClassAd classad = objectPool.classAdPool.get();
+ AMutableCharArrayString index = objectPool.strPool.get();
val1.isClassAdValue(classad);
val2.isStringValue(index);
@@ -402,8 +380,8 @@
return (SigValues.SIG_CHLD1.ordinal() | SigValues.SIG_CHLD2.ordinal());
} else if (val1.isListValue() && vt2 == ValueType.INTEGER_VALUE) {
- AMutableInt64 index = new AMutableInt64(0);
- ExprList elist = new ExprList();
+ AMutableInt64 index = objectPool.int64Pool.get();
+ ExprList elist = objectPool.exprListPool.get();
val1.isListValue(elist);
val2.isIntegerValue(index);
@@ -425,9 +403,9 @@
@Override
public boolean privateEvaluate(EvalState state, Value result) throws HyracksDataException {
- Value val1 = new Value();
- Value val2 = new Value();
- Value val3 = new Value();
+ Value val1 = objectPool.valuePool.get();
+ Value val2 = objectPool.valuePool.get();
+ Value val3 = objectPool.valuePool.get();
boolean valid1, valid2, valid3;
int rval = 0;
@@ -435,10 +413,10 @@
valid2 = false;
valid3 = false;
- AMutableInt32 operationKind = new AMutableInt32(OpKind_NO_OP);
- ExprTreeHolder child1 = new ExprTreeHolder();
- ExprTreeHolder child2 = new ExprTreeHolder();
- ExprTreeHolder child3 = new ExprTreeHolder();
+ AMutableInt32 operationKind = objectPool.int32Pool.get();
+ ExprTreeHolder child1 = objectPool.mutableExprPool.get();
+ ExprTreeHolder child2 = objectPool.mutableExprPool.get();
+ ExprTreeHolder child3 = objectPool.mutableExprPool.get();
getComponents(operationKind, child1, child2, child3);
// Evaluate all valid children
@@ -469,13 +447,13 @@
valid3 = true;
}
- rval = privateDoOperation(opKind, val1, val2, val3, valid1, valid2, valid3, result, state);
+ rval = privateDoOperation(opKind, val1, val2, val3, valid1, valid2, valid3, result, state, objectPool);
return (rval != SigValues.SIG_NONE.ordinal());
}
public boolean shortCircuit(EvalState state, Value arg1, Value result) throws HyracksDataException {
- MutableBoolean arg1_bool = new MutableBoolean();
+ MutableBoolean arg1_bool = objectPool.boolPool.get();
switch (opKind) {
case OpKind_LOGICAL_OR_OP:
if (arg1.isBooleanValueEquiv(arg1_bool) && arg1_bool.booleanValue()) {
@@ -512,21 +490,21 @@
@Override
public boolean privateEvaluate(EvalState state, Value result, ExprTreeHolder tree) throws HyracksDataException {
int sig;
- Value val1 = new Value();
- Value val2 = new Value();
- Value val3 = new Value();
- ExprTreeHolder t1 = new ExprTreeHolder();
- ExprTreeHolder t2 = new ExprTreeHolder();
- ExprTreeHolder t3 = new ExprTreeHolder();
+ Value val1 = objectPool.valuePool.get();
+ Value val2 = objectPool.valuePool.get();
+ Value val3 = objectPool.valuePool.get();
+ ExprTreeHolder t1 = objectPool.mutableExprPool.get();
+ ExprTreeHolder t2 = objectPool.mutableExprPool.get();
+ ExprTreeHolder t3 = objectPool.mutableExprPool.get();
boolean valid1 = false, valid2 = false, valid3 = false;
- AMutableInt32 opKind = new AMutableInt32(OpKind_NO_OP);
- ExprTreeHolder child1 = new ExprTreeHolder();
- ExprTreeHolder child2 = new ExprTreeHolder();
- ExprTreeHolder child3 = new ExprTreeHolder();
+ AMutableInt32 opKind = objectPool.int32Pool.get();
+ ExprTreeHolder child1 = objectPool.mutableExprPool.get();
+ ExprTreeHolder child2 = objectPool.mutableExprPool.get();
+ ExprTreeHolder child3 = objectPool.mutableExprPool.get();
getComponents(opKind, child1, child2, child3);
// Evaluate all valid children
- tree = new ExprTreeHolder();
+ tree = objectPool.mutableExprPool.get();
if (child1.getInnerTree() != null) {
if (!child1.publicEvaluate(state, val1, t1)) {
result.setErrorValue();
@@ -552,7 +530,7 @@
// do evaluation
sig = privateDoOperation(opKind.getIntegerValue().intValue(), val1, val2, val3, valid1, valid2, valid3, result,
- state);
+ state, objectPool);
// delete trees which were not significant
if (valid1 && 0 != (sig & SigValues.SIG_CHLD1.ordinal())) {
@@ -587,7 +565,7 @@
} else {
// the node operated on the value; the operator is also
// significant
- tree.setInnerTree(createOperation(opKind.getIntegerValue().intValue(), t1));
+ tree.setInnerTree(createOperation(opKind.getIntegerValue().intValue(), t1, objectPool));
}
return (true);
} else {
@@ -604,7 +582,7 @@
throw new HyracksDataException("Should not reach here");
} else {
// the node is also significant
- tree.setInnerTree(createOperation(opKind.getIntegerValue().intValue(), t1, t2));
+ tree.setInnerTree(createOperation(opKind.getIntegerValue().intValue(), t1, t2, objectPool));
return (true);
}
}
@@ -613,7 +591,7 @@
if (opKind.getIntegerValue().intValue() == OpKind_IS_OP
|| opKind.getIntegerValue().intValue() == OpKind_ISNT_OP) {
// the operation is *always* significant for IS and ISNT
- tree.setInnerTree(createOperation(opKind.getIntegerValue().intValue(), t1, t2));
+ tree.setInnerTree(createOperation(opKind.getIntegerValue().intValue(), t1, t2, objectPool));
return (true);
}
// other non-strict binary operators
@@ -621,7 +599,7 @@
|| opKind.getIntegerValue().intValue() == OpKind_LOGICAL_OR_OP) {
if ((SigValues.values()[sig].ordinal() & SigValues.SIG_CHLD1.ordinal()) != 0
&& (SigValues.values()[sig].ordinal() & SigValues.SIG_CHLD2.ordinal()) != 0) {
- tree.setInnerTree(createOperation(opKind.getIntegerValue().intValue(), t1, t2));
+ tree.setInnerTree(createOperation(opKind.getIntegerValue().intValue(), t1, t2, objectPool));
return (true);
} else if ((SigValues.values()[sig].ordinal() & SigValues.SIG_CHLD1.ordinal()) != 0) {
tree.setInnerTree(t1);
@@ -636,9 +614,9 @@
// non-strict ternary operator (conditional operator) s ? t : f
// selector is always significant (???)
if (opKind.getIntegerValue().intValue() == OpKind_TERNARY_OP) {
- Value tmpVal = new Value();
+ Value tmpVal = objectPool.valuePool.get();
tmpVal.setUndefinedValue();
- tree.setInnerTree(Literal.createLiteral(tmpVal));
+ tree.setInnerTree(Literal.createLiteral(tmpVal, objectPool));
// "true" consequent taken
if ((SigValues.values()[sig].ordinal() & SigValues.SIG_CHLD2.ordinal()) != 0) {
@@ -659,14 +637,17 @@
@Override
public boolean privateFlatten(EvalState state, Value val, ExprTreeHolder tree, AMutableInt32 opPtr)
throws HyracksDataException {
- AMutableInt32 childOp1 = new AMutableInt32(OpKind_NO_OP);
- AMutableInt32 childOp2 = new AMutableInt32(OpKind_NO_OP);
- ExprTreeHolder fChild1 = new ExprTreeHolder();
- ExprTreeHolder fChild2 = new ExprTreeHolder();;
- Value val1 = new Value();
- Value val2 = new Value();
- Value val3 = new Value();
- AMutableInt32 newOp = new AMutableInt32(opKind);
+ AMutableInt32 childOp1 = objectPool.int32Pool.get();
+ childOp1.setValue(OpKind_NO_OP);
+ AMutableInt32 childOp2 = objectPool.int32Pool.get();
+ childOp2.setValue(OpKind_NO_OP);
+ ExprTreeHolder fChild1 = objectPool.mutableExprPool.get();
+ ExprTreeHolder fChild2 = objectPool.mutableExprPool.get();;
+ Value val1 = objectPool.valuePool.get();
+ Value val2 = objectPool.valuePool.get();
+ Value val3 = objectPool.valuePool.get();
+ AMutableInt32 newOp = objectPool.int32Pool.get();
+ newOp.setValue(opKind);
int op = opKind;
tree.setInnerTree(null);; // Just to be safe... wenger 2003-12-11.
@@ -675,21 +656,22 @@
if ((op >= OpKind_COMPARISON_START && op <= OpKind_COMPARISON_END) || op == OpKind_SUBTRACTION_OP
|| op == OpKind_DIVISION_OP || op == OpKind_MODULUS_OP || op == OpKind_LEFT_SHIFT_OP
|| op == OpKind_RIGHT_SHIFT_OP || op == OpKind_URIGHT_SHIFT_OP) {
- if (opPtr != null)
+ if (opPtr != null) {
opPtr.setValue(OpKind_NO_OP);
+ }
if (child1.publicFlatten(state, val1, fChild1) && child2.publicFlatten(state, val2, fChild2)) {
if (fChild1.getInnerTree() == null && fChild2.getInnerTree() == null) {
- privateDoOperation(op, val1, val2, val3, true, true, false, val);
+ privateDoOperation(op, val1, val2, val3, true, true, false, val, objectPool);
tree.setInnerTree(null);
return true;
} else if (fChild1.getInnerTree() != null && fChild2.getInnerTree() != null) {
- tree.setInnerTree(Operation.createOperation(op, fChild1, fChild2));
+ tree.setInnerTree(Operation.createOperation(op, fChild1, fChild2, objectPool));
return true;
} else if (fChild1.getInnerTree() != null) {
- tree.setInnerTree(Operation.createOperation(op, fChild1, val2));
+ tree.setInnerTree(Operation.createOperation(op, fChild1, val2, objectPool));
return true;
} else if (fChild2.getInnerTree() != null) {
- tree.setInnerTree(Operation.createOperation(op, val1, fChild2));
+ tree.setInnerTree(Operation.createOperation(op, val1, fChild2, objectPool));
return true;
}
} else {
@@ -706,8 +688,8 @@
// any op that got past the above is binary, commutative and associative
// Flatten sub expressions
- if ((child1 != null && !child1.publicFlatten(state, val1, fChild1, childOp1))
- || (child2 != null && !child2.publicFlatten(state, val2, fChild2, childOp2))) {
+ if ((child1.getInnerTree() != null && !child1.publicFlatten(state, val1, fChild1, childOp1))
+ || (child2.getInnerTree() != null && !child2.publicFlatten(state, val2, fChild2, childOp2))) {
tree.setInnerTree(null);
return false;
}
@@ -724,7 +706,7 @@
// if splitting is disallowed, fold the value and tree into a tree
if (opPtr == null && newOp.getIntegerValue().intValue() != OpKind_NO_OP) {
- tree.setInnerTree(Operation.createOperation(newOp.getIntegerValue().intValue(), val, tree));
+ tree.setInnerTree(Operation.createOperation(newOp.getIntegerValue().intValue(), val, tree, objectPool));
if (tree.getInnerTree() == null) {
return false;
}
@@ -737,15 +719,15 @@
public boolean combine(AMutableInt32 op, Value val, ExprTreeHolder tree, AMutableInt32 op1, Value val1,
ExprTreeHolder tree1, AMutableInt32 op2, Value val2, ExprTreeHolder tree2) throws HyracksDataException {
- Operation newOp = new Operation();
- Value dummy = new Value(); // undefined
+ Operation newOp = objectPool.operationPool.get();
+ Value dummy = objectPool.valuePool.get(); // undefined
// special don't care cases for logical operators with exactly one value
if ((tree1.getInnerTree() == null || tree2.getInnerTree() == null)
&& (tree1.getInnerTree() != null || tree2.getInnerTree() != null)
&& (op.getIntegerValue() == OpKind_LOGICAL_OR_OP || op.getIntegerValue() == OpKind_LOGICAL_AND_OP)) {
privateDoOperation(op.getIntegerValue().intValue(), tree1.getInnerTree() == null ? val1 : dummy,
- tree2.getInnerTree() == null ? val2 : dummy, dummy, true, true, false, val);
+ tree2.getInnerTree() == null ? val2 : dummy, dummy, true, true, false, val, objectPool);
if (val.isBooleanValue()) {
tree.setInnerTree(null);
op.setValue(OpKind_NO_OP);
@@ -755,7 +737,7 @@
if (tree1.getInnerTree() == null && tree2.getInnerTree() == null) {
// left and rightsons are only values
- privateDoOperation(op.getIntegerValue().intValue(), val1, val2, dummy, true, true, false, val);
+ privateDoOperation(op.getIntegerValue().intValue(), val1, val2, dummy, true, true, false, val, objectPool);
tree.setInnerTree(null);
op.setValue(OpKind_NO_OP);
return true;
@@ -763,18 +745,18 @@
&& (tree2.getInnerTree() != null && op2.getIntegerValue().intValue() == OpKind_NO_OP)) {
// leftson is a value, rightson is a tree
tree.setInnerTree(tree2.getInnerTree());
- val.copyFrom(val1);
+ val.setValue(val1);
return true;
} else if (tree2.getInnerTree() == null
&& (tree1.getInnerTree() != null && op1.getIntegerValue().intValue() == OpKind_NO_OP)) {
// rightson is a value, leftson is a tree
tree.setInnerTree(tree1.getInnerTree());
- val.copyFrom(val2);
+ val.setValue(val2);
return true;
} else if ((tree1.getInnerTree() != null && op1.getIntegerValue().intValue() == OpKind_NO_OP)
&& (tree2.getInnerTree() != null && op2.getIntegerValue().intValue() == OpKind_NO_OP)) {
// left and rightsons are trees only
- if (null != (newOp = createOperation(op.getIntegerValue().intValue(), tree1, tree2))) {
+ if (null != (newOp = createOperation(op.getIntegerValue().intValue(), tree1, tree2, objectPool))) {
return false;
}
tree.setInnerTree(newOp);
@@ -787,23 +769,25 @@
&& !op.equals(op1) && !op.equals(op1)) {
// at least one of them returned a value and a tree, and parent does
// not share the same operation with either child
- ExprTreeHolder newOp1 = new ExprTreeHolder();
- ExprTreeHolder newOp2 = new ExprTreeHolder();
+ ExprTreeHolder newOp1 = objectPool.mutableExprPool.get();
+ ExprTreeHolder newOp2 = objectPool.mutableExprPool.get();
if (op1.getIntegerValue().intValue() != OpKind_NO_OP) {
- newOp1.setInnerTree(Operation.createOperation(op1.getIntegerValue().intValue(), val1, tree1));
+ newOp1.setInnerTree(
+ Operation.createOperation(op1.getIntegerValue().intValue(), val1, tree1, objectPool));
} else if (tree1.getInnerTree() != null) {
newOp1.setInnerTree(tree1.getInnerTree());
} else {
- newOp1.setInnerTree(Literal.createLiteral(val1));
+ newOp1.setInnerTree(Literal.createLiteral(val1, objectPool));
}
if (op2.getIntegerValue().intValue() != OpKind_NO_OP) {
- newOp2.setInnerTree(Operation.createOperation(op2.getIntegerValue().intValue(), val2, tree2));
+ newOp2.setInnerTree(
+ Operation.createOperation(op2.getIntegerValue().intValue(), val2, tree2, objectPool));
} else if (tree2.getInnerTree() != null) {
newOp2.setInnerTree(tree2);
} else {
- newOp2.setInnerTree(Literal.createLiteral(val2));
+ newOp2.setInnerTree(Literal.createLiteral(val2, objectPool));
}
if (newOp1.getInnerTree() == null || newOp2.getInnerTree() == null) {
@@ -811,7 +795,7 @@
op.setValue(OpKind_NO_OP);
return false;
}
- newOp = createOperation(op.getIntegerValue().intValue(), newOp1, newOp2);
+ newOp = createOperation(op.getIntegerValue().intValue(), newOp1, newOp2, objectPool);
if (newOp == null) {
tree.setInnerTree(null);
op.setValue(OpKind_NO_OP);
@@ -825,29 +809,30 @@
if (op.equals(op1) && op.equals(op2)) {
// same operators on both children . since op!=NO_OP, neither are op1,
// op2. so they both make tree and value contributions
- newOp = createOperation(op.getIntegerValue().intValue(), tree1, tree2);
+ newOp = createOperation(op.getIntegerValue().intValue(), tree1, tree2, objectPool);
if (newOp == null) {
return false;
}
- privateDoOperation(op.getIntegerValue().intValue(), val1, val2, dummy, true, true, false, val);
+ privateDoOperation(op.getIntegerValue().intValue(), val1, val2, dummy, true, true, false, val, objectPool);
tree.setInnerTree(newOp);
return true;
} else if (op.equals(op1)) {
// leftson makes a tree,value contribution
if (tree2.getInnerTree() == null) {
// rightson makes a value contribution
- privateDoOperation(op.getIntegerValue().intValue(), val1, val2, dummy, true, true, false, val);
+ privateDoOperation(op.getIntegerValue().intValue(), val1, val2, dummy, true, true, false, val,
+ objectPool);
tree.setInnerTree(tree1);
return true;
} else {
// rightson makes a tree contribution
- Operation local_newOp = createOperation(op.getIntegerValue().intValue(), tree1, tree2);
+ Operation local_newOp = createOperation(op.getIntegerValue().intValue(), tree1, tree2, objectPool);
if (local_newOp == null) {
tree.setInnerTree(null);
op.setValue(OpKind_NO_OP);
return false;
}
- val.copyFrom(val1);
+ val.setValue(val1);
tree.setInnerTree(local_newOp); // NAC - BUG FIX
return true;
}
@@ -855,19 +840,20 @@
// rightson makes a tree,value contribution
if (tree1.getInnerTree() == null) {
// leftson makes a value contribution
- privateDoOperation(op.getIntegerValue().intValue(), val1, val2, dummy, true, true, false, val);
+ privateDoOperation(op.getIntegerValue().intValue(), val1, val2, dummy, true, true, false, val,
+ objectPool);
tree.setInnerTree(tree2);
return true;
} else {
// leftson makes a tree contribution
- Operation local_newOp = createOperation(op.getIntegerValue().intValue(), tree1, tree2);
+ Operation local_newOp = createOperation(op.getIntegerValue().intValue(), tree1, tree2, objectPool);
if (local_newOp == null) {
tree.setInnerTree(null);
op.setValue(OpKind_NO_OP);
return false;
}
tree.setInnerTree(local_newOp); // NAC BUG FIX
- val.copyFrom(val2);
+ val.setValue(val2);
return true;
}
}
@@ -875,7 +861,8 @@
throw new HyracksDataException("Should not reach here");
}
- public static int doComparison(int op, Value v1, Value v2, Value result) throws HyracksDataException {
+ public static int doComparison(int op, Value v1, Value v2, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
ValueType vt1;
ValueType vt2;
ValueType coerceResult;
@@ -887,7 +874,7 @@
coerceResult = vt1;
} else {
// do numerical type promotions --- other types/values are unchanged
- coerceResult = coerceToNumber(v1, v2);
+ coerceResult = coerceToNumber(v1, v2, objectPool);
vt1 = v1.getType();
vt2 = v2.getType();
}
@@ -929,15 +916,15 @@
result.setErrorValue();
return (SigValues.SIG_CHLD1.ordinal() | SigValues.SIG_CHLD2.ordinal());
}
- compareStrings(op, v1, v2, result);
+ compareStrings(op, v1, v2, result, objectPool);
return (SigValues.SIG_CHLD1.ordinal() | SigValues.SIG_CHLD2.ordinal());
case INTEGER_VALUE:
- compareIntegers(op, v1, v2, result);
+ compareIntegers(op, v1, v2, result, objectPool);
return (SigValues.SIG_CHLD1.ordinal() | SigValues.SIG_CHLD2.ordinal());
case REAL_VALUE:
- compareReals(op, v1, v2, result);
+ compareReals(op, v1, v2, result, objectPool);
return (SigValues.SIG_CHLD1.ordinal() | SigValues.SIG_CHLD2.ordinal());
case BOOLEAN_VALUE:
@@ -946,7 +933,7 @@
result.setErrorValue();
return (SigValues.SIG_CHLD1.ordinal() | SigValues.SIG_CHLD2.ordinal());
}
- compareBools(op, v1, v2, result);
+ compareBools(op, v1, v2, result, objectPool);
return (SigValues.SIG_CHLD1.ordinal() | SigValues.SIG_CHLD2.ordinal());
case LIST_VALUE:
@@ -960,7 +947,7 @@
result.setErrorValue();
return (SigValues.SIG_CHLD1.ordinal() | SigValues.SIG_CHLD2.ordinal());
}
- compareAbsoluteTimes(op, v1, v2, result);
+ compareAbsoluteTimes(op, v1, v2, result, objectPool);
return (SigValues.SIG_CHLD1.ordinal() | SigValues.SIG_CHLD2.ordinal());
case RELATIVE_TIME_VALUE:
@@ -968,7 +955,7 @@
result.setErrorValue();
return (SigValues.SIG_CHLD1.ordinal() | SigValues.SIG_CHLD2.ordinal());
}
- compareRelativeTimes(op, v1, v2, result);
+ compareRelativeTimes(op, v1, v2, result, objectPool);
return (SigValues.SIG_CHLD1.ordinal() | SigValues.SIG_CHLD2.ordinal());
default:
@@ -977,12 +964,13 @@
}
}
- public static int doArithmetic(int op, Value v1, Value v2, Value result) throws HyracksDataException {
- AMutableInt64 i1 = new AMutableInt64(0);
- AMutableInt64 i2 = new AMutableInt64(0);
- ClassAdTime t1 = new ClassAdTime();
- AMutableDouble r1 = new AMutableDouble(0);
- MutableBoolean b1 = new MutableBoolean();
+ public static int doArithmetic(int op, Value v1, Value v2, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ AMutableInt64 i1 = objectPool.int64Pool.get();
+ AMutableInt64 i2 = objectPool.int64Pool.get();
+ ClassAdTime t1 = objectPool.classAdTimePool.get();
+ AMutableDouble r1 = objectPool.doublePool.get();
+ MutableBoolean b1 = objectPool.boolPool.get();
// ensure the operands have arithmetic types
if ((!v1.isIntegerValue() && !v1.isRealValue() && !v1.isAbsoluteTimeValue() && !v1.isRelativeTimeValue()
@@ -1009,7 +997,7 @@
result.setBooleanValue(!b1.booleanValue());
} else if (v1.isExceptional()) {
// undefined or error --- same as operand
- result.copyFrom(v1);
+ result.setValue(v1);
return SigValues.SIG_CHLD1.ordinal();
}
// unary minus not defined on any other operand type
@@ -1018,7 +1006,7 @@
}
// perform type promotions and proceed with arithmetic
- switch (coerceToNumber(v1, v2)) {
+ switch (coerceToNumber(v1, v2, objectPool)) {
case INTEGER_VALUE:
v1.isIntegerValue(i1);
v2.isIntegerValue(i2);
@@ -1057,11 +1045,11 @@
}
case REAL_VALUE: {
- return (doRealArithmetic(op, v1, v2, result));
+ return (doRealArithmetic(op, v1, v2, result, objectPool));
}
case ABSOLUTE_TIME_VALUE:
case RELATIVE_TIME_VALUE: {
- return (doTimeArithmetic(op, v1, v2, result));
+ return (doTimeArithmetic(op, v1, v2, result, objectPool));
}
default:
// should not get here
@@ -1069,9 +1057,10 @@
}
}
- public static int doLogical(int op, Value v1, Value v2, Value result) throws HyracksDataException {
- MutableBoolean b1 = new MutableBoolean();
- MutableBoolean b2 = new MutableBoolean();
+ public static int doLogical(int op, Value v1, Value v2, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ MutableBoolean b1 = objectPool.boolPool.get();
+ MutableBoolean b2 = objectPool.boolPool.get();
// first coerece inputs to boolean if they are considered equivalent
if (!v1.isBooleanValue(b1) && v1.isBooleanValueEquiv(b1)) {
@@ -1098,7 +1087,7 @@
if (vt1 == ValueType.BOOLEAN_VALUE) {
result.setBooleanValue(!b1.booleanValue());
} else {
- result.copyFrom(v1);
+ result.setValue(v1);
}
return SigValues.SIG_CHLD1.ordinal();
}
@@ -1111,9 +1100,9 @@
result.setErrorValue();
return SigValues.SIG_CHLD1.ordinal();
} else if (vt1 == ValueType.BOOLEAN_VALUE && !b1.booleanValue()) {
- result.copyFrom(v2);
+ result.setValue(v2);
} else if (vt2 != ValueType.BOOLEAN_VALUE) {
- result.copyFrom(v2);
+ result.setValue(v2);
} else if (b2.booleanValue()) {
result.setBooleanValue(true);
} else {
@@ -1128,9 +1117,9 @@
result.setErrorValue();
return SigValues.SIG_CHLD1.ordinal();
} else if (vt1 == ValueType.BOOLEAN_VALUE && b1.booleanValue()) {
- result.copyFrom(v2);
+ result.setValue(v2);
} else if (vt2 != ValueType.BOOLEAN_VALUE) {
- result.copyFrom(v2);
+ result.setValue(v2);
} else if (!b2.booleanValue()) {
result.setBooleanValue(false);
} else {
@@ -1142,9 +1131,10 @@
throw new HyracksDataException("Shouldn't reach here");
}
- public static int doBitwise(int op, Value v1, Value v2, Value result) throws HyracksDataException {
- AMutableInt64 i1 = new AMutableInt64(0);
- AMutableInt64 i2 = new AMutableInt64(0);
+ public static int doBitwise(int op, Value v1, Value v2, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ AMutableInt64 i1 = objectPool.int64Pool.get();
+ AMutableInt64 i2 = objectPool.int64Pool.get();
// bitwise operations are defined only on integers
if (op == OpKind_BITWISE_NOT_OP) {
@@ -1209,9 +1199,10 @@
//out of domain value
public static final int EDOM = 33;
- public static int doRealArithmetic(int op, Value v1, Value v2, Value result) throws HyracksDataException {
- AMutableDouble r1 = new AMutableDouble(0);
- AMutableDouble r2 = new AMutableDouble(0);
+ public static int doRealArithmetic(int op, Value v1, Value v2, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ AMutableDouble r1 = objectPool.doublePool.get();
+ AMutableDouble r2 = objectPool.doublePool.get();
double comp = 0;
// we want to prevent FPE and set the ERROR value on the result; on Unix
@@ -1251,9 +1242,9 @@
return (SigValues.SIG_CHLD1.ordinal() | SigValues.SIG_CHLD2.ordinal());
}
- public static int doTimeArithmetic(int op, Value v1, Value v2, Value result) {
- ClassAdTime asecs1 = new ClassAdTime();
- ClassAdTime asecs2 = new ClassAdTime();
+ public static int doTimeArithmetic(int op, Value v1, Value v2, Value result, ClassAdObjectPool objectPool) {
+ ClassAdTime asecs1 = objectPool.classAdTimePool.get();
+ ClassAdTime asecs2 = objectPool.classAdTimePool.get();
ValueType vt1 = v1.getType();
ValueType vt2 = v2.getType();
@@ -1309,8 +1300,8 @@
if (op == OpKind_MULTIPLICATION_OP || op == OpKind_DIVISION_OP) {
if (vt1 == ValueType.RELATIVE_TIME_VALUE && vt2 == ValueType.INTEGER_VALUE) {
- AMutableInt64 num = new AMutableInt64(0);
- ClassAdTime msecs = new ClassAdTime();
+ AMutableInt64 num = objectPool.int64Pool.get();
+ ClassAdTime msecs = objectPool.classAdTimePool.get();
v1.isRelativeTimeValue(asecs1);
v2.isIntegerValue(num);
if (op == OpKind_MULTIPLICATION_OP) {
@@ -1323,8 +1314,8 @@
}
if (vt1 == ValueType.RELATIVE_TIME_VALUE && vt2 == ValueType.REAL_VALUE) {
- AMutableDouble num = new AMutableDouble(0);
- AMutableDouble msecs = new AMutableDouble(0);
+ AMutableDouble num = objectPool.doublePool.get();
+ AMutableDouble msecs = objectPool.doublePool.get();
v1.isRelativeTimeValue(asecs1);
v2.isRealValue(num);
if (op == OpKind_MULTIPLICATION_OP) {
@@ -1332,25 +1323,30 @@
} else {
msecs.setValue(asecs1.getRelativeTime() * num.getDoubleValue());
}
- result.setRelativeTimeValue(new ClassAdTime(1000L * ((long) msecs.getDoubleValue()), false));
+ ClassAdTime time = objectPool.classAdTimePool.get();
+ time.setRelativeTime(1000L * ((long) msecs.getDoubleValue()));
+ result.setRelativeTimeValue(time);
return (SigValues.SIG_CHLD1.ordinal() | SigValues.SIG_CHLD2.ordinal());
}
if (vt1 == ValueType.INTEGER_VALUE && vt2 == ValueType.RELATIVE_TIME_VALUE
&& op == OpKind_MULTIPLICATION_OP) {
- AMutableInt64 num = new AMutableInt64(0);
+ AMutableInt64 num = objectPool.int64Pool.get();
v1.isIntegerValue(num);
v2.isRelativeTimeValue(asecs1);
- result.setRelativeTimeValue(new ClassAdTime(num.getLongValue() * asecs1.getRelativeTime(), false));
+ ClassAdTime time = objectPool.classAdTimePool.get();
+ time.setRelativeTime(num.getLongValue() * asecs1.getRelativeTime());
+ result.setRelativeTimeValue(time);
return (SigValues.SIG_CHLD1.ordinal() | SigValues.SIG_CHLD2.ordinal());
}
if (vt2 == ValueType.RELATIVE_TIME_VALUE && vt1 == ValueType.REAL_VALUE && op == OpKind_MULTIPLICATION_OP) {
- AMutableDouble num = new AMutableDouble(0);
+ AMutableDouble num = objectPool.doublePool.get();
v1.isRelativeTimeValue(asecs1);
v2.isRealValue(num);
- result.setRelativeTimeValue(
- new ClassAdTime((long) (asecs1.getRelativeTime() * num.getDoubleValue()), false));
+ ClassAdTime time = objectPool.classAdTimePool.get();
+ time.setRelativeTime((long) (asecs1.getRelativeTime() * num.getDoubleValue()));
+ result.setRelativeTimeValue(time);
return (SigValues.SIG_CHLD1.ordinal() | SigValues.SIG_CHLD2.ordinal());
}
}
@@ -1359,9 +1355,9 @@
return (SigValues.SIG_CHLD1.ordinal() | SigValues.SIG_CHLD2.ordinal());
}
- public static void compareStrings(int op, Value v1, Value v2, Value result) {
- AMutableCharArrayString s1 = new AMutableCharArrayString();
- AMutableCharArrayString s2 = new AMutableCharArrayString();
+ public static void compareStrings(int op, Value v1, Value v2, Value result, ClassAdObjectPool objectPool) {
+ AMutableCharArrayString s1 = objectPool.strPool.get();
+ AMutableCharArrayString s2 = objectPool.strPool.get();
int cmp;
v1.isStringValue(s1);
v2.isStringValue(s2);
@@ -1392,9 +1388,10 @@
}
}
- public static void compareAbsoluteTimes(int op, Value v1, Value v2, Value result) throws HyracksDataException {
- ClassAdTime asecs1 = new ClassAdTime();
- ClassAdTime asecs2 = new ClassAdTime();
+ public static void compareAbsoluteTimes(int op, Value v1, Value v2, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ ClassAdTime asecs1 = objectPool.classAdTimePool.get();
+ ClassAdTime asecs2 = objectPool.classAdTimePool.get();
boolean compResult = false;
v1.isAbsoluteTimeValue(asecs1);
v2.isAbsoluteTimeValue(asecs2);
@@ -1430,9 +1427,10 @@
result.setBooleanValue(compResult);
}
- public static void compareRelativeTimes(int op, Value v1, Value v2, Value result) throws HyracksDataException {
- ClassAdTime rsecs1 = new ClassAdTime();
- ClassAdTime rsecs2 = new ClassAdTime();
+ public static void compareRelativeTimes(int op, Value v1, Value v2, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ ClassAdTime rsecs1 = objectPool.classAdTimePool.get();
+ ClassAdTime rsecs2 = objectPool.classAdTimePool.get();
boolean compResult = false;
v1.isRelativeTimeValue(rsecs1);
@@ -1472,9 +1470,10 @@
result.setBooleanValue(compResult);
}
- public static void compareBools(int op, Value v1, Value v2, Value result) throws HyracksDataException {
- MutableBoolean b1 = new MutableBoolean();
- MutableBoolean b2 = new MutableBoolean();
+ public static void compareBools(int op, Value v1, Value v2, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ MutableBoolean b1 = objectPool.boolPool.get();
+ MutableBoolean b2 = objectPool.boolPool.get();
boolean compResult = false;
v1.isBooleanValue(b1);
v2.isBooleanValue(b2);
@@ -1511,9 +1510,10 @@
result.setBooleanValue(compResult);
}
- public static void compareIntegers(int op, Value v1, Value v2, Value result) throws HyracksDataException {
- AMutableInt64 i1 = new AMutableInt64(0);
- AMutableInt64 i2 = new AMutableInt64(0);
+ public static void compareIntegers(int op, Value v1, Value v2, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ AMutableInt64 i1 = objectPool.int64Pool.get();
+ AMutableInt64 i2 = objectPool.int64Pool.get();
boolean compResult = false;
v1.isIntegerValue(i1);
v2.isIntegerValue(i2);
@@ -1549,9 +1549,10 @@
result.setBooleanValue(compResult);
}
- public static void compareReals(int op, Value v1, Value v2, Value result) throws HyracksDataException {
- AMutableDouble r1 = new AMutableDouble(0);
- AMutableDouble r2 = new AMutableDouble(0);
+ public static void compareReals(int op, Value v1, Value v2, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ AMutableDouble r1 = objectPool.doublePool.get();
+ AMutableDouble r2 = objectPool.doublePool.get();
boolean compResult = false;
v1.isRealValue(r1);
@@ -1594,26 +1595,33 @@
// + if both v1 and v2 are Numbers and of the same type, return type
// + if v1 is an int and v2 is a real, convert v1 to real; return REAL_VALUE
// + if v1 is a real and v2 is an int, convert v2 to real; return REAL_VALUE
- public static ValueType coerceToNumber(Value v1, Value v2) {
- AMutableInt64 i = new AMutableInt64(0);
- AMutableDouble r = new AMutableDouble(0);
- MutableBoolean b = new MutableBoolean();
+ public static ValueType coerceToNumber(Value v1, Value v2, ClassAdObjectPool objectPool) {
+ AMutableInt64 i = objectPool.int64Pool.get();
+ AMutableDouble r = objectPool.doublePool.get();
+ MutableBoolean b = objectPool.boolPool.get();
// either of v1, v2 not numerical?
- if (v1.isClassAdValue() || v2.isClassAdValue())
+ if (v1.isClassAdValue() || v2.isClassAdValue()) {
return ValueType.CLASSAD_VALUE;
- if (v1.isListValue() || v2.isListValue())
+ }
+ if (v1.isListValue() || v2.isListValue()) {
return ValueType.LIST_VALUE;
- if (v1.isStringValue() || v2.isStringValue())
+ }
+ if (v1.isStringValue() || v2.isStringValue()) {
return ValueType.STRING_VALUE;
- if (v1.isUndefinedValue() || v2.isUndefinedValue())
+ }
+ if (v1.isUndefinedValue() || v2.isUndefinedValue()) {
return ValueType.UNDEFINED_VALUE;
- if (v1.isErrorValue() || v2.isErrorValue())
+ }
+ if (v1.isErrorValue() || v2.isErrorValue()) {
return ValueType.ERROR_VALUE;
- if (v1.isAbsoluteTimeValue() || v2.isAbsoluteTimeValue())
+ }
+ if (v1.isAbsoluteTimeValue() || v2.isAbsoluteTimeValue()) {
return ValueType.ABSOLUTE_TIME_VALUE;
- if (v1.isRelativeTimeValue() || v2.isRelativeTimeValue())
+ }
+ if (v1.isRelativeTimeValue() || v2.isRelativeTimeValue()) {
return ValueType.RELATIVE_TIME_VALUE;
+ }
// promote booleans to integers
if (v1.isBooleanValue(b)) {
@@ -1633,41 +1641,51 @@
}
// both v1 and v2 of same numerical type
- if (v1.isIntegerValue(i) && v2.isIntegerValue(i))
+ if (v1.isIntegerValue(i) && v2.isIntegerValue(i)) {
return ValueType.INTEGER_VALUE;
- if (v1.isRealValue(r) && v2.isRealValue(r))
+ }
+ if (v1.isRealValue(r) && v2.isRealValue(r)) {
return ValueType.REAL_VALUE;
+ }
// type promotions required
- if (v1.isIntegerValue(i) && v2.isRealValue(r))
+ if (v1.isIntegerValue(i) && v2.isRealValue(r)) {
v1.setRealValue(i.getLongValue());
- else if (v1.isRealValue(r) && v2.isIntegerValue(i))
+ } else if (v1.isRealValue(r) && v2.isIntegerValue(i)) {
v2.setRealValue(i.getLongValue());
+ }
return ValueType.REAL_VALUE;
}
- public Operation(int op, ExprTreeHolder e1, ExprTreeHolder e2, ExprTreeHolder e3) {
+ public Operation(int op, ExprTreeHolder e1, ExprTreeHolder e2, ExprTreeHolder e3, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ super(objectPool);
this.opKind = op;
- this.child1 = e1 == null ? null : e1.self();
- this.child2 = e2 == null ? null : e2.self();
- this.child3 = e3 == null ? null : e3.self();
+ this.child1 = new ExprTreeHolder(objectPool);
+ this.child2 = new ExprTreeHolder(objectPool);
+ this.child3 = new ExprTreeHolder(objectPool);
+ child1.copyFrom(e1);
+ child2.copyFrom(e2);
+ child3.copyFrom(e3);
}
- public static Operation createOperation(int op, ExprTree e1, ExprTree e2, ExprTree e3) {
- Operation opnode = new Operation();
+ public static Operation createOperation(int op, ExprTree e1, ExprTree e2, ExprTree e3, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Operation opnode = objectPool.operationPool.get();
opnode.opKind = op;
- opnode.child1 = e1 == null ? null : e1.self();
- opnode.child2 = e2 == null ? null : e2.self();
- opnode.child3 = e3 == null ? null : e3.self();
+ opnode.child1.copyFrom(e1);
+ opnode.child2.copyFrom(e2);
+ opnode.child3.copyFrom(e3);
return opnode;
}
- public static void createOperation(int op, ExprTree e1, ExprTree e2, ExprTree e3, Operation opnode) {
+ public static void createOperation(int op, ExprTree e1, ExprTree e2, ExprTree e3, Operation opnode)
+ throws HyracksDataException {
opnode.opKind = op;
- opnode.child1 = e1 == null ? null : e1.self();
- opnode.child2 = e2 == null ? null : e2.self();
- opnode.child3 = e3 == null ? null : e3.self();
+ opnode.child1.copyFrom(e1);
+ opnode.child2.copyFrom(e2);
+ opnode.child3.copyFrom(e3);
}
public void getComponents(AMutableInt32 op, ExprTreeHolder e1, ExprTreeHolder e2, ExprTreeHolder e3) {
@@ -1677,37 +1695,39 @@
e3.setInnerTree(child3);
}
- public static Operation createOperation(int op, Value val, ExprTreeHolder tree) throws HyracksDataException {
+ public static Operation createOperation(int op, Value val, ExprTreeHolder tree, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
if (tree.getInnerTree() == null) {
return null;
}
- Literal lit = Literal.createLiteral(val);
+ Literal lit = Literal.createLiteral(val, objectPool);
if (lit == null) {
return null;
}
- Operation newOp = createOperation(op, lit, tree);
+ Operation newOp = createOperation(op, lit, tree, objectPool);
return newOp;
}
- public static Operation createOperation(int op, ExprTreeHolder tree, Value val) throws HyracksDataException {
+ public static Operation createOperation(int op, ExprTreeHolder tree, Value val, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
if (tree.getInnerTree() == null) {
return null;
}
- Literal lit = Literal.createLiteral(val);
+ Literal lit = Literal.createLiteral(val, objectPool);
if (lit == null) {
return null;
}
- Operation newOp = createOperation(op, lit, tree);
+ Operation newOp = createOperation(op, lit, tree, objectPool);
return newOp;
}
public boolean flattenSpecials(EvalState state, Value val, ExprTreeHolder tree) throws HyracksDataException {
- ExprTreeHolder fChild1 = new ExprTreeHolder();
- ExprTreeHolder fChild2 = new ExprTreeHolder();
- ExprTreeHolder fChild3 = new ExprTreeHolder();
- Value eval1 = new Value();
- Value eval2 = new Value();
- Value eval3 = new Value();
+ ExprTreeHolder fChild1 = objectPool.mutableExprPool.get();
+ ExprTreeHolder fChild2 = objectPool.mutableExprPool.get();
+ ExprTreeHolder fChild3 = objectPool.mutableExprPool.get();
+ Value eval1 = objectPool.valuePool.get();
+ Value eval2 = objectPool.valuePool.get();
+ Value eval3 = objectPool.valuePool.get();
switch (opKind) {
case OpKind_UNARY_PLUS_OP:
@@ -1720,12 +1740,12 @@
return false;
}
if (fChild1.getInnerTree() != null) {
- tree.setInnerTree(Operation.createOperation(opKind, fChild1));
+ tree.setInnerTree(Operation.createOperation(opKind, fChild1, objectPool));
return (tree.getInnerTree() != null);
} else {
- privateDoOperation(opKind, eval1, null, null, true, false, false, val);
+ privateDoOperation(opKind, eval1, null, null, true, false, false, val, objectPool);
tree.setInnerTree(null);
- eval1.clear();
+ eval1.setUndefinedValue();
return true;
}
case OpKind_TERNARY_OP:
@@ -1737,11 +1757,11 @@
// check if selector expression collapsed to a non-undefined value
if (fChild1.getInnerTree() == null && !eval1.isUndefinedValue()) {
- MutableBoolean b = new MutableBoolean();
+ MutableBoolean b = objectPool.boolPool.get();
// if the selector is not boolean-equivalent, propagate error
if (!eval1.isBooleanValueEquiv(b)) {
val.setErrorValue();
- eval1.clear();
+ eval1.setUndefinedValue();
tree.setInnerTree(null);
return true;
}
@@ -1761,10 +1781,12 @@
}
// if any arm collapsed into a value, make it a Literal
- if (fChild2.getInnerTree() == null)
- fChild2.setInnerTree(Literal.createLiteral(eval2));
- if (fChild3.getInnerTree() == null)
- fChild3.setInnerTree(Literal.createLiteral(eval3));
+ if (fChild2.getInnerTree() == null) {
+ fChild2.setInnerTree(Literal.createLiteral(eval2, objectPool));
+ }
+ if (fChild3.getInnerTree() == null) {
+ fChild3.setInnerTree(Literal.createLiteral(eval3, objectPool));
+ }
if (fChild2.getInnerTree() == null || fChild3.getInnerTree() == null) {
tree.setInnerTree(null);;
return false;
@@ -1775,7 +1797,7 @@
fChild1.setInnerTree(child1.copy());
}
- tree.setInnerTree(Operation.createOperation(opKind, fChild1, fChild2, fChild3));
+ tree.setInnerTree(Operation.createOperation(opKind, fChild1, fChild2, fChild3, objectPool));
if (tree.getInnerTree() == null) {
return false;
}
@@ -1790,22 +1812,24 @@
// if both arguments Flattened to values, Evaluate now
if (fChild1.getInnerTree() == null && fChild2.getInnerTree() == null) {
- privateDoOperation(opKind, eval1, eval2, null, true, true, false, val);
+ privateDoOperation(opKind, eval1, eval2, null, true, true, false, val, objectPool);
tree.setInnerTree(null);
return true;
}
// otherwise convert Flattened values into literals
- if (fChild1.getInnerTree() == null)
- fChild1.setInnerTree(Literal.createLiteral(eval1));
- if (fChild2.getInnerTree() == null)
- fChild2.setInnerTree(Literal.createLiteral(eval2));
+ if (fChild1.getInnerTree() == null) {
+ fChild1.setInnerTree(Literal.createLiteral(eval1, objectPool));
+ }
+ if (fChild2.getInnerTree() == null) {
+ fChild2.setInnerTree(Literal.createLiteral(eval2, objectPool));
+ }
if (fChild1.getInnerTree() == null || fChild2.getInnerTree() == null) {
tree.setInnerTree(null);
return false;
}
- tree.setInnerTree(Operation.createOperation(opKind, fChild1, fChild2));
+ tree.setInnerTree(Operation.createOperation(opKind, fChild1, fChild2, objectPool));
if (tree.getInnerTree() == null) {
return false;
}
@@ -1892,11 +1916,8 @@
@Override
public void reset() {
opKind = OpKind_NO_OP;
- if (child1 != null)
- child1.reset();
- if (child2 != null)
- child2.reset();
- if (child3 != null)
- child3.reset();
+ child1.reset();
+ child2.reset();
+ child3.reset();
}
}
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/PrettyPrint.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/PrettyPrint.java
index e738bb9..2553bfa 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/PrettyPrint.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/PrettyPrint.java
@@ -22,6 +22,7 @@
import java.util.Map.Entry;
import org.apache.asterix.external.classad.ExprTree.NodeKind;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.asterix.om.base.AMutableInt32;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -32,7 +33,8 @@
private boolean minimalParens;
private int indentLevel;
- public PrettyPrint() {
+ public PrettyPrint(ClassAdObjectPool objectPool) {
+ super(objectPool);
classadIndent = 4;
listIndent = 3;
wantStringQuotes = true;
@@ -116,8 +118,9 @@
return;
}
// all others are binary ops
- AMutableInt32 top = new AMutableInt32(0);
- ExprTreeHolder t1 = new ExprTreeHolder(), t2 = new ExprTreeHolder(), t3 = new ExprTreeHolder();
+ AMutableInt32 top = objectPool.int32Pool.get();
+ ExprTreeHolder t1 = objectPool.mutableExprPool.get(), t2 = objectPool.mutableExprPool.get(),
+ t3 = objectPool.mutableExprPool.get();
if (op1.getKind() == NodeKind.OP_NODE) {
((Operation) op1.getInnerTree()).getComponents(top, t1, t2, t3);
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Value.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Value.java
index f0466b2..427e2a2 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Value.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Value.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.external.classad;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.asterix.om.base.AMutableDouble;
import org.apache.asterix.om.base.AMutableInt32;
import org.apache.asterix.om.base.AMutableInt64;
@@ -30,9 +31,9 @@
private long longVal;
private double doubleVal;
private boolean boolVal;
- private ClassAdTime timeVal = new ClassAdTime();
- private ClassAd classadVal = new ClassAd();
- private ExprList listVal = new ExprList();
+ private final ClassAdTime timeVal;
+ private final ClassAd classadVal;
+ private final ExprList listVal;
private String stringVal;
/// Value types
@@ -164,7 +165,7 @@
}
}
- public boolean isClassAdValue(ClassAd ad) {
+ public boolean isClassAdValue(ClassAd ad) throws HyracksDataException {
if (valueType == ValueType.CLASSAD_VALUE) {
ad.setValue(classadVal);
return true;
@@ -265,49 +266,11 @@
1024.0 * 1024.0 * 1024.0 * 1024.0 // Terra
};
- public Value() {
+ public Value(ClassAdObjectPool objectPool) {
valueType = ValueType.UNDEFINED_VALUE;
- }
-
- public Value(Value value) throws HyracksDataException {
- valueType = value.valueType;
- switch (value.valueType) {
- case ABSOLUTE_TIME_VALUE:
- timeVal = new ClassAdTime(value.timeVal);
- break;
- case BOOLEAN_VALUE:
- this.boolVal = value.boolVal;
- break;
- case CLASSAD_VALUE:
- this.classadVal = new ClassAd(value.classadVal);
- break;
- case ERROR_VALUE:
- break;
- case INTEGER_VALUE:
- this.longVal = value.longVal;
- break;
- case LIST_VALUE:
- this.listVal = new ExprList(value.listVal);
- break;
- case NULL_VALUE:
- break;
- case REAL_VALUE:
- this.doubleVal = value.doubleVal;
- break;
- case RELATIVE_TIME_VALUE:
- this.timeVal = new ClassAdTime(value.timeVal);
- break;
- case SLIST_VALUE:
- this.listVal = new ExprList(value.listVal);
- break;
- case STRING_VALUE:
- this.stringVal = value.stringVal;
- break;
- case UNDEFINED_VALUE:
- break;
- default:
- break;
- }
+ this.timeVal = new ClassAdTime();
+ this.listVal = new ExprList(objectPool);
+ this.classadVal = new ClassAd(objectPool);
}
public void setValue(Value value) throws HyracksDataException {
@@ -351,14 +314,15 @@
}
}
- public void assign(Value value) throws HyracksDataException {
- if (this != value) {
- setValue(value);
- }
+ private void clear() {
+ valueType = ValueType.UNDEFINED_VALUE;
}
- public void clear() {
+ public void reset() {
valueType = ValueType.UNDEFINED_VALUE;
+ listVal.reset();
+ timeVal.reset();
+ classadVal.reset();
}
public void setRealValue(double r) {
@@ -409,7 +373,7 @@
listVal.setValue(expList);
}
- public void setClassAdValue(ClassAd ad) {
+ public void setClassAdValue(ClassAd ad) throws HyracksDataException {
clear();
valueType = ValueType.CLASSAD_VALUE;
classadVal.setValue(ad);
@@ -483,7 +447,7 @@
@Override
public String toString() {
- ClassAdUnParser unparser = new PrettyPrint();
+ ClassAdUnParser unparser = new PrettyPrint(new ClassAdObjectPool());
AMutableCharArrayString unparsed_text = new AMutableCharArrayString();
switch (valueType) {
case ABSOLUTE_TIME_VALUE:
@@ -521,14 +485,15 @@
return null;
}
- public static boolean convertValueToRealValue(Value value, Value realValue) throws HyracksDataException {
+ public static boolean convertValueToRealValue(Value value, Value realValue, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
boolean could_convert;
- AMutableCharArrayString buf = new AMutableCharArrayString();
+ AMutableCharArrayString buf = objectPool.strPool.get();
int endIndex;
char end;
- AMutableInt64 ivalue = new AMutableInt64(0);
- ClassAdTime atvalue = new ClassAdTime();
- MutableBoolean bvalue = new MutableBoolean();
+ AMutableInt64 ivalue = objectPool.int64Pool.get();
+ ClassAdTime atvalue = objectPool.classAdTimePool.get();
+ MutableBoolean bvalue = objectPool.boolPool.get();
double rvalue;
NumberFactor nf = NumberFactor.NO_FACTOR;
@@ -604,7 +569,7 @@
break;
case REAL_VALUE:
- realValue.copyFrom(value);
+ realValue.setValue(value);
could_convert = true;
break;
@@ -627,14 +592,15 @@
return could_convert;
}
- public static boolean convertValueToIntegerValue(Value value, Value integerValue) throws HyracksDataException {
+ public static boolean convertValueToIntegerValue(Value value, Value integerValue, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
boolean could_convert;
- AMutableCharArrayString buf = new AMutableCharArrayString();
+ AMutableCharArrayString buf = objectPool.strPool.get();
char end;
- AMutableInt64 ivalue = new AMutableInt64(0);
- AMutableDouble rtvalue = new AMutableDouble(0);
- ClassAdTime atvalue = new ClassAdTime();
- MutableBoolean bvalue = new MutableBoolean();
+ AMutableInt64 ivalue = objectPool.int64Pool.get();
+ AMutableDouble rtvalue = objectPool.doublePool.get();
+ ClassAdTime atvalue = objectPool.classAdTimePool.get();
+ MutableBoolean bvalue = objectPool.boolPool.get();
NumberFactor nf;
switch (value.getType()) {
@@ -700,7 +666,7 @@
break;
case INTEGER_VALUE:
- integerValue.copyFrom(value);
+ integerValue.setValue(value);
could_convert = true;
break;
@@ -729,51 +695,12 @@
return could_convert;
}
- public void copyFrom(Value val) throws HyracksDataException {
- clear();
- valueType = val.valueType;
- switch (val.valueType) {
- case STRING_VALUE:
- stringVal = val.stringVal;
- return;
-
- case BOOLEAN_VALUE:
- boolVal = val.boolVal;
- return;
-
- case INTEGER_VALUE:
- longVal = val.longVal;
- return;
-
- case REAL_VALUE:
-
- doubleVal = val.doubleVal;
- return;
- case UNDEFINED_VALUE:
- case ERROR_VALUE:
- return;
- case LIST_VALUE:
- case SLIST_VALUE:
- listVal.copyFrom(val.listVal);
- return;
- case CLASSAD_VALUE:
- classadVal.copyFrom(val.classadVal);
- return;
-
- case RELATIVE_TIME_VALUE:
- case ABSOLUTE_TIME_VALUE:
- timeVal.setValue(val.timeVal);
- return;
- default:
- setUndefinedValue();
- }
- }
-
- public static boolean convertValueToStringValue(Value value, Value stringValue) throws HyracksDataException {
+ public static boolean convertValueToStringValue(Value value, Value stringValue, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
boolean could_convert = false;
- ClassAdTime atvalue = new ClassAdTime();
- AMutableCharArrayString string_representation = new AMutableCharArrayString();
- ClassAdUnParser unparser = new PrettyPrint();
+ ClassAdTime atvalue = objectPool.classAdTimePool.get();
+ AMutableCharArrayString string_representation = objectPool.strPool.get();
+ ClassAdUnParser unparser = objectPool.prettyPrintPool.get();
switch (value.getType()) {
case UNDEFINED_VALUE:
@@ -787,7 +714,7 @@
break;
case STRING_VALUE:
- stringValue.copyFrom(value);
+ stringValue.setValue(value);
could_convert = true;
break;
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/AMutableCharArrayStringPool.java
similarity index 65%
copy from asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java
copy to asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/AMutableCharArrayStringPool.java
index b10452e..b434522 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/AMutableCharArrayStringPool.java
@@ -16,14 +16,20 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.external.api;
+package org.apache.asterix.external.classad.object.pool;
-import org.apache.asterix.external.input.stream.AInputStream;
-import org.apache.asterix.external.util.FeedLogManager;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.asterix.external.classad.AMutableCharArrayString;
-public interface IInputStreamProvider {
- public AInputStream getInputStream() throws HyracksDataException;
+public class AMutableCharArrayStringPool extends Pool<AMutableCharArrayString> {
- public void setFeedLogManager(FeedLogManager feedLogManager);
+ @Override
+ public AMutableCharArrayString newInstance() {
+ return new AMutableCharArrayString();
+ }
+
+ @Override
+ protected void reset(AMutableCharArrayString str) {
+ str.reset();
+ }
+
}
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/AttributeReferencePool.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/AttributeReferencePool.java
index d7bcfcc..9eb2c0c 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/AttributeReferencePool.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/AttributeReferencePool.java
@@ -22,9 +22,16 @@
public class AttributeReferencePool extends Pool<AttributeReference> {
+ private final ClassAdObjectPool objectPool;
+
+ public AttributeReferencePool(ClassAdObjectPool objectPool) {
+ this.objectPool = objectPool;
+
+ }
+
@Override
public AttributeReference newInstance() {
- return new AttributeReference();
+ return new AttributeReference(objectPool);
}
@Override
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/CaseInsensitiveStringPool.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/CaseInsensitiveStringPool.java
index 7e88961..f8a70f1 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/CaseInsensitiveStringPool.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/CaseInsensitiveStringPool.java
@@ -18,32 +18,10 @@
*/
package org.apache.asterix.external.classad.object.pool;
-import java.util.Stack;
-
import org.apache.asterix.external.classad.CaseInsensitiveString;
public class CaseInsensitiveStringPool extends Pool<CaseInsensitiveString> {
- protected Stack<CaseInsensitiveString> stock = new Stack<CaseInsensitiveString>();
-
- @Override
- public CaseInsensitiveString get() {
- if (!stock.isEmpty()) {
- return stock.pop();
- } else {
- return newInstance();
-
- }
- }
-
- @Override
- public void reset() {
- }
-
- public void put(CaseInsensitiveString aString) {
- stock.push(aString);
- }
-
@Override
public CaseInsensitiveString newInstance() {
return new CaseInsensitiveString();
@@ -51,5 +29,6 @@
@Override
protected void reset(CaseInsensitiveString obj) {
+ obj.set(null);
}
}
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdObjectPool.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdObjectPool.java
new file mode 100644
index 0000000..c6e265f
--- /dev/null
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdObjectPool.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.classad.object.pool;
+
+import org.apache.asterix.external.classad.CaseInsensitiveString;
+import org.apache.asterix.external.classad.ExprTree;
+
+public class ClassAdObjectPool {
+ public final ExprHolderPool mutableExprPool;
+ public final TokenValuePool tokenValuePool;
+ public final ClassAdPool classAdPool;
+ public final ExprListPool exprListPool;
+ public final ValuePool valuePool;
+ public final LiteralPool literalPool;
+ public final BitSetPool bitSetPool;
+ public final OperationPool operationPool;
+ public final AttributeReferencePool attrRefPool;
+ public final ClassAdParserPool classAdParserPool;
+ public final AMutableCharArrayStringPool strPool;
+ public final FunctionCallPool funcPool;
+ public final MutableBooleanPool boolPool;
+ public final CaseInsensitiveStringPool caseInsensitiveStringPool;
+ public final StringArrayListPool stringArrayListPool;
+ public final Int32Pool int32Pool;
+ public final Int64Pool int64Pool;
+ public final ClassAdTimePool classAdTimePool;
+ public final DoublePool doublePool;
+ public final EvalStatePool evalStatePool;
+ public final HashMapPool<CaseInsensitiveString, ExprTree> strToExprPool;
+ public final PrettyPrintPool prettyPrintPool;
+ public final TreeSetPool<String> strSetPool;
+ public final MutableNumberFactorPool numFactorPool;
+
+ public ClassAdObjectPool() {
+ int32Pool = new Int32Pool();
+ int64Pool = new Int64Pool();
+ mutableExprPool = new ExprHolderPool(this);
+ tokenValuePool = new TokenValuePool();
+ classAdPool = new ClassAdPool(this);
+ exprListPool = new ExprListPool(this);
+ valuePool = new ValuePool(this);
+ literalPool = new LiteralPool(this);
+ bitSetPool = new BitSetPool();
+ operationPool = new OperationPool(this);
+ attrRefPool = new AttributeReferencePool(this);
+ strPool = new AMutableCharArrayStringPool();
+ funcPool = new FunctionCallPool(this);
+ boolPool = new MutableBooleanPool();
+ caseInsensitiveStringPool = new CaseInsensitiveStringPool();
+ stringArrayListPool = new StringArrayListPool();
+ classAdParserPool = new ClassAdParserPool(this);
+ classAdTimePool = new ClassAdTimePool();
+ doublePool = new DoublePool();
+ evalStatePool = new EvalStatePool(this);
+ strToExprPool = new HashMapPool<CaseInsensitiveString, ExprTree>();
+ prettyPrintPool = new PrettyPrintPool(this);
+ strSetPool = new TreeSetPool<String>();
+ numFactorPool = new MutableNumberFactorPool();
+ }
+
+ public void reset() {
+ mutableExprPool.reset();
+ tokenValuePool.reset();
+ classAdPool.reset();
+ exprListPool.reset();
+ valuePool.reset();
+ literalPool.reset();
+ bitSetPool.reset();
+ operationPool.reset();
+ attrRefPool.reset();
+ strPool.reset();
+ classAdParserPool.reset();
+ funcPool.reset();
+ boolPool.reset();
+ caseInsensitiveStringPool.reset();
+ stringArrayListPool.reset();
+ int32Pool.reset();
+ int64Pool.reset();
+ classAdTimePool.reset();
+ doublePool.reset();
+ evalStatePool.reset();
+ strToExprPool.reset();
+ prettyPrintPool.reset();
+ strSetPool.reset();
+ numFactorPool.reset();
+ }
+}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProviderFactory.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdParserPool.java
similarity index 62%
copy from asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProviderFactory.java
copy to asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdParserPool.java
index f52f7d3..4f85e7c 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProviderFactory.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdParserPool.java
@@ -16,18 +16,24 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.external.api;
+package org.apache.asterix.external.classad.object.pool;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.asterix.external.library.ClassAdParser;
-public interface IInputStreamProviderFactory extends IExternalDataSourceFactory {
+public class ClassAdParserPool extends Pool<ClassAdParser> {
+ private final ClassAdObjectPool objectPool;
- public IInputStreamProvider createInputStreamProvider(IHyracksTaskContext ctx, int partition)
- throws HyracksDataException;
+ public ClassAdParserPool(ClassAdObjectPool objectPool) {
+ this.objectPool = objectPool;
+ }
@Override
- public default DataSourceType getDataSourceType() {
- return DataSourceType.STREAM;
+ public ClassAdParser newInstance() {
+ return new ClassAdParser(objectPool);
}
+
+ @Override
+ protected void reset(ClassAdParser obj) {
+ }
+
}
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdPool.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdPool.java
index 40b8eb3..039e414 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdPool.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdPool.java
@@ -22,9 +22,15 @@
public class ClassAdPool extends Pool<ClassAd> {
+ private final ClassAdObjectPool objectPool;
+
+ public ClassAdPool(ClassAdObjectPool objectPool) {
+ this.objectPool = objectPool;
+ }
+
@Override
public ClassAd newInstance() {
- return new ClassAd(false, false);
+ return new ClassAd(objectPool);
}
@Override
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdTimePool.java
similarity index 66%
copy from asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java
copy to asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdTimePool.java
index b10452e..06a2c2f 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdTimePool.java
@@ -16,14 +16,20 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.external.api;
+package org.apache.asterix.external.classad.object.pool;
-import org.apache.asterix.external.input.stream.AInputStream;
-import org.apache.asterix.external.util.FeedLogManager;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.asterix.external.classad.ClassAdTime;
-public interface IInputStreamProvider {
- public AInputStream getInputStream() throws HyracksDataException;
+public class ClassAdTimePool extends Pool<ClassAdTime> {
- public void setFeedLogManager(FeedLogManager feedLogManager);
+ @Override
+ public ClassAdTime newInstance() {
+ return new ClassAdTime();
+ }
+
+ @Override
+ protected void reset(ClassAdTime obj) {
+ obj.setCurrentAbsolute();
+ }
+
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/DoublePool.java
similarity index 66%
copy from asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java
copy to asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/DoublePool.java
index b10452e..52c1d26 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/DoublePool.java
@@ -16,14 +16,19 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.external.api;
+package org.apache.asterix.external.classad.object.pool;
-import org.apache.asterix.external.input.stream.AInputStream;
-import org.apache.asterix.external.util.FeedLogManager;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.asterix.om.base.AMutableDouble;
-public interface IInputStreamProvider {
- public AInputStream getInputStream() throws HyracksDataException;
+public class DoublePool extends Pool<AMutableDouble> {
- public void setFeedLogManager(FeedLogManager feedLogManager);
+ @Override
+ public AMutableDouble newInstance() {
+ return new AMutableDouble(0);
+ }
+
+ @Override
+ protected void reset(AMutableDouble obj) {
+ obj.setValue(0);
+ }
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProviderFactory.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/EvalStatePool.java
similarity index 63%
copy from asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProviderFactory.java
copy to asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/EvalStatePool.java
index f52f7d3..a047cc9 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProviderFactory.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/EvalStatePool.java
@@ -16,18 +16,26 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.external.api;
+package org.apache.asterix.external.classad.object.pool;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.asterix.external.classad.EvalState;
-public interface IInputStreamProviderFactory extends IExternalDataSourceFactory {
+public class EvalStatePool extends Pool<EvalState> {
- public IInputStreamProvider createInputStreamProvider(IHyracksTaskContext ctx, int partition)
- throws HyracksDataException;
+ public final ClassAdObjectPool objectPool;
+
+ public EvalStatePool(ClassAdObjectPool objectPool) {
+ this.objectPool = objectPool;
+ }
@Override
- public default DataSourceType getDataSourceType() {
- return DataSourceType.STREAM;
+ public EvalState newInstance() {
+ return new EvalState(objectPool);
}
+
+ @Override
+ protected void reset(EvalState obj) {
+ obj.reset();
+ }
+
}
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ExprHolderPool.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ExprHolderPool.java
index b34e863..156f012 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ExprHolderPool.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ExprHolderPool.java
@@ -21,9 +21,16 @@
import org.apache.asterix.external.classad.ExprTreeHolder;
public class ExprHolderPool extends Pool<ExprTreeHolder> {
+
+ private final ClassAdObjectPool objectPool;
+
+ public ExprHolderPool(ClassAdObjectPool objectPool) {
+ this.objectPool = objectPool;
+ }
+
@Override
public ExprTreeHolder newInstance() {
- return new ExprTreeHolder();
+ return new ExprTreeHolder(objectPool);
}
@Override
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ExprListPool.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ExprListPool.java
index 1f2958a..66f5329 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ExprListPool.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ExprListPool.java
@@ -22,9 +22,15 @@
public class ExprListPool extends Pool<ExprList> {
+ private final ClassAdObjectPool objectPool;
+
+ public ExprListPool(ClassAdObjectPool objectPool) {
+ this.objectPool = objectPool;
+ }
+
@Override
public ExprList newInstance() {
- return new ExprList();
+ return new ExprList(objectPool);
}
@Override
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProviderFactory.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/FunctionCallPool.java
similarity index 62%
copy from asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProviderFactory.java
copy to asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/FunctionCallPool.java
index f52f7d3..9281ed7 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProviderFactory.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/FunctionCallPool.java
@@ -16,18 +16,26 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.external.api;
+package org.apache.asterix.external.classad.object.pool;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.asterix.external.classad.FunctionCall;
-public interface IInputStreamProviderFactory extends IExternalDataSourceFactory {
+public class FunctionCallPool extends Pool<FunctionCall> {
- public IInputStreamProvider createInputStreamProvider(IHyracksTaskContext ctx, int partition)
- throws HyracksDataException;
+ private final ClassAdObjectPool objectPool;
+
+ public FunctionCallPool(ClassAdObjectPool objectPool) {
+ this.objectPool = objectPool;
+ }
@Override
- public default DataSourceType getDataSourceType() {
- return DataSourceType.STREAM;
+ public FunctionCall newInstance() {
+ return new FunctionCall(objectPool);
}
+
+ @Override
+ protected void reset(FunctionCall obj) {
+ obj.reset();
+ }
+
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/HashMapPool.java
similarity index 66%
copy from asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java
copy to asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/HashMapPool.java
index b10452e..b0365f3 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/HashMapPool.java
@@ -16,14 +16,19 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.external.api;
+package org.apache.asterix.external.classad.object.pool;
-import org.apache.asterix.external.input.stream.AInputStream;
-import org.apache.asterix.external.util.FeedLogManager;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
+import java.util.HashMap;
-public interface IInputStreamProvider {
- public AInputStream getInputStream() throws HyracksDataException;
+public class HashMapPool<K, V> extends Pool<HashMap<K, V>> {
- public void setFeedLogManager(FeedLogManager feedLogManager);
+ @Override
+ public HashMap<K, V> newInstance() {
+ return new HashMap<K, V>();
+ }
+
+ @Override
+ protected void reset(HashMap<K, V> obj) {
+ obj.clear();
+ }
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/Int32Pool.java
similarity index 66%
copy from asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java
copy to asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/Int32Pool.java
index b10452e..504f147 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/Int32Pool.java
@@ -16,14 +16,20 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.external.api;
+package org.apache.asterix.external.classad.object.pool;
-import org.apache.asterix.external.input.stream.AInputStream;
-import org.apache.asterix.external.util.FeedLogManager;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.asterix.om.base.AMutableInt32;
-public interface IInputStreamProvider {
- public AInputStream getInputStream() throws HyracksDataException;
+public class Int32Pool extends Pool<AMutableInt32> {
- public void setFeedLogManager(FeedLogManager feedLogManager);
+ @Override
+ public AMutableInt32 newInstance() {
+ return new AMutableInt32(0);
+ }
+
+ @Override
+ protected void reset(AMutableInt32 obj) {
+ obj.setValue(0);
+ }
+
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/Int64Pool.java
similarity index 66%
rename from asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java
rename to asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/Int64Pool.java
index b10452e..f86b57a 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/Int64Pool.java
@@ -16,14 +16,19 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.external.api;
+package org.apache.asterix.external.classad.object.pool;
-import org.apache.asterix.external.input.stream.AInputStream;
-import org.apache.asterix.external.util.FeedLogManager;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.asterix.om.base.AMutableInt64;
-public interface IInputStreamProvider {
- public AInputStream getInputStream() throws HyracksDataException;
+public class Int64Pool extends Pool<AMutableInt64> {
- public void setFeedLogManager(FeedLogManager feedLogManager);
+ @Override
+ public AMutableInt64 newInstance() {
+ return new AMutableInt64(0);
+ }
+
+ @Override
+ protected void reset(AMutableInt64 obj) {
+ }
+
}
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/LiteralPool.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/LiteralPool.java
index 23d8dae..2a85a4a 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/LiteralPool.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/LiteralPool.java
@@ -21,9 +21,15 @@
import org.apache.asterix.external.classad.Literal;
public class LiteralPool extends Pool<Literal> {
+ private final ClassAdObjectPool objectPool;
+
+ public LiteralPool(ClassAdObjectPool objectPool) {
+ this.objectPool = objectPool;
+ }
+
@Override
public Literal newInstance() {
- return new Literal();
+ return new Literal(objectPool);
}
@Override
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/MutableBooleanPool.java
similarity index 66%
copy from asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java
copy to asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/MutableBooleanPool.java
index b10452e..8713272 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/MutableBooleanPool.java
@@ -16,14 +16,20 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.external.api;
+package org.apache.asterix.external.classad.object.pool;
-import org.apache.asterix.external.input.stream.AInputStream;
-import org.apache.asterix.external.util.FeedLogManager;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.commons.lang3.mutable.MutableBoolean;
-public interface IInputStreamProvider {
- public AInputStream getInputStream() throws HyracksDataException;
+public class MutableBooleanPool extends Pool<MutableBoolean> {
- public void setFeedLogManager(FeedLogManager feedLogManager);
+ @Override
+ public MutableBoolean newInstance() {
+ return new MutableBoolean();
+ }
+
+ @Override
+ protected void reset(MutableBoolean obj) {
+ obj.setValue(false);
+ }
+
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProviderFactory.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/MutableNumberFactorPool.java
similarity index 63%
copy from asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProviderFactory.java
copy to asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/MutableNumberFactorPool.java
index f52f7d3..9d452d5 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProviderFactory.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/MutableNumberFactorPool.java
@@ -16,18 +16,21 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.external.api;
+package org.apache.asterix.external.classad.object.pool;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.asterix.external.classad.AMutableNumberFactor;
+import org.apache.asterix.external.classad.Value.NumberFactor;
-public interface IInputStreamProviderFactory extends IExternalDataSourceFactory {
-
- public IInputStreamProvider createInputStreamProvider(IHyracksTaskContext ctx, int partition)
- throws HyracksDataException;
+public class MutableNumberFactorPool extends Pool<AMutableNumberFactor> {
@Override
- public default DataSourceType getDataSourceType() {
- return DataSourceType.STREAM;
+ public AMutableNumberFactor newInstance() {
+ return new AMutableNumberFactor();
}
+
+ @Override
+ protected void reset(AMutableNumberFactor obj) {
+ obj.setFactor(NumberFactor.NO_FACTOR);
+ }
+
}
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/OperationPool.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/OperationPool.java
index 062d870..1607f6e 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/OperationPool.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/OperationPool.java
@@ -22,9 +22,15 @@
public class OperationPool extends Pool<Operation> {
+ private final ClassAdObjectPool objectPool;
+
+ public OperationPool(ClassAdObjectPool objectPool) {
+ this.objectPool = objectPool;
+ }
+
@Override
public Operation newInstance() {
- return new Operation();
+ return new Operation(objectPool);
}
@Override
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProviderFactory.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/PrettyPrintPool.java
similarity index 63%
copy from asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProviderFactory.java
copy to asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/PrettyPrintPool.java
index f52f7d3..d38c5d6 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProviderFactory.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/PrettyPrintPool.java
@@ -16,18 +16,25 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.external.api;
+package org.apache.asterix.external.classad.object.pool;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.asterix.external.classad.PrettyPrint;
-public interface IInputStreamProviderFactory extends IExternalDataSourceFactory {
+public class PrettyPrintPool extends Pool<PrettyPrint> {
- public IInputStreamProvider createInputStreamProvider(IHyracksTaskContext ctx, int partition)
- throws HyracksDataException;
+ private final ClassAdObjectPool objectPool;
+
+ public PrettyPrintPool(ClassAdObjectPool objectPool) {
+ this.objectPool = objectPool;
+ }
@Override
- public default DataSourceType getDataSourceType() {
- return DataSourceType.STREAM;
+ public PrettyPrint newInstance() {
+ return new PrettyPrint(objectPool);
}
+
+ @Override
+ protected void reset(PrettyPrint obj) {
+ }
+
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/StringArrayListPool.java
similarity index 66%
copy from asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java
copy to asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/StringArrayListPool.java
index b10452e..0ec687c 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/StringArrayListPool.java
@@ -16,14 +16,20 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.external.api;
+package org.apache.asterix.external.classad.object.pool;
-import org.apache.asterix.external.input.stream.AInputStream;
-import org.apache.asterix.external.util.FeedLogManager;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
+import java.util.ArrayList;
-public interface IInputStreamProvider {
- public AInputStream getInputStream() throws HyracksDataException;
+public class StringArrayListPool extends Pool<ArrayList<String>> {
- public void setFeedLogManager(FeedLogManager feedLogManager);
+ @Override
+ public ArrayList<String> newInstance() {
+ return new ArrayList<String>();
+ }
+
+ @Override
+ protected void reset(ArrayList<String> obj) {
+ obj.clear();
+ }
+
}
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/TreeSetPool.java
similarity index 66%
copy from asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java
copy to asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/TreeSetPool.java
index b10452e..d09c170 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/TreeSetPool.java
@@ -16,14 +16,20 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.external.api;
+package org.apache.asterix.external.classad.object.pool;
-import org.apache.asterix.external.input.stream.AInputStream;
-import org.apache.asterix.external.util.FeedLogManager;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
+import java.util.TreeSet;
-public interface IInputStreamProvider {
- public AInputStream getInputStream() throws HyracksDataException;
+public class TreeSetPool<T> extends Pool<TreeSet<T>> {
- public void setFeedLogManager(FeedLogManager feedLogManager);
+ @Override
+ public TreeSet<T> newInstance() {
+ return new TreeSet<T>();
+ }
+
+ @Override
+ protected void reset(TreeSet<T> obj) {
+ obj.clear();
+ }
+
}
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ValuePool.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ValuePool.java
index 698e947..9ad4424 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ValuePool.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ValuePool.java
@@ -21,13 +21,20 @@
import org.apache.asterix.external.classad.Value;
public class ValuePool extends Pool<Value> {
+
+ private final ClassAdObjectPool objectPool;
+
+ public ValuePool(ClassAdObjectPool objectPool) {
+ this.objectPool = objectPool;
+ }
+
@Override
public Value newInstance() {
- return new Value();
+ return new Value(objectPool);
}
@Override
protected void reset(Value obj) {
- obj.clear();
+ obj.reset();
}
}
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdFunctionalTest.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdFunctionalTest.java
index 09e8001..41aa7c7 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdFunctionalTest.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdFunctionalTest.java
@@ -20,6 +20,8 @@
import java.net.URLDecoder;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
+
import junit.framework.Test;
import junit.framework.TestCase;
import junit.framework.TestSuite;
@@ -48,9 +50,10 @@
public void testApp() {
try {
- String[] args = { "", "-d", "-v", URLDecoder.decode(getClass().getResource("/functional_tests.txt").getPath(), "UTF-8") };
- FunctionalTester.test(args.length, args);
- } catch (Exception e) {
+ String[] args = { "", "-d", "-v",
+ URLDecoder.decode(getClass().getResource("/functional_tests.txt").getPath(), "UTF-8") };
+ FunctionalTester.test(args.length, args, new ClassAdObjectPool());
+ } catch (Throwable e) {
e.printStackTrace();
assertTrue(false);
}
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdParserTest.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdParserTest.java
index d65fecc..ab393ba 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdParserTest.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdParserTest.java
@@ -24,12 +24,13 @@
import java.nio.file.Files;
import java.nio.file.Paths;
+import org.apache.asterix.external.classad.ClassAd;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
+
import junit.framework.Test;
import junit.framework.TestCase;
import junit.framework.TestSuite;
-import org.apache.asterix.external.classad.ClassAd;
-
public class ClassAdParserTest extends TestCase {
/**
* Create the test case
@@ -54,18 +55,21 @@
public void test() {
try {
// test here
- ClassAd pAd = new ClassAd();
+ ClassAdObjectPool objectPool = new ClassAdObjectPool();
+ ClassAd pAd = new ClassAd(objectPool);
String szInput;
String[] files = new String[] { "/testdata.txt" };
BufferedReader infile = null;
for (String path : files) {
- infile = Files.newBufferedReader(Paths.get(URLDecoder.decode(getClass().getResource(path).getPath(), "UTF-8")),
+ infile = Files.newBufferedReader(
+ Paths.get(URLDecoder.decode(getClass().getResource(path).getPath(), "UTF-8")),
StandardCharsets.UTF_8);
szInput = infile.readLine();
while (szInput != null) {
if (szInput.trim().length() == 0) {
// ClassAdChain completed
pAd.clear();
+ objectPool.reset();
} else if (!pAd.insert(szInput)) {
// Problem
System.out.println("BARFED ON:" + szInput);
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdToADMTest.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdToADMTest.java
index 57b0dcd..d822310 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdToADMTest.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdToADMTest.java
@@ -28,9 +28,12 @@
import org.apache.asterix.external.classad.ClassAd;
import org.apache.asterix.external.classad.ExprTree;
import org.apache.asterix.external.classad.Value;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.asterix.external.input.record.reader.stream.SemiStructuredRecordReader;
-import org.apache.asterix.external.input.stream.LocalFileSystemInputStream;
+import org.apache.asterix.external.input.stream.LocalFSInputStream;
import org.apache.asterix.external.library.ClassAdParser;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.dataflow.std.file.FileSplit;
import junit.framework.Test;
import junit.framework.TestCase;
@@ -60,17 +63,20 @@
public void test() {
try {
// test here
- ClassAd pAd = new ClassAd();
+ ClassAdObjectPool objectPool = new ClassAdObjectPool();
+ ClassAd pAd = new ClassAd(objectPool);
String[] files = new String[] { "/jobads.txt" };
- ClassAdParser parser = new ClassAdParser(null, false, true, false, null, null, null);
+ ClassAdParser parser = new ClassAdParser(objectPool);
CharArrayLexerSource lexerSource = new CharArrayLexerSource();
for (String path : files) {
- LocalFileSystemInputStream in = new LocalFileSystemInputStream(
- Paths.get(getClass().getResource(path).toURI()), null, false);
+ LocalFSInputStream in = new LocalFSInputStream(
+ new FileSplit[] { new FileSplit("",
+ new FileReference(Paths.get(getClass().getResource(path).toURI()).toFile())) },
+ null, null, 0, null, false);
SemiStructuredRecordReader recordReader = new SemiStructuredRecordReader(in, null, "[", "]");
- Value val = new Value();
+ Value val = new Value(objectPool);
while (recordReader.hasNext()) {
- val.clear();
+ val.reset();
IRawRecord<char[]> record = recordReader.next();
lexerSource.setNewSource(record.get());
parser.setLexerSource(lexerSource);
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdUnitTest.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdUnitTest.java
index b9f32f5..32ce485 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdUnitTest.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdUnitTest.java
@@ -18,7 +18,7 @@
*/
package org.apache.asterix.external.classad.test;
-import java.io.IOException;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import junit.framework.Test;
import junit.framework.TestCase;
@@ -51,8 +51,9 @@
public void test() {
String[] args = { "", "-d", "-vv", "-all" };
try {
- ClassAdUnitTester.test(args.length, args);
- } catch (IOException e) {
+ ClassAdUnitTester.test(args.length, args, new ClassAdObjectPool());
+ } catch (Throwable e) {
+ e.printStackTrace();
assertTrue(false);
}
assertTrue(true);
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdUnitTester.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdUnitTester.java
index cec9299..e3c9bf7 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdUnitTester.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdUnitTester.java
@@ -32,6 +32,7 @@
import org.apache.asterix.external.classad.Util;
import org.apache.asterix.external.classad.Value;
import org.apache.asterix.external.classad.Value.ValueType;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.asterix.external.library.ClassAdParser;
import org.apache.asterix.om.base.AMutableDouble;
import org.apache.asterix.om.base.AMutableInt32;
@@ -200,7 +201,7 @@
*
* @throws IOException
*********************************************************************/
- public static boolean test(int argc, String[] argv) throws IOException {
+ public static boolean test(int argc, String[] argv, ClassAdObjectPool objectPool) throws IOException {
AMutableInt32 numberOfErrors = new AMutableInt32(0);
AMutableInt32 numberOfTests = new AMutableInt32(0);
boolean have_errors;
@@ -213,17 +214,22 @@
/* ----- Run tests ----- */
if (parameters.checkAll || parameters.checkParsing) {
- testParsing(parameters, results);
+ testParsing(parameters, results, objectPool);
}
if (parameters.checkAll || parameters.checkClassad) {
- testClassad(parameters, results);
+ try {
+ testClassad(parameters, results, objectPool);
+ } catch (Throwable th) {
+ th.printStackTrace();
+ throw th;
+ }
}
if (parameters.checkAll || parameters.checkExprlist) {
- testExprList(parameters, results);
+ testExprList(parameters, results, objectPool);
}
if (parameters.checkAll || parameters.checkValue) {
- testValue(parameters, results);
+ testValue(parameters, results, objectPool);
}
if (parameters.checkAll || parameters.checkLiteral) {
}
@@ -255,10 +261,11 @@
}
public static void test(String name, boolean test, String testLine, Results results) {
- if (test)
+ if (test) {
results.AddSuccessfulTest(name, testLine);
- else
+ } else {
results.AddFailedTest(name, testLine);
+ }
}
public static void test(String name, boolean test, Results results) {
@@ -272,8 +279,9 @@
*
* @throws IOException
*********************************************************************/
- public static void testParsing(Parameters parameters, Results results) throws IOException {
- ClassAdParser parser = new ClassAdParser(null, false, true, false, null, null, null);
+ public static void testParsing(Parameters parameters, Results results, ClassAdObjectPool objectPool)
+ throws IOException {
+ ClassAdParser parser = new ClassAdParser(objectPool);
ExprTree tree;
// My goal is to ensure that these expressions don't crash
@@ -312,23 +320,25 @@
/*********************************************************************
* Function: test_classad
* Purpose: Test the ClassAd class.
+ * @param objectPool
*
* @throws IOException
*********************************************************************/
- public static void testClassad(Parameters parameters, Results results) throws IOException {
- ClassAdParser parser = new ClassAdParser(null, false, true, false, null, null, null);
+ public static void testClassad(Parameters parameters, Results results, ClassAdObjectPool objectPool)
+ throws IOException {
+ ClassAdParser parser = new ClassAdParser(objectPool);
boolean haveAttribute;
boolean success;
System.out.println("Testing the ClassAd class...");
String input_basic = "[ A = 3; B = 4.0; C = \"babyzilla\"; D = true; E = {1}; F = [ AA = 3; ]; G =\"deleteme\";]";
- ClassAd basic = new ClassAd();
+ ClassAd basic = new ClassAd(objectPool);
AMutableInt64 i = new AMutableInt64(0);
MutableBoolean b = new MutableBoolean();
AMutableDouble r = new AMutableDouble(0);
AMutableCharArrayString s = new AMutableCharArrayString();
- ClassAd c = new ClassAd();
+ ClassAd c = new ClassAd(objectPool);
// ExprList *l;
basic = parser.parseClassAd(input_basic);
@@ -426,8 +436,8 @@
/* ----- Test Parsing multiple ClassAds ----- */
String twoClassads = "[ a = 3; ][ b = 4; ]";
- ClassAd classad1 = new ClassAd();
- ClassAd classad2 = new ClassAd();
+ ClassAd classad1 = new ClassAd(objectPool);
+ ClassAd classad2 = new ClassAd(objectPool);
AMutableInt32 offset = new AMutableInt32(0);
parser.parseClassAd(twoClassads, classad1, offset);
@@ -439,7 +449,7 @@
/* ----- Test chained ClassAds ----- */
// classad1 and classad2 from above test are used.
- ClassAd classad3 = new ClassAd();
+ ClassAd classad3 = new ClassAd(objectPool);
classad1.chainToAd(classad2);
test("classad1's parent is classad2", classad1.getChainedParentAd().equals(classad2), "Test chained ClassAds 1",
@@ -488,7 +498,8 @@
*
* @throws IOException
*********************************************************************/
- public static void testExprList(Parameters parameters, Results results) throws IOException {
+ public static void testExprList(Parameters parameters, Results results, ClassAdObjectPool objectPool)
+ throws IOException {
System.out.println("Testing the ExprList class...");
Literal literal10;
@@ -506,17 +517,17 @@
ExprList list2Copy;
/* ----- Setup Literals, the vectors, then ExprLists ----- */
- literal10 = Literal.createReal("1.0");
- literal20 = Literal.createReal("2.0");
- literal21 = Literal.createReal("2.1");
+ literal10 = Literal.createReal("1.0", objectPool);
+ literal20 = Literal.createReal("2.0", objectPool);
+ literal21 = Literal.createReal("2.1", objectPool);
vector1.add(literal10);
vector2.add(literal20);
vector2.add(literal21);
- list0 = new ExprList();
- list1 = new ExprList(vector1);
- list2 = new ExprList(vector2);
+ list0 = new ExprList(objectPool);
+ list1 = new ExprList(vector1, objectPool);
+ list2 = new ExprList(vector2, objectPool);
/* ----- Did the lists get made? ----- */
test("Made list 0", (list0 != null), "Did the lists get made? 0", results);
@@ -558,7 +569,7 @@
/* ----- Test adding and deleting from a list ----- */
Literal add;
- add = Literal.createReal("2.2");
+ add = Literal.createReal("2.2", objectPool);
if (list2Copy != null) {
list2Copy.insert(add);
@@ -574,11 +585,11 @@
/* ----- Test an ExprList bug that Nate Mueller found ----- */
ClassAd classad;
- ClassAdParser parser = new ClassAdParser(null, false, true, false, null, null, null);
+ ClassAdParser parser = new ClassAdParser(objectPool);
MutableBoolean b = new MutableBoolean();
boolean haveAttribute;
boolean canEvaluate;
- Value value = new Value();
+ Value value = new Value(objectPool);
String listClassadText = "[foo = 3; have_foo = member(foo, {1, 2, 3});]";
classad = parser.parseClassAd(listClassadText);
@@ -598,8 +609,9 @@
*
* @throws HyracksDataException
*********************************************************************/
- public static void testValue(Parameters parameters, Results results) throws HyracksDataException {
- Value v = new Value();
+ public static void testValue(Parameters parameters, Results results, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value v = new Value(objectPool);
boolean isExpectedType;
System.out.println("Testing the Value class...");
test("New value is undefined", (v.isUndefinedValue()), "test_value 1", results);
@@ -659,25 +671,25 @@
test("Relative time is 10", (10 == rt.getRelativeTime()), results);
test("GetType gives RELATIVE_TIME_VALUE", (v.getType() == ValueType.RELATIVE_TIME_VALUE), results);
- ExprList l = new ExprList();
- ExprList ll = new ExprList();
+ ExprList l = new ExprList(objectPool);
+ ExprList ll = new ExprList(objectPool);
v.setListValue(l);
isExpectedType = v.isListValue(ll);
test("Value is list value", isExpectedType, results);
test("List value is correct", l.equals(ll), results);
test("GetType gives LIST_VALUE", (v.getType() == ValueType.LIST_VALUE), results);
- ExprList sl = new ExprList(true);
- ll = new ExprList(true);
+ ExprList sl = new ExprList(true, objectPool);
+ ll = new ExprList(true, objectPool);
v.setListValue(sl);
isExpectedType = v.isListValue(ll);
test("Value is list value", isExpectedType, results);
test("List value is correct", sl.equals(ll), results);
test("GetType gives SLIST_VALUE", (v.getType() == ValueType.SLIST_VALUE), results);
- ClassAd c = new ClassAd();
+ ClassAd c = new ClassAd(objectPool);
c.insertAttr("test_int", 10);
- ClassAd cc = new ClassAd();
+ ClassAd cc = new ClassAd(objectPool);
v.setClassAdValue(c);
isExpectedType = v.isClassAdValue(cc);
test("Value is ClassAd value", isExpectedType, results);
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/FunctionalTester.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/FunctionalTester.java
index c9ce47c..75b2800 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/FunctionalTester.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/FunctionalTester.java
@@ -38,6 +38,7 @@
import org.apache.asterix.external.classad.PrettyPrint;
import org.apache.asterix.external.classad.StringLexerSource;
import org.apache.asterix.external.classad.Value;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.asterix.external.library.ClassAdParser;
import org.apache.asterix.om.base.AMutableString;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -54,15 +55,15 @@
public ExprTreeHolder tree;
public Value value;
- public Variable(String name, ExprTree tree) {
+ public Variable(String name, ExprTree tree, ClassAdObjectPool objectPool) {
this.name = name;
- this.tree = new ExprTreeHolder(tree);
+ this.tree = new ExprTreeHolder(tree, objectPool);
this.isTree = true;
}
- public Variable() {
+ public Variable(ClassAdObjectPool objectPool) {
this.name = null;
- this.tree = new ExprTreeHolder(null);
+ this.tree = new ExprTreeHolder(objectPool);
this.isTree = false;
}
@@ -73,8 +74,9 @@
this.tree = null;
}
- public void getStringRepresentation(AMutableCharArrayString representation) throws HyracksDataException {
- ClassAdUnParser unparser = new PrettyPrint();
+ public void getStringRepresentation(AMutableCharArrayString representation, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ ClassAdUnParser unparser = new PrettyPrint(objectPool);
if (isTree) {
unparser.unparse(representation, tree);
@@ -179,7 +181,7 @@
*
*--------------------------------------------------------------------*/
- public static int test(int argc, String[] argv) throws IOException {
+ public static int test(int argc, String[] argv, ClassAdObjectPool objectPool) throws IOException {
// here
boolean quit;
AMutableString line = new AMutableString(null);
@@ -194,10 +196,10 @@
boolean good_line;
Command command;
- good_line = replace_variables(line, state, parameters);
+ good_line = replace_variables(line, state, parameters, objectPool);
if (good_line) {
command = get_command(line, parameters);
- quit = handle_command(command, line, state, parameters);
+ quit = handle_command(command, line, state, parameters, objectPool);
}
}
print_final_state(state);
@@ -327,14 +329,14 @@
*
* @throws HyracksDataException
*********************************************************************/
- public static boolean replace_variables(AMutableString mutableLine, State state, Parameters parameters)
- throws HyracksDataException {
+ public static boolean replace_variables(AMutableString mutableLine, State state, Parameters parameters,
+ ClassAdObjectPool objectPool) throws HyracksDataException {
boolean good_line;
String error;
good_line = true;
error = "";
- Variable var = new Variable();
+ Variable var = new Variable(objectPool);
for (;;) {
int dollar;
int current_position;
@@ -366,7 +368,7 @@
error += "'";
break;
}
- var.getStringRepresentation(variable_value);
+ var.getStringRepresentation(variable_value, objectPool);
// We have to be careful with substr() because with gcc 2.96, it likes to
// assert/except if you give it values that are too large.
@@ -457,8 +459,8 @@
*
* @throws IOException
*********************************************************************/
- public static boolean handle_command(Command command, AMutableString line, State state, Parameters parameters)
- throws IOException {
+ public static boolean handle_command(Command command, AMutableString line, State state, Parameters parameters,
+ ClassAdObjectPool objectPool) throws IOException {
boolean quit = false;
switch (command) {
@@ -469,31 +471,31 @@
print_error_message("Unknown command on line", state);
break;
case cmd_Let:
- handle_let(line, state, parameters);
+ handle_let(line, state, parameters, objectPool);
break;
case cmd_Eval:
- handle_eval(line, state, parameters);
+ handle_eval(line, state, parameters, objectPool);
break;
case cmd_Print:
- handle_print(line, state, parameters);
+ handle_print(line, state, parameters, objectPool);
break;
case cmd_Same:
- handle_same(line, state, parameters);
+ handle_same(line, state, parameters, objectPool);
break;
case cmd_Sameq:
- handle_sameq(line, state, parameters);
+ handle_sameq(line, state, parameters, objectPool);
break;
case cmd_Diff:
- handle_diff(line, state, parameters);
+ handle_diff(line, state, parameters, objectPool);
break;
case cmd_Diffq:
- handle_diffq(line, state, parameters);
+ handle_diffq(line, state, parameters, objectPool);
break;
case cmd_Set:
- handle_set(line, state, parameters);
+ handle_set(line, state, parameters, objectPool);
break;
case cmd_Show:
- handle_show(line, state, parameters);
+ handle_show(line, state, parameters, objectPool);
break;
case cmd_Writexml:
// handle_writexml(line, state, parameters);
@@ -520,18 +522,19 @@
*
* @throws IOException
*********************************************************************/
- public static void handle_let(AMutableString line, State state, Parameters parameters) throws IOException {
+ public static void handle_let(AMutableString line, State state, Parameters parameters, ClassAdObjectPool objectPool)
+ throws IOException {
AMutableString variable_name = new AMutableString(null);
ExprTree tree;
Variable variable;
if (get_variable_name(line, true, variable_name, state, parameters)) {
- tree = get_expr(line, state, parameters);
+ tree = get_expr(line, state, parameters, objectPool);
if (tree != null) {
- variable = new Variable(variable_name.getStringValue(), tree);
+ variable = new Variable(variable_name.getStringValue(), tree, objectPool);
variables.put(variable_name.getStringValue(), variable);
if (parameters.interactive) {
- print_expr(tree, state, parameters);
+ print_expr(tree, state, parameters, objectPool);
}
}
}
@@ -544,16 +547,17 @@
*
* @throws IOException
*********************************************************************/
- public static void handle_eval(AMutableString line, State state, Parameters parameters) throws IOException {
+ public static void handle_eval(AMutableString line, State state, Parameters parameters,
+ ClassAdObjectPool objectPool) throws IOException {
AMutableString variable_name = new AMutableString("");
ExprTree tree;
Variable variable;
if (get_variable_name(line, true, variable_name, state, parameters)) {
- tree = get_expr(line, state, parameters);
+ tree = get_expr(line, state, parameters, objectPool);
if (tree != null) {
- Value value = new Value();
- if (!evaluate_expr(tree, value, parameters)) {
+ Value value = new Value(objectPool);
+ if (!evaluate_expr(tree, value, parameters, objectPool)) {
print_error_message("Couldn't evaluate rvalue", state);
} else {
variable = new Variable(variable_name.getStringValue(), value);
@@ -570,11 +574,12 @@
*
* @throws IOException
*********************************************************************/
- public static void handle_print(AMutableString line, State state, Parameters parameters) throws IOException {
+ public static void handle_print(AMutableString line, State state, Parameters parameters,
+ ClassAdObjectPool objectPool) throws IOException {
ExprTree tree;
- tree = get_expr(line, state, parameters);
+ tree = get_expr(line, state, parameters, objectPool);
if (tree != null) {
- print_expr(tree, state, parameters);
+ print_expr(tree, state, parameters, objectPool);
}
}
@@ -584,25 +589,26 @@
*
* @throws IOException
*********************************************************************/
- public static void handle_same(AMutableString line, State state, Parameters parameters) throws IOException {
- ExprTreeHolder tree = new ExprTreeHolder();
- ExprTreeHolder tree2 = new ExprTreeHolder();
- Value value1 = new Value();
- Value value2 = new Value();
+ public static void handle_same(AMutableString line, State state, Parameters parameters,
+ ClassAdObjectPool objectPool) throws IOException {
+ ExprTreeHolder tree = new ExprTreeHolder(objectPool);
+ ExprTreeHolder tree2 = new ExprTreeHolder(objectPool);
+ Value value1 = new Value(objectPool);
+ Value value2 = new Value(objectPool);
try {
- get_two_exprs(line, tree, tree2, state, parameters);
+ get_two_exprs(line, tree, tree2, state, parameters, objectPool);
if (tree.getInnerTree() != null || tree2.getInnerTree() != null) {
if (parameters.debug) {
System.out.println("Sameeval has two trees:");
System.out.print(" ");
- print_expr(tree, state, parameters);
+ print_expr(tree, state, parameters, objectPool);
System.out.print(" ");
- print_expr(tree2, state, parameters);
+ print_expr(tree2, state, parameters, objectPool);
}
- if (!evaluate_expr(tree, value1, parameters)) {
+ if (!evaluate_expr(tree, value1, parameters, objectPool)) {
print_error_message("Couldn't evaluate first expression.\n", state);
- } else if (!evaluate_expr(tree2, value2, parameters)) {
+ } else if (!evaluate_expr(tree2, value2, parameters, objectPool)) {
print_error_message("Couldn't evaluate second expressions.\n", state);
} else if (!value1.sameAs(value2)) {
print_error_message("the expressions are different.", state);
@@ -624,14 +630,16 @@
/*********************************************************************
* Function: handle_sameq
* Purpose:
+ * @param objectPool
*
* @throws IOException
*********************************************************************/
- public static void handle_sameq(AMutableString line, State state, Parameters parameters) throws IOException {
- ExprTreeHolder tree = new ExprTreeHolder();
- ExprTreeHolder tree2 = new ExprTreeHolder();
+ public static void handle_sameq(AMutableString line, State state, Parameters parameters,
+ ClassAdObjectPool objectPool) throws IOException {
+ ExprTreeHolder tree = new ExprTreeHolder(objectPool);
+ ExprTreeHolder tree2 = new ExprTreeHolder(objectPool);
- get_two_exprs(line, tree, tree2, state, parameters);
+ get_two_exprs(line, tree, tree2, state, parameters, objectPool);
if (tree.getInnerTree() != null || tree2.getInnerTree() != null) {
if (!tree.sameAs(tree2)) {
print_error_message("the expressions are different.", state);
@@ -646,17 +654,18 @@
*
* @throws IOException
*********************************************************************/
- public static void handle_diff(AMutableString line, State state, Parameters parameters) throws IOException {
- ExprTreeHolder tree = new ExprTreeHolder();
- ExprTreeHolder tree2 = new ExprTreeHolder();
- Value value1 = new Value();
- Value value2 = new Value();
+ public static void handle_diff(AMutableString line, State state, Parameters parameters,
+ ClassAdObjectPool objectPool) throws IOException {
+ ExprTreeHolder tree = new ExprTreeHolder(objectPool);
+ ExprTreeHolder tree2 = new ExprTreeHolder(objectPool);
+ Value value1 = new Value(objectPool);
+ Value value2 = new Value(objectPool);
- get_two_exprs(line, tree, tree2, state, parameters);
+ get_two_exprs(line, tree, tree2, state, parameters, objectPool);
if (tree.getInnerTree() != null || tree2.getInnerTree() != null) {
- if (!evaluate_expr(tree, value1, parameters)) {
+ if (!evaluate_expr(tree, value1, parameters, objectPool)) {
print_error_message("Couldn't evaluate first expression.\n", state);
- } else if (!evaluate_expr(tree2, value2, parameters)) {
+ } else if (!evaluate_expr(tree2, value2, parameters, objectPool)) {
print_error_message("Couldn't evaluate second expressions.\n", state);
} else if (value1.sameAs(value2)) {
print_error_message("the expressions are the same.", state);
@@ -669,14 +678,16 @@
/*********************************************************************
* Function: handle_diffq
* Purpose:
+ * @param objectPool
*
* @throws IOException
*********************************************************************/
- public static void handle_diffq(AMutableString line, State state, Parameters parameters) throws IOException {
- ExprTreeHolder tree = new ExprTreeHolder();
- ExprTreeHolder tree2 = new ExprTreeHolder();
+ public static void handle_diffq(AMutableString line, State state, Parameters parameters,
+ ClassAdObjectPool objectPool) throws IOException {
+ ExprTreeHolder tree = new ExprTreeHolder(objectPool);
+ ExprTreeHolder tree2 = new ExprTreeHolder(objectPool);
- get_two_exprs(line, tree, tree2, state, parameters);
+ get_two_exprs(line, tree, tree2, state, parameters, objectPool);
if (tree.getInnerTree() != null || tree2.getInnerTree() != null) {
if (tree.sameAs(tree2)) {
print_error_message("the expressions are the same.", state);
@@ -688,8 +699,10 @@
/*********************************************************************
* Function: handle_set
* Purpose:
+ * @param objectPool
*********************************************************************/
- public static void handle_set(AMutableString line, State state, Parameters parameters) {
+ public static void handle_set(AMutableString line, State state, Parameters parameters,
+ ClassAdObjectPool objectPool) {
AMutableString option_name = new AMutableString(null);
AMutableString option_value = new AMutableString(null);
@@ -718,8 +731,10 @@
/*********************************************************************
* Function: handle_show
* Purpose:
+ * @param objectPool
*********************************************************************/
- public static void handle_show(AMutableString line, State state, Parameters parameters) {
+ public static void handle_show(AMutableString line, State state, Parameters parameters,
+ ClassAdObjectPool objectPool) {
AMutableString option_name = new AMutableString(null);
if (get_variable_name(line, false, option_name, state, parameters)) {
@@ -906,10 +921,11 @@
*
* @throws IOException
*********************************************************************/
- public static ExprTree get_expr(AMutableString line, State state, Parameters parameters) throws IOException {
+ public static ExprTree get_expr(AMutableString line, State state, Parameters parameters,
+ ClassAdObjectPool objectPool) throws IOException {
int offset;
ExprTree tree;
- ClassAdParser parser = new ClassAdParser(null, false, true, false, null, null, null);
+ ClassAdParser parser = new ClassAdParser(objectPool);
StringLexerSource lexer_source = new StringLexerSource(line.getStringValue());
tree = parser.parseExpression(lexer_source, false);
@@ -930,9 +946,9 @@
* @throws IOException
*********************************************************************/
public static void get_two_exprs(AMutableString line, ExprTreeHolder tree1, ExprTreeHolder tree2, State state,
- Parameters parameters) throws IOException {
+ Parameters parameters, ClassAdObjectPool objectPool) throws IOException {
int offset;
- ClassAdParser parser = new ClassAdParser(null, false, true, false, null, null, null);
+ ClassAdParser parser = new ClassAdParser(objectPool);
StringLexerSource lexer_source = new StringLexerSource(line.getStringValue());
tree1.setInnerTree(parser.parseExpression(lexer_source, false));
@@ -943,7 +959,7 @@
} else {
if (parameters.debug) {
System.out.print("# Tree1: ");
- print_expr(tree1, state, parameters);
+ print_expr(tree1, state, parameters, objectPool);
}
if (parser.peekToken() != TokenType.LEX_COMMA) {
@@ -961,9 +977,9 @@
throw new IOException();
} else if (parameters.debug) {
System.out.print("# Tree2: ");
- print_expr(tree2, state, parameters);
+ print_expr(tree2, state, parameters, objectPool);
System.out.print("# Tree1: ");
- print_expr(tree1, state, parameters);
+ print_expr(tree1, state, parameters, objectPool);
System.out.println();
}
}
@@ -978,14 +994,15 @@
*
* @throws HyracksDataException
*********************************************************************/
- public static void print_expr(ExprTree tree, State state, Parameters parameters) throws HyracksDataException {
+ public static void print_expr(ExprTree tree, State state, Parameters parameters, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
AMutableCharArrayString output = new AMutableCharArrayString();
if (state.format == PrintFormat.print_Compact) {
- ClassAdUnParser unparser = new ClassAdUnParser();
+ ClassAdUnParser unparser = new ClassAdUnParser(objectPool);
unparser.unparse(output, tree);
} else if (state.format == PrintFormat.print_Pretty) {
- PrettyPrint unparser = new PrettyPrint();
+ PrettyPrint unparser = new PrettyPrint(objectPool);
unparser.unparse(output, tree);
} else if (state.format == PrintFormat.print_XML) {/*
* ClassAdXMLUnParser unparser = new
@@ -1009,8 +1026,9 @@
*
* @throws HyracksDataException
*********************************************************************/
- public static boolean evaluate_expr(ExprTree tree, Value value, Parameters parameters) throws HyracksDataException {
- ClassAd classad = new ClassAd();
+ public static boolean evaluate_expr(ExprTree tree, Value value, Parameters parameters, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ ClassAd classad = new ClassAd(objectPool);
boolean success = false;;
classad.insert("internal___", tree);
success = classad.evaluateAttr("internal___", value);
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java b/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java
index d6db6c2..5fd6f21 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java
@@ -59,15 +59,7 @@
import org.apache.asterix.external.classad.TokenValue;
import org.apache.asterix.external.classad.Value;
import org.apache.asterix.external.classad.Value.NumberFactor;
-import org.apache.asterix.external.classad.object.pool.AttributeReferencePool;
-import org.apache.asterix.external.classad.object.pool.BitSetPool;
-import org.apache.asterix.external.classad.object.pool.ClassAdPool;
-import org.apache.asterix.external.classad.object.pool.ExprHolderPool;
-import org.apache.asterix.external.classad.object.pool.ExprListPool;
-import org.apache.asterix.external.classad.object.pool.LiteralPool;
-import org.apache.asterix.external.classad.object.pool.OperationPool;
-import org.apache.asterix.external.classad.object.pool.TokenValuePool;
-import org.apache.asterix.external.classad.object.pool.ValuePool;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.asterix.external.parser.AbstractDataParser;
import org.apache.asterix.om.base.ABoolean;
import org.apache.asterix.om.base.AMutableInt32;
@@ -94,16 +86,7 @@
private Lexer lexer = new Lexer();
private LexerSource currentSource = null;
private boolean isExpr = false;
- // object pools
- private final ExprHolderPool mutableExprPool = new ExprHolderPool();
- private final TokenValuePool tokenValuePool = new TokenValuePool();
- private final ClassAdPool classAdPool = new ClassAdPool();
- private final ExprListPool exprListPool = new ExprListPool();
- private final ValuePool valuePool = new ValuePool();
- private final LiteralPool literalPool = new LiteralPool();
- private final BitSetPool bitSetPool = new BitSetPool();
- private final OperationPool operationPool = new OperationPool();
- private final AttributeReferencePool attrRefPool = new AttributeReferencePool();
+ private final ClassAdObjectPool objectPool;
// asterix objects
private ARecordType recordType;
private IObjectPool<IARecordBuilder, ATypeTag> recordBuilderPool = new ListObjectPool<IARecordBuilder, ATypeTag>(
@@ -112,17 +95,22 @@
new ListBuilderFactory());
private IObjectPool<IMutableValueStorage, ATypeTag> abvsBuilderPool = new ListObjectPool<IMutableValueStorage, ATypeTag>(
new AbvsBuilderFactory());
- private ClassAd rootAd = new ClassAd(false, true);
+ private final ClassAd rootAd;
private String exprPrefix = "expr=";
private String exprSuffix = "";
private boolean evaluateExpr = true;
private String exprFieldNameSuffix = "Expr";
private boolean keepBoth = true;
- private boolean oldFormat = false;
+ private boolean oldFormat = true;
private StringLexerSource stringLexerSource = new StringLexerSource("");
public ClassAdParser(ARecordType recordType, boolean oldFormat, boolean evaluateExpr, boolean keepBoth,
- String exprPrefix, String exprSuffix, String exprFieldNameSuffix) {
+ String exprPrefix, String exprSuffix, String exprFieldNameSuffix, ClassAdObjectPool objectPool) {
+ if (objectPool == null) {
+ System.out.println();
+ }
+ this.objectPool = objectPool;
+ this.rootAd = new ClassAd(objectPool);
this.recordType = recordType;
this.currentSource = new CharArrayLexerSource();
this.recordType = recordType;
@@ -137,6 +125,15 @@
this.exprFieldNameSuffix = exprFieldNameSuffix;
}
+ public ClassAdParser(ClassAdObjectPool objectPool) {
+ if (objectPool == null) {
+ System.out.println();
+ }
+ this.objectPool = objectPool;
+ this.currentSource = new CharArrayLexerSource();
+ rootAd = null;
+ }
+
/***********************************
* AsterixDB Specific begin
*
@@ -160,8 +157,8 @@
return false;
}
tt = lexer.peekToken();
- TokenValue tv = tokenValuePool.get();
- ExprTreeHolder tree = mutableExprPool.get();
+ TokenValue tv = objectPool.tokenValuePool.get();
+ ExprTreeHolder tree = objectPool.mutableExprPool.get();
while (tt != TokenType.LEX_CLOSE_BOX) {
// Get the name of the expression
tv.reset();
@@ -200,8 +197,8 @@
// we will store a string representation of the expression
int len = lexer.getLexSource().getPosition() - positionBefore;
// add it as it is to the classAd
- Literal lit = literalPool.get();
- Value exprVal = valuePool.get();
+ Literal lit = objectPool.literalPool.get();
+ Value exprVal = objectPool.valuePool.get();
exprVal.setStringValue(exprPrefix
+ String.valueOf(lexer.getLexSource().getBuffer(), positionBefore, len) + exprSuffix);
Literal.createLiteral(lit, exprVal, NumberFactor.NO_FACTOR);
@@ -263,15 +260,7 @@
listBuilderPool.reset();
recordBuilderPool.reset();
abvsBuilderPool.reset();
- mutableExprPool.reset();
- tokenValuePool.reset();
- classAdPool.reset();
- exprListPool.reset();
- valuePool.reset();
- literalPool.reset();
- bitSetPool.reset();
- operationPool.reset();
- attrRefPool.reset();
+ objectPool.reset();
}
private ATypeTag getTargetTypeTag(ATypeTag expectedTypeTag, IAType aObjectType) throws IOException {
@@ -384,7 +373,7 @@
case EXPR_LIST_NODE:
case FN_CALL_NODE:
case OP_NODE:
- val = valuePool.get();
+ val = objectPool.valuePool.get();
if (pAd.evaluateAttr(name, val)) {
} else {
@@ -393,7 +382,7 @@
}
break;
case LITERAL_NODE:
- val = ((Literal) tree).getValue();
+ val = ((Literal) tree.getTree()).getValue();
break;
default:
throw new HyracksDataException("Unknown Expression type detected: " + tree.getKind());
@@ -574,7 +563,7 @@
}
private BitSet getBitSet() {
- return bitSetPool.get();
+ return objectPool.bitSetPool.get();
}
public static int checkNullConstraints(ARecordType recType, BitSet nulls) {
@@ -710,7 +699,7 @@
public ClassAd parseClassAd(LexerSource lexer_source, boolean full) throws IOException {
System.out.println("Don't use this call. instead, pass a mutable classad instance");
- ClassAd ad = classAdPool.get();
+ ClassAd ad = objectPool.classAdPool.get();
if (lexer.initialize(lexer_source)) {
if (!parseClassAd(ad, full)) {
return null;
@@ -804,7 +793,7 @@
*/
public ExprTree parseExpression(String buffer, boolean full) throws IOException {
stringLexerSource.setNewSource(buffer);
- ExprTreeHolder mutableExpr = mutableExprPool.get();
+ ExprTreeHolder mutableExpr = objectPool.mutableExprPool.get();
if (lexer.initialize(stringLexerSource)) {
parseExpression(mutableExpr, full);
}
@@ -816,7 +805,7 @@
}
public ExprTree parseExpression(LexerSource lexer_source, boolean full) throws IOException {
- ExprTreeHolder mutableExpr = mutableExprPool.get();
+ ExprTreeHolder mutableExpr = objectPool.mutableExprPool.get();
if (lexer.initialize(lexer_source)) {
parseExpression(mutableExpr, full);
}
@@ -827,7 +816,7 @@
if (!lexer.wasInitialized()) {
return null;
} else {
- ExprTreeHolder expr = mutableExprPool.get();
+ ExprTreeHolder expr = objectPool.mutableExprPool.get();
parseExpression(expr, false);
ExprTree innerTree = expr.getInnerTree();
return innerTree;
@@ -849,20 +838,21 @@
private boolean parseExpression(ExprTreeHolder tree, boolean full) throws IOException {
TokenType tt;
- if (!parseLogicalORExpression(tree))
+ if (!parseLogicalORExpression(tree)) {
return false;
+ }
if ((tt = lexer.peekToken()) == TokenType.LEX_QMARK) {
lexer.consumeToken();
ExprTreeHolder treeL = tree;
- ExprTreeHolder treeM = mutableExprPool.get();
- ExprTreeHolder treeR = mutableExprPool.get();
+ ExprTreeHolder treeM = objectPool.mutableExprPool.get();
+ ExprTreeHolder treeR = objectPool.mutableExprPool.get();
parseExpression(treeM);
if ((tt = lexer.consumeToken()) != TokenType.LEX_COLON) {
throw new HyracksDataException("expected LEX_COLON, but got " + Lexer.strLexToken(tt));
}
parseExpression(treeR);
if (treeL.getInnerTree() != null && treeM.getInnerTree() != null && treeR.getInnerTree() != null) {
- Operation newTree = operationPool.get();
+ Operation newTree = objectPool.operationPool.get();
Operation.createOperation(Operation.OpKind_TERNARY_OP, treeL, treeM, treeR, newTree);
tree.setInnerTree(newTree);
return (true);
@@ -882,15 +872,16 @@
// | LogicalORExpression '||' LogicalANDExpression
private boolean parseLogicalORExpression(ExprTreeHolder tree) throws IOException {
- if (!parseLogicalANDExpression(tree))
+ if (!parseLogicalANDExpression(tree)) {
return false;
+ }
while ((lexer.peekToken()) == TokenType.LEX_LOGICAL_OR) {
ExprTreeHolder treeL = tree;
- ExprTreeHolder treeR = mutableExprPool.get();
+ ExprTreeHolder treeR = objectPool.mutableExprPool.get();
lexer.consumeToken();
parseLogicalANDExpression(treeR);
if (treeL.getInnerTree() != null && treeR.getInnerTree() != null) {
- Operation newTree = operationPool.get();
+ Operation newTree = objectPool.operationPool.get();
Operation.createOperation(Operation.OpKind_LOGICAL_OR_OP, treeL, treeR, null, newTree);
tree.setInnerTree(newTree);
} else {
@@ -904,15 +895,16 @@
// LogicalANDExpression .= InclusiveORExpression
// | LogicalANDExpression '&&' InclusiveORExpression
private boolean parseLogicalANDExpression(ExprTreeHolder tree) throws IOException {
- if (!parseInclusiveORExpression(tree))
+ if (!parseInclusiveORExpression(tree)) {
return false;
+ }
while ((lexer.peekToken()) == TokenType.LEX_LOGICAL_AND) {
ExprTreeHolder treeL = tree;
- ExprTreeHolder treeR = mutableExprPool.get();
+ ExprTreeHolder treeR = objectPool.mutableExprPool.get();
lexer.consumeToken();
parseInclusiveORExpression(treeR);
if (treeL.getInnerTree() != null && treeR.getInnerTree() != null) {
- Operation newTree = operationPool.get();
+ Operation newTree = objectPool.operationPool.get();
Operation.createOperation(Operation.OpKind_LOGICAL_AND_OP, treeL, treeR, null, newTree);
tree.setInnerTree(newTree);
} else {
@@ -926,15 +918,16 @@
// InclusiveORExpression .= ExclusiveORExpression
// | InclusiveORExpression '|' ExclusiveORExpression
public boolean parseInclusiveORExpression(ExprTreeHolder tree) throws IOException {
- if (!parseExclusiveORExpression(tree))
+ if (!parseExclusiveORExpression(tree)) {
return false;
+ }
while ((lexer.peekToken()) == TokenType.LEX_BITWISE_OR) {
ExprTreeHolder treeL = tree;
- ExprTreeHolder treeR = mutableExprPool.get();
+ ExprTreeHolder treeR = objectPool.mutableExprPool.get();
lexer.consumeToken();
parseExclusiveORExpression(treeR);
if (treeL.getInnerTree() != null && treeR.getInnerTree() != null) {
- Operation newTree = operationPool.get();
+ Operation newTree = objectPool.operationPool.get();
Operation.createOperation(Operation.OpKind_BITWISE_OR_OP, treeL, treeR, null, newTree);
tree.setInnerTree(newTree);
} else {
@@ -948,15 +941,16 @@
// ExclusiveORExpression .= ANDExpression
// | ExclusiveORExpression '^' ANDExpression
private boolean parseExclusiveORExpression(ExprTreeHolder tree) throws IOException {
- if (!parseANDExpression(tree))
+ if (!parseANDExpression(tree)) {
return false;
+ }
while ((lexer.peekToken()) == TokenType.LEX_BITWISE_XOR) {
lexer.consumeToken();
ExprTreeHolder treeL = tree;
- ExprTreeHolder treeR = mutableExprPool.get();
+ ExprTreeHolder treeR = objectPool.mutableExprPool.get();
parseANDExpression(treeR);
if (treeL.getInnerTree() != null && treeR.getInnerTree() != null) {
- Operation newTree = operationPool.get();
+ Operation newTree = objectPool.operationPool.get();
Operation.createOperation(Operation.OpKind_BITWISE_XOR_OP, treeL, treeR, null, newTree);
tree.setInnerTree(newTree);
} else {
@@ -970,15 +964,16 @@
// ANDExpression .= EqualityExpression
// | ANDExpression '&' EqualityExpression
private boolean parseANDExpression(ExprTreeHolder tree) throws IOException {
- if (!parseEqualityExpression(tree))
+ if (!parseEqualityExpression(tree)) {
return false;
+ }
while ((lexer.peekToken()) == TokenType.LEX_BITWISE_AND) {
ExprTreeHolder treeL = tree;
- ExprTreeHolder treeR = mutableExprPool.get();
+ ExprTreeHolder treeR = objectPool.mutableExprPool.get();
lexer.consumeToken();
parseEqualityExpression(treeR);
if (treeL.getInnerTree() != null && treeR.getInnerTree() != null) {
- Operation newTree = operationPool.get();
+ Operation newTree = objectPool.operationPool.get();
Operation.createOperation(Operation.OpKind_BITWISE_AND_OP, treeL, treeR, null, newTree);
tree.setInnerTree(newTree);
} else {
@@ -997,13 +992,14 @@
private boolean parseEqualityExpression(ExprTreeHolder tree) throws IOException {
TokenType tt;
int op = Operation.OpKind_NO_OP;
- if (!parseRelationalExpression(tree))
+ if (!parseRelationalExpression(tree)) {
return false;
+ }
tt = lexer.peekToken();
while (tt == TokenType.LEX_EQUAL || tt == TokenType.LEX_NOT_EQUAL || tt == TokenType.LEX_META_EQUAL
|| tt == TokenType.LEX_META_NOT_EQUAL) {
ExprTreeHolder treeL = tree;
- ExprTreeHolder treeR = mutableExprPool.get();
+ ExprTreeHolder treeR = objectPool.mutableExprPool.get();
lexer.consumeToken();
parseRelationalExpression(treeR);
switch (tt) {
@@ -1023,7 +1019,7 @@
throw new HyracksDataException("ClassAd: Should not reach here");
}
if (treeL.getInnerTree() != null && treeR.getInnerTree() != null) {
- Operation newTree = operationPool.get();
+ Operation newTree = objectPool.operationPool.get();
Operation.createOperation(op, treeL, treeR, null, newTree);
tree.setInnerTree(newTree);
} else {
@@ -1042,14 +1038,15 @@
// | RelationalExpression '>=' ShiftExpression
private boolean parseRelationalExpression(ExprTreeHolder tree) throws IOException {
TokenType tt;
- if (!parseShiftExpression(tree))
+ if (!parseShiftExpression(tree)) {
return false;
+ }
tt = lexer.peekToken();
while (tt == TokenType.LEX_LESS_THAN || tt == TokenType.LEX_GREATER_THAN || tt == TokenType.LEX_LESS_OR_EQUAL
|| tt == TokenType.LEX_GREATER_OR_EQUAL) {
int op = Operation.OpKind_NO_OP;
ExprTreeHolder treeL = tree;
- ExprTreeHolder treeR = mutableExprPool.get();
+ ExprTreeHolder treeR = objectPool.mutableExprPool.get();
lexer.consumeToken();
parseShiftExpression(treeR);
switch (tt) {
@@ -1069,7 +1066,7 @@
throw new HyracksDataException("ClassAd: Should not reach here");
}
if (treeL.getInnerTree() != null && treeR.getInnerTree() != null) {
- Operation newTree = operationPool.get();
+ Operation newTree = objectPool.operationPool.get();
Operation.createOperation(op, treeL, treeR, null, newTree);
tree.setInnerTree(newTree);
} else {
@@ -1086,13 +1083,14 @@
// | ShiftExpression '>>' AdditiveExpression
// | ShiftExpression '>>>' AditiveExpression
private boolean parseShiftExpression(ExprTreeHolder tree) throws IOException {
- if (!parseAdditiveExpression(tree))
+ if (!parseAdditiveExpression(tree)) {
return false;
+ }
TokenType tt = lexer.peekToken();
while (tt == TokenType.LEX_LEFT_SHIFT || tt == TokenType.LEX_RIGHT_SHIFT || tt == TokenType.LEX_URIGHT_SHIFT) {
ExprTreeHolder treeL = tree;
- ExprTreeHolder treeR = mutableExprPool.get();
+ ExprTreeHolder treeR = objectPool.mutableExprPool.get();
int op;
lexer.consumeToken();
parseAdditiveExpression(treeR);
@@ -1112,7 +1110,7 @@
}
if (treeL.getInnerTree() != null && treeR.getInnerTree() != null) {
- Operation newTree = operationPool.get();
+ Operation newTree = objectPool.operationPool.get();
Operation.createOperation(op, treeL, treeR, null, newTree);
tree.setInnerTree(newTree);
} else {
@@ -1128,17 +1126,18 @@
// | AdditiveExpression '+' MultiplicativeExpression
// | AdditiveExpression '-' MultiplicativeExpression
private boolean parseAdditiveExpression(ExprTreeHolder tree) throws IOException {
- if (!parseMultiplicativeExpression(tree))
+ if (!parseMultiplicativeExpression(tree)) {
return false;
+ }
TokenType tt = lexer.peekToken();
while (tt == TokenType.LEX_PLUS || tt == TokenType.LEX_MINUS) {
ExprTreeHolder treeL = tree;
- ExprTreeHolder treeR = mutableExprPool.get();
+ ExprTreeHolder treeR = objectPool.mutableExprPool.get();
lexer.consumeToken();
parseMultiplicativeExpression(treeR);
if (treeL.getInnerTree() != null && treeR.getInnerTree() != null) {
- Operation newTree = operationPool.get();
+ Operation newTree = objectPool.operationPool.get();
Operation.createOperation(
(tt == TokenType.LEX_PLUS) ? Operation.OpKind_ADDITION_OP : Operation.OpKind_SUBTRACTION_OP,
treeL, treeR, null, newTree);
@@ -1157,13 +1156,14 @@
// | MultiplicativeExpression '/' UnaryExpression
// | MultiplicativeExpression '%' UnaryExpression
private boolean parseMultiplicativeExpression(ExprTreeHolder tree) throws IOException {
- if (!parseUnaryExpression(tree))
+ if (!parseUnaryExpression(tree)) {
return false;
+ }
TokenType tt = lexer.peekToken();
while (tt == TokenType.LEX_MULTIPLY || tt == TokenType.LEX_DIVIDE || tt == TokenType.LEX_MODULUS) {
ExprTreeHolder treeL = tree;
- ExprTreeHolder treeR = mutableExprPool.get();
+ ExprTreeHolder treeR = objectPool.mutableExprPool.get();
int op;
lexer.consumeToken();
parseUnaryExpression(treeR);
@@ -1182,7 +1182,7 @@
throw new HyracksDataException("ClassAd: Should not reach here");
}
if (treeL.getInnerTree() != null && treeR.getInnerTree() != null) {
- Operation newTree = operationPool.get();
+ Operation newTree = objectPool.operationPool.get();
Operation.createOperation(op, treeL, treeR, null, newTree);
tree.setInnerTree(newTree);
} else {
@@ -1202,7 +1202,7 @@
if (tt == TokenType.LEX_MINUS || tt == TokenType.LEX_PLUS || tt == TokenType.LEX_BITWISE_NOT
|| tt == TokenType.LEX_LOGICAL_NOT) {
lexer.consumeToken();
- ExprTreeHolder treeM = mutableExprPool.get();
+ ExprTreeHolder treeM = objectPool.mutableExprPool.get();
int op = Operation.OpKind_NO_OP;
parseUnaryExpression(treeM);
switch (tt) {
@@ -1222,7 +1222,7 @@
throw new HyracksDataException("ClassAd: Shouldn't Get here");
}
if (treeM.getInnerTree() != null) {
- Operation newTree = operationPool.get();
+ Operation newTree = objectPool.operationPool.get();
Operation.createOperation(op, treeM, null, null, newTree);
tree.setInnerTree(newTree);
} else {
@@ -1240,18 +1240,19 @@
// | PostfixExpression '[' Expression ']'
private boolean parsePostfixExpression(ExprTreeHolder tree) throws IOException {
TokenType tt;
- if (!parsePrimaryExpression(tree))
+ if (!parsePrimaryExpression(tree)) {
return false;
+ }
while ((tt = lexer.peekToken()) == TokenType.LEX_OPEN_BOX || tt == TokenType.LEX_SELECTION) {
ExprTreeHolder treeL = tree;
- ExprTreeHolder treeR = mutableExprPool.get();
- TokenValue tv = tokenValuePool.get();
+ ExprTreeHolder treeR = objectPool.mutableExprPool.get();
+ TokenValue tv = objectPool.tokenValuePool.get();
lexer.consumeToken();
if (tt == TokenType.LEX_OPEN_BOX) {
// subscript operation
parseExpression(treeR);
if (treeL.getInnerTree() != null && treeR.getInnerTree() != null) {
- Operation newTree = operationPool.get();
+ Operation newTree = objectPool.operationPool.get();
Operation.createOperation(Operation.OpKind_SUBSCRIPT_OP, treeL, treeR, null, newTree);
if (lexer.consumeToken() == TokenType.LEX_CLOSE_BOX) {
tree.setInnerTree(newTree);
@@ -1266,7 +1267,7 @@
throw new HyracksDataException("second argument of selector must be an " + "identifier (got"
+ String.valueOf(Lexer.strLexToken(tt)) + ")");
}
- AttributeReference newTree = attrRefPool.get();
+ AttributeReference newTree = objectPool.attrRefPool.get();
AttributeReference.createAttributeReference(treeL, tv.getStrValue(), false, newTree);
tree.setInnerTree(newTree);
}
@@ -1285,7 +1286,7 @@
// ( ArgumentList non-terminal includes parentheses )
private boolean parsePrimaryExpression(ExprTreeHolder tree) throws IOException {
ExprTreeHolder treeL;
- TokenValue tv = tokenValuePool.get();
+ TokenValue tv = objectPool.tokenValuePool.get();
TokenType tt;
tree.setInnerTree(null);
switch ((tt = lexer.peekToken(tv))) {
@@ -1295,22 +1296,24 @@
lexer.consumeToken();
// check for funcion call
if ((tt = lexer.peekToken()) == TokenType.LEX_OPEN_PAREN) {
- ExprList argList = exprListPool.get();
+ ExprList argList = objectPool.exprListPool.get();
if (!parseArgumentList(argList)) {
tree.setInnerTree(null);
return false;
- };
+ } ;
// special case function-calls should be converted
// into a literal expression if the argument is a
// string literal
if (shouldEvaluateAtParseTime(tv.getStrValue().toString(), argList)) {
tree.setInnerTree(evaluateFunction(tv.getStrValue().toString(), argList));
} else {
- tree.setInnerTree(FunctionCall.createFunctionCall(tv.getStrValue().toString(), argList));
+ tree.setInnerTree(
+ FunctionCall.createFunctionCall(tv.getStrValue().toString(), argList, objectPool));
}
} else {
// I don't think this is ever hit
- tree.setInnerTree(AttributeReference.createAttributeReference(null, tv.getStrValue(), false));
+ tree.setInnerTree(
+ AttributeReference.createAttributeReference(null, tv.getStrValue(), false, objectPool));
}
return (tree.getInnerTree() != null);
case LEX_SELECTION:
@@ -1318,7 +1321,8 @@
lexer.consumeToken();
if ((tt = lexer.consumeToken(tv)) == TokenType.LEX_IDENTIFIER) {
// the boolean final arg signifies that reference is absolute
- tree.setInnerTree(AttributeReference.createAttributeReference(null, tv.getStrValue(), true));
+ tree.setInnerTree(
+ AttributeReference.createAttributeReference(null, tv.getStrValue(), true, objectPool));
return (tree.size() != 0);
}
// not an identifier following the '.'
@@ -1328,7 +1332,7 @@
case LEX_OPEN_PAREN: {
isExpr = true;
lexer.consumeToken();
- treeL = mutableExprPool.get();
+ treeL = objectPool.mutableExprPool.get();
parseExpression(treeL);
if (treeL.getInnerTree() == null) {
tree.resetExprTree(null);
@@ -1341,13 +1345,13 @@
// return false;
}
// assume make operation will return a new tree
- tree.setInnerTree(Operation.createOperation(Operation.OpKind_PARENTHESES_OP, treeL));
+ tree.setInnerTree(Operation.createOperation(Operation.OpKind_PARENTHESES_OP, treeL, objectPool));
return (tree.size() != 0);
}
- // constants
+ // constants
case LEX_OPEN_BOX: {
isExpr = true;
- ClassAd newAd = classAdPool.get();
+ ClassAd newAd = objectPool.classAdPool.get();
if (!parseClassAd(newAd)) {
tree.resetExprTree(null);
return false;
@@ -1358,7 +1362,7 @@
case LEX_OPEN_BRACE: {
isExpr = true;
- ExprList newList = exprListPool.get();
+ ExprList newList = objectPool.exprListPool.get();
if (!parseExprList(newList)) {
tree.setInnerTree(null);
return false;
@@ -1368,66 +1372,66 @@
return true;
case LEX_UNDEFINED_VALUE: {
- Value val = valuePool.get();
+ Value val = objectPool.valuePool.get();
lexer.consumeToken();
val.setUndefinedValue();
- tree.setInnerTree(Literal.createLiteral(val));
+ tree.setInnerTree(Literal.createLiteral(val, objectPool));
return (tree.getInnerTree() != null);
}
case LEX_ERROR_VALUE: {
- Value val = valuePool.get();
+ Value val = objectPool.valuePool.get();
lexer.consumeToken();
val.setErrorValue();
- tree.setInnerTree(Literal.createLiteral(val));
+ tree.setInnerTree(Literal.createLiteral(val, objectPool));
return (tree.getInnerTree() != null);
}
case LEX_BOOLEAN_VALUE: {
- Value val = valuePool.get();
+ Value val = objectPool.valuePool.get();
MutableBoolean b = new MutableBoolean();
tv.getBoolValue(b);
lexer.consumeToken();
val.setBooleanValue(b);
- tree.setInnerTree(Literal.createLiteral(val));
+ tree.setInnerTree(Literal.createLiteral(val, objectPool));
return (tree.getInnerTree() != null);
}
case LEX_INTEGER_VALUE: {
- Value val = valuePool.get();
+ Value val = objectPool.valuePool.get();
lexer.consumeToken();
val.setIntegerValue(tv.getIntValue());
- tree.setInnerTree(Literal.createLiteral(val, tv.getFactor()));
+ tree.setInnerTree(Literal.createLiteral(val, tv.getFactor(), objectPool));
return (tree.getInnerTree() != null);
}
case LEX_REAL_VALUE: {
- Value val = valuePool.get();
+ Value val = objectPool.valuePool.get();
lexer.consumeToken();
val.setRealValue(tv.getRealValue());
- tree.setInnerTree(Literal.createLiteral(val, tv.getFactor()));
+ tree.setInnerTree(Literal.createLiteral(val, tv.getFactor(), objectPool));
return (tree.getInnerTree() != null);
}
case LEX_STRING_VALUE: {
- Value val = valuePool.get();
+ Value val = objectPool.valuePool.get();
lexer.consumeToken();
val.setStringValue(tv.getStrValue());
- tree.setInnerTree(Literal.createLiteral(val));
+ tree.setInnerTree(Literal.createLiteral(val, objectPool));
return (tree.getInnerTree() != null);
}
case LEX_ABSOLUTE_TIME_VALUE: {
- Value val = valuePool.get();
+ Value val = objectPool.valuePool.get();
lexer.consumeToken();
val.setAbsoluteTimeValue(tv.getTimeValue());
- tree.setInnerTree(Literal.createLiteral(val));
+ tree.setInnerTree(Literal.createLiteral(val, objectPool));
return (tree.getInnerTree() != null);
}
case LEX_RELATIVE_TIME_VALUE: {
- Value val = valuePool.get();
+ Value val = objectPool.valuePool.get();
lexer.consumeToken();
val.setRelativeTimeValue(tv.getTimeValue().getRelativeTime());
- tree.setInnerTree(Literal.createLiteral(val));
+ tree.setInnerTree(Literal.createLiteral(val, objectPool));
return (tree.getInnerTree() != null);
}
@@ -1448,7 +1452,7 @@
// return false;
}
tt = lexer.peekToken();
- ExprTreeHolder tree = mutableExprPool.get();
+ ExprTreeHolder tree = objectPool.mutableExprPool.get();
while (tt != TokenType.LEX_CLOSE_PAREN) {
// parse the expression
tree.reset();
@@ -1462,9 +1466,9 @@
// the next token must be a ',' or a ')'
// or it can be a ';' if using old ClassAd semantics
tt = lexer.peekToken();
- if (tt == TokenType.LEX_COMMA || (tt == TokenType.LEX_SEMICOLON && false))
+ if (tt == TokenType.LEX_COMMA || (tt == TokenType.LEX_SEMICOLON && false)) {
lexer.consumeToken();
- else if (tt != TokenType.LEX_CLOSE_PAREN) {
+ } else if (tt != TokenType.LEX_CLOSE_PAREN) {
argList.clear();
throw new HyracksDataException(
"expected LEX_COMMA or LEX_CLOSE_PAREN but got " + String.valueOf(Lexer.strLexToken(tt)));
@@ -1490,11 +1494,12 @@
public boolean parseClassAd(ClassAd ad, boolean full) throws IOException {
TokenType tt;
ad.clear();
- if ((tt = lexer.consumeToken()) != TokenType.LEX_OPEN_BOX)
+ if ((tt = lexer.consumeToken()) != TokenType.LEX_OPEN_BOX) {
return false;
+ }
tt = lexer.peekToken();
- TokenValue tv = tokenValuePool.get();
- ExprTreeHolder tree = mutableExprPool.get();
+ TokenValue tv = objectPool.tokenValuePool.get();
+ ExprTreeHolder tree = objectPool.mutableExprPool.get();
while (tt != TokenType.LEX_CLOSE_BOX) {
// Get the name of the expression
tv.reset();
@@ -1567,8 +1572,8 @@
public boolean parseExprList(ExprList list, boolean full) throws IOException {
TokenType tt;
- ExprTreeHolder tree = new ExprTreeHolder();
- ExprList loe = new ExprList();
+ ExprTreeHolder tree = objectPool.mutableExprPool.get();
+ ExprList loe = objectPool.exprListPool.get();
if ((tt = lexer.consumeToken()) != TokenType.LEX_OPEN_BRACE) {
throw new HyracksDataException(
@@ -1589,16 +1594,16 @@
// the next token must be a ',' or a '}'
tt = lexer.peekToken();
- if (tt == TokenType.LEX_COMMA)
+ if (tt == TokenType.LEX_COMMA) {
lexer.consumeToken();
- else if (tt != TokenType.LEX_CLOSE_BRACE) {
+ } else if (tt != TokenType.LEX_CLOSE_BRACE) {
throw new HyracksDataException("while parsing expression list: expected "
+ "LEX_CLOSE_BRACE or LEX_COMMA but got " + Lexer.strLexToken(tt));
}
}
lexer.consumeToken();
- list.setValue(ExprList.createExprList(loe));
+ list.setValue(ExprList.createExprList(loe, objectPool));
// if a full parse was requested, ensure that input is exhausted
if (full && (lexer.consumeToken() != TokenType.LEX_END_OF_INPUT)) {
@@ -1613,8 +1618,8 @@
boolean should_eval = false;
if (functionName.equalsIgnoreCase("absTime") || functionName.equalsIgnoreCase("relTime")) {
if (argList.size() == 1 && argList.get(0).getKind() == NodeKind.LITERAL_NODE) {
- Value val = new Value();
- AMutableNumberFactor factor = new AMutableNumberFactor();
+ Value val = objectPool.valuePool.get();
+ AMutableNumberFactor factor = objectPool.numFactorPool.get();
((Literal) argList.get(0)).getComponents(val, factor);
if (val.isStringValue()) {
should_eval = true;
@@ -1625,22 +1630,22 @@
}
public ExprTree evaluateFunction(String functionName, ExprList argList) throws HyracksDataException {
- Value val = new Value();
- AMutableNumberFactor factor = new AMutableNumberFactor();
- ExprTreeHolder tree = new ExprTreeHolder();
+ Value val = objectPool.valuePool.get();
+ AMutableNumberFactor factor = objectPool.numFactorPool.get();
+ ExprTreeHolder tree = objectPool.mutableExprPool.get();
((Literal) argList.get(0)).getComponents(val, factor);
- AMutableCharArrayString string_value = new AMutableCharArrayString();
+ AMutableCharArrayString string_value = objectPool.strPool.get();
if (val.isStringValue(string_value)) {
if (functionName.equalsIgnoreCase("absTime")) {
- tree.setInnerTree(Literal.createAbsTime(string_value));
+ tree.setInnerTree(Literal.createAbsTime(string_value, objectPool));
} else if (functionName.equalsIgnoreCase("relTime")) {
- tree.setInnerTree(Literal.createRelTime(string_value));
+ tree.setInnerTree(Literal.createRelTime(string_value, objectPool));
} else {
- tree.setInnerTree(FunctionCall.createFunctionCall(functionName, argList));
+ tree.setInnerTree(FunctionCall.createFunctionCall(functionName, argList, objectPool));
}
} else {
- tree.setInnerTree(FunctionCall.createFunctionCall(functionName, argList));
+ tree.setInnerTree(FunctionCall.createFunctionCall(functionName, argList, objectPool));
}
return tree;
}
@@ -1688,17 +1693,10 @@
this.currentSource = lexerSource;
}
- public void reset() {
- resetPools();
- }
-
- public Literal getLiteral() {
- return literalPool.get();
- }
-
@Override
public void parse(IRawRecord<? extends char[]> record, DataOutput out) throws IOException {
try {
+ resetPools();
if (oldFormat) {
int maxOffset = record.size();
rootAd.clear();
@@ -1718,7 +1716,6 @@
line = readLine(buffer, aInt32, maxOffset);
}
} else {
- resetPools();
currentSource.setNewSource(record.get());
rootAd.reset();
asterixParseClassAd(rootAd);
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParserFactory.java b/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParserFactory.java
index 5064992..17b83c2 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParserFactory.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParserFactory.java
@@ -25,6 +25,7 @@
import org.apache.asterix.external.api.IRecordDataParser;
import org.apache.asterix.external.api.IRecordDataParserFactory;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.om.types.ARecordType;
import org.apache.hyracks.api.context.IHyracksTaskContext;
@@ -42,8 +43,8 @@
private ARecordType recordType;
private Map<String, String> configuration;
private boolean oldFormat = false;
- private boolean evaluateExpr = true;
- private boolean keepBoth;
+ private boolean evaluateExpr = false;
+ private boolean keepBoth = false;
private String exprPrefix;
private String exprSuffix;
private String exprFieldNameSuffix;
@@ -77,14 +78,12 @@
}
parserConfig = configuration.get(ClassAdParserFactory.KEY_EVALUATE);
- if ((parserConfig != null) && parserConfig.equalsIgnoreCase("false")) {
- evaluateExpr = false;
- keepBoth = false;
+ if ((parserConfig != null) && parserConfig.equalsIgnoreCase(ExternalDataConstants.TRUE)) {
+ evaluateExpr = true;
}
parserConfig = configuration.get(ClassAdParserFactory.KEY_KEEP_EXPR);
- if ((parserConfig != null) && parserConfig.equalsIgnoreCase("false")) {
- keepBoth = false;
- evaluateExpr = true;
+ if ((parserConfig != null) && parserConfig.equalsIgnoreCase(ExternalDataConstants.TRUE)) {
+ keepBoth = true;
}
parserConfig = configuration.get(ClassAdParserFactory.KEY_EXPR_PREFIX);
@@ -110,7 +109,7 @@
@Override
public IRecordDataParser<char[]> createRecordParser(IHyracksTaskContext ctx) throws HyracksDataException {
return new ClassAdParser(recordType, oldFormat, evaluateExpr, keepBoth, exprPrefix, exprSuffix,
- exprFieldNameSuffix);
+ exprFieldNameSuffix, new ClassAdObjectPool());
}
@Override
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java b/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java
index 521cee0..851a7e0 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java
@@ -28,7 +28,7 @@
import org.apache.asterix.external.api.IRawRecord;
import org.apache.asterix.external.input.record.converter.CSVToRecordWithMetadataAndPKConverter;
import org.apache.asterix.external.input.record.reader.stream.QuotedLineRecordReader;
-import org.apache.asterix.external.input.stream.LocalFileSystemInputStream;
+import org.apache.asterix.external.input.stream.LocalFSInputStream;
import org.apache.asterix.external.parser.ADMDataParser;
import org.apache.asterix.external.parser.RecordWithMetadataParser;
import org.apache.asterix.external.util.ExternalDataConstants;
@@ -42,7 +42,9 @@
import org.apache.hyracks.algebricks.data.IPrinterFactory;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.junit.Assert;
import org.junit.Test;
@@ -78,8 +80,10 @@
int[] pkIndicators = { 1 };
// create input stream
- LocalFileSystemInputStream inputStream = new LocalFileSystemInputStream(
- Paths.get(getClass().getResource("/beer.csv").toURI()).toAbsolutePath(), null, false);
+ LocalFSInputStream inputStream = new LocalFSInputStream(
+ new FileSplit[] { new FileSplit(null,
+ new FileReference(Paths.get(getClass().getResource("/beer.csv").toURI()).toFile())) },
+ null, null, 0, null, false);
// create reader record reader
QuotedLineRecordReader lineReader = new QuotedLineRecordReader(true, inputStream, null,
diff --git a/asterix-external-data/src/test/resources/functional_tests.txt b/asterix-external-data/src/test/resources/functional_tests.txt
index 42c0e95..3aaae46 100644
--- a/asterix-external-data/src/test/resources/functional_tests.txt
+++ b/asterix-external-data/src/test/resources/functional_tests.txt
@@ -1,3 +1,14 @@
+echo Testing select on lists...
+eval x = {
+ [a = 3; b = "I see London"],
+ [a = 2; b = "I see France"],
+ [a = 1; b = "I see Alain's funky pants"]
+ }
+same $x.a, {3, 2, 1}
+same $x.b, {"I see London", "I see France", "I see Alain's funky pants"}
+same $x.c, {undefined, undefined, undefined}
+same {}.a, {}
+
/////////////////////////////////
echo Testing basic math...
eval x = (1 + 2)
@@ -74,16 +85,7 @@
// same $y, $z[1]
/////////////////////////////////
-echo Testing select on lists...
-eval x = {
- [a = 3; b = "I see London"],
- [a = 2; b = "I see France"],
- [a = 1; b = "I see Alain's funky pants"]
- }
-same $x.a, {3, 2, 1}
-same $x.b, {"I see London", "I see France", "I see Alain's funky pants"}
-same $x.c, {undefined, undefined, undefined}
-same {}.a, {}
+
/////////////////////////////////
echo Testing subscripts