Temp dataset support:
1. DDLs for creating a temporary dataset
2. Garbage collection for temporary dataset
3. Reading, inserting, and deleting data from (to) a temporary dataset is locking-free (except metadata locks) and logging-free (except flush/merge/job-commit logs).
Change-Id: Id0960acd1b4dfbfa7685dba227634572ca50b126
Reviewed-on: https://asterix-gerrit.ics.uci.edu/241
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Young-Seok Kim <kisskys@gmail.com>
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitPOperator.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitPOperator.java
index 18c0793..ee71851 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitPOperator.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitPOperator.java
@@ -18,6 +18,7 @@
import java.util.List;
import edu.uci.ics.asterix.common.transactions.JobId;
+import edu.uci.ics.asterix.metadata.declared.AqlMetadataProvider;
import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
@@ -29,7 +30,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.AbstractPhysicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
@@ -39,14 +39,11 @@
private final List<LogicalVariable> primaryKeyLogicalVars;
private final JobId jobId;
private final int datasetId;
- private final boolean isWriteTransaction;
- public CommitPOperator(JobId jobId, int datasetId, List<LogicalVariable> primaryKeyLogicalVars,
- boolean isWriteTransaction) {
+ public CommitPOperator(JobId jobId, int datasetId, List<LogicalVariable> primaryKeyLogicalVars) {
this.jobId = jobId;
this.datasetId = datasetId;
this.primaryKeyLogicalVars = primaryKeyLogicalVars;
- this.isWriteTransaction = isWriteTransaction;
}
@Override
@@ -69,7 +66,7 @@
public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context)
throws AlgebricksException {
AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
- deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
+ deliveredProperties = op2.getDeliveredPhysicalProperties().clone();
}
@Override
@@ -80,7 +77,10 @@
RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema,
context);
int[] primaryKeyFields = JobGenHelper.variablesToFieldIndexes(primaryKeyLogicalVars, inputSchemas[0]);
- CommitRuntimeFactory runtime = new CommitRuntimeFactory(jobId, datasetId, primaryKeyFields, isWriteTransaction);
+
+ AqlMetadataProvider metadataProvider = (AqlMetadataProvider) context.getMetadataProvider();
+ CommitRuntimeFactory runtime = new CommitRuntimeFactory(jobId, datasetId, primaryKeyFields,
+ metadataProvider.isTemporaryDatasetWriteJob(), metadataProvider.isWriteTransaction());
builder.contributeMicroOperator(op, runtime, recDesc);
ILogicalOperator src = op.getInputs().get(0).getValue();
builder.contributeGraphEdge(src, 0, op, 0);
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitRuntime.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitRuntime.java
index af83b83..bed18fe 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitRuntime.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitRuntime.java
@@ -44,17 +44,17 @@
private final JobId jobId;
private final int datasetId;
private final int[] primaryKeyFields;
+ private final boolean isTemporaryDatasetWriteJob;
private final boolean isWriteTransaction;
private final long[] longHashes;
private final LogRecord logRecord;
private ITransactionContext transactionContext;
- private RecordDescriptor inputRecordDesc;
private FrameTupleAccessor frameTupleAccessor;
- private FrameTupleReference frameTupleReference;
+ private final FrameTupleReference frameTupleReference;
public CommitRuntime(IHyracksTaskContext ctx, JobId jobId, int datasetId, int[] primaryKeyFields,
- boolean isWriteTransaction) {
+ boolean isTemporaryDatasetWriteJob, boolean isWriteTransaction) {
this.hyracksTaskCtx = ctx;
IAsterixAppRuntimeContext runtimeCtx = (IAsterixAppRuntimeContext) ctx.getJobletContext()
.getApplicationContext().getApplicationObject();
@@ -64,6 +64,7 @@
this.datasetId = datasetId;
this.primaryKeyFields = primaryKeyFields;
this.frameTupleReference = new FrameTupleReference();
+ this.isTemporaryDatasetWriteJob = isTemporaryDatasetWriteJob;
this.isWriteTransaction = isWriteTransaction;
this.longHashes = new long[2];
this.logRecord = new LogRecord();
@@ -85,14 +86,28 @@
frameTupleAccessor.reset(buffer);
int nTuple = frameTupleAccessor.getTupleCount();
for (int t = 0; t < nTuple; t++) {
- frameTupleReference.reset(frameTupleAccessor, t);
- pkHash = computePrimaryKeyHashValue(frameTupleReference, primaryKeyFields);
- logRecord.formEntityCommitLogRecord(transactionContext, datasetId, pkHash, frameTupleReference,
- primaryKeyFields);
- try {
- logMgr.log(logRecord);
- } catch (ACIDException e) {
- throw new HyracksDataException(e);
+ if (isTemporaryDatasetWriteJob) {
+ /**
+ * This "if branch" is for writes over temporary datasets.
+ * A temporary dataset does not require any lock and does not generate any write-ahead
+ * update and commit log but generates flush log and job commit log.
+ * However, a temporary dataset still MUST guarantee no-steal policy so that this
+ * notification call should be delivered to PrimaryIndexOptracker and used correctly in order
+ * to decrement number of active operation count of PrimaryIndexOptracker.
+ * By maintaining the count correctly and only allowing flushing when the count is 0, it can
+ * guarantee the no-steal policy for temporary datasets, too.
+ */
+ transactionContext.notifyOptracker(false);
+ } else {
+ frameTupleReference.reset(frameTupleAccessor, t);
+ pkHash = computePrimaryKeyHashValue(frameTupleReference, primaryKeyFields);
+ logRecord.formEntityCommitLogRecord(transactionContext, datasetId, pkHash, frameTupleReference,
+ primaryKeyFields);
+ try {
+ logMgr.log(logRecord);
+ } catch (ACIDException e) {
+ throw new HyracksDataException(e);
+ }
}
}
}
@@ -104,13 +119,12 @@
@Override
public void fail() throws HyracksDataException {
- // TODO Auto-generated method stub
}
@Override
public void close() throws HyracksDataException {
- // TODO Auto-generated method stub
+
}
@Override
@@ -120,7 +134,6 @@
@Override
public void setInputRecordDescriptor(int index, RecordDescriptor recordDescriptor) {
- this.inputRecordDesc = recordDescriptor;
this.frameTupleAccessor = new FrameTupleAccessor(hyracksTaskCtx.getFrameSize(), recordDescriptor);
}
-}
+}
\ No newline at end of file
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitRuntimeFactory.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitRuntimeFactory.java
index 6776f28..b996630 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitRuntimeFactory.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitRuntimeFactory.java
@@ -28,12 +28,15 @@
private final JobId jobId;
private final int datasetId;
private final int[] primaryKeyFields;
+ private final boolean isTemporaryDatasetWriteJob;
private final boolean isWriteTransaction;
- public CommitRuntimeFactory(JobId jobId, int datasetId, int[] primaryKeyFields, boolean isWriteTransaction) {
+ public CommitRuntimeFactory(JobId jobId, int datasetId, int[] primaryKeyFields, boolean isTemporaryDatasetWriteJob,
+ boolean isWriteTransaction) {
this.jobId = jobId;
this.datasetId = datasetId;
this.primaryKeyFields = primaryKeyFields;
+ this.isTemporaryDatasetWriteJob = isTemporaryDatasetWriteJob;
this.isWriteTransaction = isWriteTransaction;
}
@@ -44,6 +47,7 @@
@Override
public IPushRuntime createPushRuntime(IHyracksTaskContext ctx) throws AlgebricksException {
- return new CommitRuntime(ctx, jobId, datasetId, primaryKeyFields, isWriteTransaction);
+ return new CommitRuntime(ctx, jobId, datasetId, primaryKeyFields, isTemporaryDatasetWriteJob,
+ isWriteTransaction);
}
}
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/InvertedIndexPOperator.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/InvertedIndexPOperator.java
index ff21aff..67be923 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/InvertedIndexPOperator.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/InvertedIndexPOperator.java
@@ -172,10 +172,12 @@
throw new AlgebricksException("Only record types can be indexed.");
}
ARecordType recordType = (ARecordType) itemType;
- Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(secondaryKeyTypeEntries.get(0), secondaryKeyFieldEntries.get(0), recordType);
+ Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(secondaryKeyTypeEntries.get(0),
+ secondaryKeyFieldEntries.get(0), recordType);
IAType secondaryKeyType = keyPairType.first;
if (secondaryKeyType == null) {
- throw new AlgebricksException("Could not find field " + secondaryKeyFieldEntries.get(0) + " in the schema.");
+ throw new AlgebricksException("Could not find field " + secondaryKeyFieldEntries.get(0)
+ + " in the schema.");
}
// TODO: For now we assume the type of the generated tokens is the
@@ -236,8 +238,8 @@
IAsterixApplicationContextInfo appContext = (IAsterixApplicationContextInfo) context.getAppContext();
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> secondarySplitsAndConstraint = metadataProvider
- .splitProviderAndPartitionConstraintsForDataset(dataset.getDataverseName(),
- datasetName, indexName);
+ .splitProviderAndPartitionConstraintsForDataset(dataset.getDataverseName(), datasetName, indexName,
+ dataset.getDatasetDetails().isTemp());
// TODO: Here we assume there is only one search key field.
int queryField = keyFields[0];
// Get tokenizer and search modifier factories.
@@ -250,6 +252,7 @@
AsterixStorageProperties storageProperties = AsterixAppContextInfo.getInstance().getStorageProperties();
Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(
dataset, metadataProvider.getMetadataTxnContext());
+ boolean temp = dataset.getDatasetDetails().isTemp();
if (!isPartitioned) {
dataflowHelperFactory = new LSMInvertedIndexDataflowHelperFactory(
new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()), compactionInfo.first,
@@ -258,7 +261,7 @@
LSMInvertedIndexIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(), invertedIndexFields, filterTypeTraits,
filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps,
- invertedIndexFieldsForNonBulkLoadOps);
+ invertedIndexFieldsForNonBulkLoadOps, !temp);
} else {
dataflowHelperFactory = new PartitionedLSMInvertedIndexDataflowHelperFactory(
new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()), compactionInfo.first,
@@ -267,7 +270,7 @@
LSMInvertedIndexIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(), invertedIndexFields, filterTypeTraits,
filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps,
- invertedIndexFieldsForNonBulkLoadOps);
+ invertedIndexFieldsForNonBulkLoadOps, !temp);
}
LSMInvertedIndexSearchOperatorDescriptor invIndexSearchOp = new LSMInvertedIndexSearchOperatorDescriptor(
jobSpec, queryField, appContext.getStorageManagerInterface(), secondarySplitsAndConstraint.first,
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java
index ef9d6cd..8de761a 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java
@@ -99,8 +99,7 @@
//create the logical and physical operator
CommitOperator commitOperator = new CommitOperator(primaryKeyLogicalVars);
- CommitPOperator commitPOperator = new CommitPOperator(jobId, datasetId, primaryKeyLogicalVars,
- mp.isWriteTransaction());
+ CommitPOperator commitPOperator = new CommitPOperator(jobId, datasetId, primaryKeyLogicalVars);
commitOperator.setPhysicalOperator(commitPOperator);
//create ExtensionOperator and put the commitOperator in it.
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/APIFramework.java b/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/APIFramework.java
index 51fef41..0ce15ed 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/APIFramework.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/APIFramework.java
@@ -72,7 +72,6 @@
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IOptimizationContextFactory;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
-import edu.uci.ics.hyracks.api.job.IJobletEventListenerFactory;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobSpecification;
@@ -156,8 +155,7 @@
}
public static Pair<Query, Integer> reWriteQuery(List<FunctionDecl> declaredFunctions,
- AqlMetadataProvider metadataProvider, Query q, SessionConfig conf)
- throws AsterixException {
+ AqlMetadataProvider metadataProvider, Query q, SessionConfig conf) throws AsterixException {
if (conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS) && conf.is(SessionConfig.OOB_EXPR_TREE)) {
conf.out().println();
@@ -185,8 +183,8 @@
public static JobSpecification compileQuery(List<FunctionDecl> declaredFunctions,
AqlMetadataProvider queryMetadataProvider, Query rwQ, int varCounter, String outputDatasetName,
- SessionConfig conf, ICompiledDmlStatement statement)
- throws AsterixException, AlgebricksException, JSONException, RemoteException, ACIDException {
+ SessionConfig conf, ICompiledDmlStatement statement) throws AsterixException, AlgebricksException,
+ JSONException, RemoteException, ACIDException {
if (conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS) && conf.is(SessionConfig.OOB_REWRITTEN_EXPR_TREE)) {
conf.out().println();
@@ -219,7 +217,6 @@
} else {
plan = t.translateLoad();
}
- boolean isWriteTransaction = queryMetadataProvider.isWriteTransaction();
LogicalOperatorPrettyPrintVisitor pvisitor = new LogicalOperatorPrettyPrintVisitor();
if (conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS) && conf.is(SessionConfig.OOB_LOGICAL_PLAN)) {
@@ -340,8 +337,8 @@
builder.setTypeTraitProvider(format.getTypeTraitProvider());
builder.setNormalizedKeyComputerFactoryProvider(format.getNormalizedKeyComputerFactoryProvider());
- IJobletEventListenerFactory jobEventListenerFactory = new JobEventListenerFactory(asterixJobId,
- isWriteTransaction);
+ JobEventListenerFactory jobEventListenerFactory = new JobEventListenerFactory(asterixJobId,
+ queryMetadataProvider.isWriteTransaction());
JobSpecification spec = compiler.createJob(AsterixAppContextInfo.getInstance(), jobEventListenerFactory);
if (conf.is(SessionConfig.OOB_HYRACKS_JOB)) {
@@ -365,7 +362,7 @@
}
public static void executeJobArray(IHyracksClientConnection hcc, JobSpecification[] specs, PrintWriter out)
- throws Exception {
+ throws Exception {
for (int i = 0; i < specs.length; i++) {
specs[i].setMaxReattempts(0);
JobId jobId = hcc.startJob(specs[i]);
@@ -378,8 +375,7 @@
}
- public static void executeJobArray(IHyracksClientConnection hcc, Job[] jobs, PrintWriter out)
- throws Exception {
+ public static void executeJobArray(IHyracksClientConnection hcc, Job[] jobs, PrintWriter out) throws Exception {
for (int i = 0; i < jobs.length; i++) {
jobs[i].getJobSpec().setMaxReattempts(0);
long startTime = System.currentTimeMillis();
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
index 4c47c7d..e5ddf2b 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
@@ -159,7 +159,6 @@
import edu.uci.ics.hyracks.api.dataset.IHyracksDataset;
import edu.uci.ics.hyracks.api.dataset.ResultSetId;
import edu.uci.ics.hyracks.api.io.FileReference;
-import edu.uci.ics.hyracks.api.job.IJobletEventListenerFactory;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
@@ -210,7 +209,7 @@
/**
* Compiles and submits for execution a list of AQL statements.
- *
+ *
* @param hcc
* A Hyracks client connection that is used to submit a jobspec to Hyracks.
* @param hdc
@@ -478,6 +477,7 @@
String compactionPolicy = dd.getDatasetDetailsDecl().getCompactionPolicy();
Map<String, String> compactionPolicyProperties = dd.getDatasetDetailsDecl().getCompactionPolicyProperties();
boolean defaultCompactionPolicy = (compactionPolicy == null);
+ boolean temp = dd.getDatasetDetailsDecl().isTemp();
MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
boolean bActiveTxn = true;
@@ -539,7 +539,7 @@
datasetDetails = new InternalDatasetDetails(InternalDatasetDetails.FileStructure.BTREE,
InternalDatasetDetails.PartitioningStrategy.HASH, partitioningExprs, partitioningExprs,
partitioningTypes, ngName, autogenerated, compactionPolicy, compactionPolicyProperties,
- filterField);
+ filterField, temp);
break;
}
case EXTERNAL: {
@@ -2532,8 +2532,10 @@
throw new AlgebricksException("Tried to access non-existing dataset: " + datasetNameFrom);
}
+ Dataset datasetFrom = MetadataManager.INSTANCE.getDataset(mdTxnCtx, dataverseNameFrom, datasetNameFrom);
IFileSplitProvider fromSplits = metadataProvider.splitProviderAndPartitionConstraintsForDataset(
- dataverseNameFrom, datasetNameFrom, fromIndex.getIndexName()).first;
+ dataverseNameFrom, datasetNameFrom, fromIndex.getIndexName(), datasetFrom.getDatasetDetails()
+ .isTemp()).first;
StringBuilder fromSplitsPaths = new StringBuilder();
for (FileSplit f : fromSplits.getFileSplits()) {
@@ -2555,8 +2557,9 @@
throw new AlgebricksException("Tried to access non-existing dataset: " + datasetNameTo);
}
+ Dataset datasetTo = MetadataManager.INSTANCE.getDataset(mdTxnCtx, dataverseNameTo, datasetNameTo);
IFileSplitProvider toSplits = metadataProvider.splitProviderAndPartitionConstraintsForDataset(
- dataverseNameTo, datasetNameTo, toIndex.getIndexName()).first;
+ dataverseNameTo, datasetNameTo, toIndex.getIndexName(), datasetTo.getDatasetDetails().isTemp()).first;
StringBuilder toSplitsPaths = new StringBuilder();
for (FileSplit f : toSplits.getFileSplits()) {
@@ -2573,7 +2576,8 @@
IDatasetDetailsDecl idd = new InternalDetailsDecl(new Identifier(toDataset.getDatasetDetails()
.getNodeGroupName()), toIndex.getKeyFieldNames(), false, toDataset.getDatasetDetails()
- .getCompactionPolicy(), toDataset.getDatasetDetails().getCompactionPolicyProperties(), null);
+ .getCompactionPolicy(), toDataset.getDatasetDetails().getCompactionPolicyProperties(), null,
+ toDataset.getDatasetDetails().isTemp());
DatasetDecl createToDataset = new DatasetDecl(new Identifier(dataverseNameTo),
pregelixStmt.getDatasetNameTo(), new Identifier(toDataset.getItemTypeName()),
toDataset.getHints(), toDataset.getDatasetType(), idd, false);
@@ -2691,19 +2695,21 @@
new IPushRuntimeFactory[] { new EmptyTupleSourceRuntimeFactory() }, rDescs);
edu.uci.ics.asterix.common.transactions.JobId jobId = JobIdFactory.generateJobId();
+ Dataset dataset = MetadataManager.INSTANCE.getDataset(mdTxnCtx, dataverseName, datasetName);
FlushDatasetOperatorDescriptor flushOperator = new FlushDatasetOperatorDescriptor(spec, jobId,
- MetadataManager.INSTANCE.getDataset(mdTxnCtx, dataverseName, datasetName).getDatasetId());
+ dataset.getDatasetId());
spec.connect(new OneToOneConnectorDescriptor(spec), emptySource, 0, flushOperator, 0);
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> primarySplitsAndConstraint = metadataProvider
- .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, indexName);
+ .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, indexName, dataset
+ .getDatasetDetails().isTemp());
AlgebricksPartitionConstraint primaryPartitionConstraint = primarySplitsAndConstraint.second;
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, emptySource,
primaryPartitionConstraint);
- IJobletEventListenerFactory jobEventListenerFactory = new JobEventListenerFactory(jobId, false);
+ JobEventListenerFactory jobEventListenerFactory = new JobEventListenerFactory(jobId, true);
spec.setJobletEventListenerFactory(jobEventListenerFactory);
runJob(hcc, spec, true);
}
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
index 7a9e1b2..b2da545 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
@@ -86,6 +86,7 @@
if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
return JobSpecificationUtils.createJobSpecification();
}
+ boolean temp = dataset.getDatasetDetails().isTemp();
Dataverse dataverse = MetadataManager.INSTANCE.getDataverse(metadataProvider.getMetadataTxnContext(),
dataverseName);
@@ -106,11 +107,13 @@
JobSpecification specPrimary = JobSpecificationUtils.createJobSpecification();
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadataProvider
- .splitProviderAndPartitionConstraintsForDataset(dataset.getDataverseName(), datasetName,
- datasetName);
+ .splitProviderAndPartitionConstraintsForDataset(dataset.getDataverseName(), datasetName, datasetName,
+ temp);
AsterixStorageProperties storageProperties = AsterixAppContextInfo.getInstance().getStorageProperties();
Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(dataset,
metadataProvider.getMetadataTxnContext());
+
+ // The index drop operation should be persistent regardless of temp datasets or permanent dataset
IndexDropOperatorDescriptor primaryBtreeDrop = new IndexDropOperatorDescriptor(specPrimary,
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
splitsAndConstraint.first, new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(
@@ -118,7 +121,7 @@
new PrimaryIndexOperationTrackerProvider(dataset.getDatasetId()),
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMBTreeIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(), true, filterTypeTraits,
- filterCmpFactories, btreeFields, filterFields));
+ filterCmpFactories, btreeFields, filterFields, true));
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(specPrimary, primaryBtreeDrop,
splitsAndConstraint.second);
@@ -140,6 +143,7 @@
if (dataset == null) {
throw new AsterixException("Could not find dataset " + datasetName + " in dataverse " + dataverseName);
}
+ boolean temp = dataset.getDatasetDetails().isTemp();
ARecordType itemType = (ARecordType) metadata.findType(dataverseName, dataset.getItemTypeName());
JobSpecification spec = JobSpecificationUtils.createJobSpecification();
IBinaryComparatorFactory[] comparatorFactories = DatasetUtils.computeKeysBinaryComparatorFactories(dataset,
@@ -154,7 +158,7 @@
int[] btreeFields = DatasetUtils.createBTreeFieldsWhenThereisAFilter(dataset);
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadata
- .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, datasetName);
+ .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, datasetName, temp);
FileSplit[] fs = splitsAndConstraint.first.getFileSplits();
StringBuilder sb = new StringBuilder();
for (int i = 0; i < fs.length; i++) {
@@ -172,6 +176,7 @@
ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
localResourceMetadata, LocalResource.LSMBTreeResource);
+ // The index create operation should be persistent regardless of temp datasets or permanent dataset
TreeIndexCreateOperatorDescriptor indexCreateOp = new TreeIndexCreateOperatorDescriptor(spec,
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
splitsAndConstraint.first, typeTraits, comparatorFactories, bloomFilterKeyFields,
@@ -180,7 +185,7 @@
.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
LSMBTreeIOOperationCallbackFactory.INSTANCE, storageProperties
.getBloomFilterFalsePositiveRate(), true, filterTypeTraits, filterCmpFactories,
- btreeFields, filterFields), localResourceFactoryProvider,
+ btreeFields, filterFields, true), localResourceFactoryProvider,
NoOpOperationCallbackFactory.INSTANCE);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, indexCreateOp,
splitsAndConstraint.second);
@@ -205,6 +210,8 @@
if (dataset == null) {
throw new AsterixException("Could not find dataset " + datasetName + " in dataverse " + dataverseName);
}
+ boolean temp = dataset.getDatasetDetails().isTemp();
+
ARecordType itemType = (ARecordType) metadata.findType(dataverseName, dataset.getItemTypeName());
JobSpecification spec = JobSpecificationUtils.createJobSpecification();
IBinaryComparatorFactory[] comparatorFactories = DatasetUtils.computeKeysBinaryComparatorFactories(dataset,
@@ -219,7 +226,7 @@
int[] btreeFields = DatasetUtils.createBTreeFieldsWhenThereisAFilter(dataset);
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadata
- .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, datasetName);
+ .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, datasetName, temp);
AsterixStorageProperties storageProperties = AsterixAppContextInfo.getInstance().getStorageProperties();
@@ -233,7 +240,7 @@
dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
LSMBTreeIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(), true, filterTypeTraits,
- filterCmpFactories, btreeFields, filterFields), NoOpOperationCallbackFactory.INSTANCE);
+ filterCmpFactories, btreeFields, filterFields, !temp), NoOpOperationCallbackFactory.INSTANCE);
AlgebricksPartitionConstraintHelper
.setPartitionConstraintInJobSpec(spec, compactOp, splitsAndConstraint.second);
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/ExternalIndexingOperations.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/ExternalIndexingOperations.java
index ba6357b..a519eaf 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/ExternalIndexingOperations.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/ExternalIndexingOperations.java
@@ -212,7 +212,7 @@
public static FileSystem getFileSystemObject(Map<String, String> map) throws IOException {
Configuration conf = new Configuration();
- conf.set("fs.default.name", ((String) map.get(HDFSAdapterFactory.KEY_HDFS_URL)).trim());
+ conf.set("fs.default.name", map.get(HDFSAdapterFactory.KEY_HDFS_URL).trim());
conf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
return FileSystem.get(conf);
}
@@ -241,7 +241,7 @@
mergePolicyFactory, mergePolicyFactoryProperties, new SecondaryIndexOperationTrackerProvider(
dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
LSMBTreeIOOperationCallbackFactory.INSTANCE, storageProperties.getBloomFilterFalsePositiveRate(),
- ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset));
+ ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset), true);
ExternalFilesIndexOperatorDescriptor externalFilesOp = new ExternalFilesIndexOperatorDescriptor(spec,
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
secondaryFileSplitProvider, indexDataflowHelperFactory, localResourceFactoryProvider,
@@ -408,6 +408,7 @@
: indexDropStmt.getDataverseName();
String datasetName = indexDropStmt.getDatasetName();
String indexName = indexDropStmt.getIndexName();
+ boolean temp = dataset.getDatasetDetails().isTemp();
JobSpecification spec = JobSpecificationUtils.createJobSpecification();
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadataProvider
.splitProviderAndPartitionConstraintsForFilesIndex(dataverseName, datasetName, indexName, true);
@@ -420,7 +421,7 @@
dataset.getDatasetId()), compactionInfo.first, compactionInfo.second,
new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMBTreeIOOperationCallbackFactory.INSTANCE,
- storageProperties.getBloomFilterFalsePositiveRate(), false, null, null, null, null));
+ storageProperties.getBloomFilterFalsePositiveRate(), false, null, null, null, null, !temp));
AlgebricksPartitionConstraintHelper
.setPartitionConstraintInJobSpec(spec, btreeDrop, splitsAndConstraint.second);
spec.addRoot(btreeDrop);
@@ -488,11 +489,12 @@
AsterixStorageProperties storageProperties = asterixPropertiesProvider.getStorageProperties();
Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(ds,
metadataProvider.getMetadataTxnContext());
+ boolean temp = ds.getDatasetDetails().isTemp();
ILSMMergePolicyFactory mergePolicyFactory = compactionInfo.first;
Map<String, String> mergePolicyFactoryProperties = compactionInfo.second;
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> filesIndexSplitsAndConstraint = metadataProvider
.splitProviderAndPartitionConstraintsForDataset(ds.getDataverseName(), ds.getDatasetName(),
- getFilesIndexName(ds.getDatasetName()));
+ getFilesIndexName(ds.getDatasetName()), temp);
IFileSplitProvider filesIndexSplitProvider = filesIndexSplitsAndConstraint.first;
ExternalBTreeDataflowHelperFactory filesIndexDataflowHelperFactory = getFilesIndexDataflowHelperFactory(ds,
mergePolicyFactory, mergePolicyFactoryProperties, storageProperties, spec);
@@ -508,7 +510,7 @@
if (isValidIndexName(index.getDatasetName(), index.getIndexName())) {
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> indexSplitsAndConstraint = metadataProvider
.splitProviderAndPartitionConstraintsForDataset(ds.getDataverseName(), ds.getDatasetName(),
- index.getIndexName());
+ index.getIndexName(), temp);
if (index.getIndexType() == IndexType.BTREE) {
btreeDataflowHelperFactories.add(getBTreeDataflowHelperFactory(ds, index, mergePolicyFactory,
mergePolicyFactoryProperties, storageProperties, spec));
@@ -543,7 +545,7 @@
new SecondaryIndexOperationTrackerProvider(ds.getDatasetId()),
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMBTreeIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(),
- ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(ds));
+ ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(ds), true);
}
private static ExternalBTreeWithBuddyDataflowHelperFactory getBTreeDataflowHelperFactory(Dataset ds, Index index,
@@ -554,7 +556,7 @@
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
LSMBTreeWithBuddyIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(), new int[] { index.getKeyFieldNames().size() },
- ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(ds));
+ ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(ds), true);
}
@SuppressWarnings("rawtypes")
@@ -609,7 +611,7 @@
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMRTreeIOOperationCallbackFactory.INSTANCE,
AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
storageProperties.getBloomFilterFalsePositiveRate(), new int[] { index.getKeyFieldNames().size() },
- ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(ds));
+ ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(ds), true);
}
public static JobSpecification buildAbortOp(Dataset ds, List<Index> indexes, AqlMetadataProvider metadataProvider)
@@ -621,9 +623,11 @@
metadataProvider.getMetadataTxnContext());
ILSMMergePolicyFactory mergePolicyFactory = compactionInfo.first;
Map<String, String> mergePolicyFactoryProperties = compactionInfo.second;
+
+ boolean temp = ds.getDatasetDetails().isTemp();
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> filesIndexSplitsAndConstraint = metadataProvider
.splitProviderAndPartitionConstraintsForDataset(ds.getDataverseName(), ds.getDatasetName(),
- getFilesIndexName(ds.getDatasetName()));
+ getFilesIndexName(ds.getDatasetName()), temp);
IFileSplitProvider filesIndexSplitProvider = filesIndexSplitsAndConstraint.first;
ExternalBTreeDataflowHelperFactory filesIndexDataflowHelperFactory = getFilesIndexDataflowHelperFactory(ds,
mergePolicyFactory, mergePolicyFactoryProperties, storageProperties, spec);
@@ -639,7 +643,7 @@
if (isValidIndexName(index.getDatasetName(), index.getIndexName())) {
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> indexSplitsAndConstraint = metadataProvider
.splitProviderAndPartitionConstraintsForDataset(ds.getDataverseName(), ds.getDatasetName(),
- index.getIndexName());
+ index.getIndexName(), temp);
if (index.getIndexType() == IndexType.BTREE) {
btreeDataflowHelperFactories.add(getBTreeDataflowHelperFactory(ds, index, mergePolicyFactory,
mergePolicyFactoryProperties, storageProperties, spec));
@@ -677,9 +681,11 @@
metadataProvider.getMetadataTxnContext());
ILSMMergePolicyFactory mergePolicyFactory = compactionInfo.first;
Map<String, String> mergePolicyFactoryProperties = compactionInfo.second;
+ boolean temp = ds.getDatasetDetails().isTemp();
+
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> filesIndexSplitsAndConstraint = metadataProvider
.splitProviderAndPartitionConstraintsForDataset(ds.getDataverseName(), ds.getDatasetName(),
- getFilesIndexName(ds.getDatasetName()));
+ getFilesIndexName(ds.getDatasetName()), temp);
IFileSplitProvider filesIndexSplitProvider = filesIndexSplitsAndConstraint.first;
ExternalBTreeDataflowHelperFactory filesIndexDataflowHelperFactory = getFilesIndexDataflowHelperFactory(ds,
mergePolicyFactory, mergePolicyFactoryProperties, storageProperties, spec);
@@ -695,7 +701,7 @@
if (isValidIndexName(index.getDatasetName(), index.getIndexName())) {
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> indexSplitsAndConstraint = metadataProvider
.splitProviderAndPartitionConstraintsForDataset(ds.getDataverseName(), ds.getDatasetName(),
- index.getIndexName());
+ index.getIndexName(), temp);
if (index.getIndexType() == IndexType.BTREE) {
btreeDataflowHelperFactories.add(getBTreeDataflowHelperFactory(ds, index, mergePolicyFactory,
mergePolicyFactoryProperties, storageProperties, spec));
@@ -740,7 +746,7 @@
mergePolicyFactory, mergePolicyFactoryProperties, new SecondaryIndexOperationTrackerProvider(
dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
LSMBTreeIOOperationCallbackFactory.INSTANCE, storageProperties.getBloomFilterFalsePositiveRate(),
- ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset));
+ ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset), true);
LSMTreeIndexCompactOperatorDescriptor compactOp = new LSMTreeIndexCompactOperatorDescriptor(spec,
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
secondaryFileSplitProvider, FilesIndexDescription.EXTERNAL_FILE_INDEX_TYPE_TRAITS,
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java
index 53dfc6e..5159277 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java
@@ -94,19 +94,22 @@
String datasetName = indexDropStmt.getDatasetName();
String indexName = indexDropStmt.getIndexName();
JobSpecification spec = JobSpecificationUtils.createJobSpecification();
+ boolean temp = dataset.getDatasetDetails().isTemp();
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadataProvider
- .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, indexName);
+ .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, indexName, temp);
AsterixStorageProperties storageProperties = AsterixAppContextInfo.getInstance().getStorageProperties();
Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(dataset,
metadataProvider.getMetadataTxnContext());
+
+ // The index drop operation should be persistent regardless of temp datasets or permanent dataset.
IndexDropOperatorDescriptor btreeDrop = new IndexDropOperatorDescriptor(spec,
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
splitsAndConstraint.first, new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(
dataset.getDatasetId()), compactionInfo.first, compactionInfo.second,
new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMBTreeIOOperationCallbackFactory.INSTANCE,
- storageProperties.getBloomFilterFalsePositiveRate(), false, null, null, null, null));
+ storageProperties.getBloomFilterFalsePositiveRate(), false, null, null, null, null, true));
AlgebricksPartitionConstraintHelper
.setPartitionConstraintInJobSpec(spec, btreeDrop, splitsAndConstraint.second);
spec.addRoot(btreeDrop);
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeOperationsHelper.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeOperationsHelper.java
index a061451..333d6e7 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeOperationsHelper.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeOperationsHelper.java
@@ -92,12 +92,13 @@
secondaryBTreeFields, secondaryFilterFields);
localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(localResourceMetadata,
LocalResource.LSMBTreeResource);
+ // The index create operation should be persistent regardless of temp datasets or permanent dataset.
indexDataflowHelperFactory = new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(
dataset.getDatasetId()), mergePolicyFactory, mergePolicyFactoryProperties,
new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMBTreeIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(), false, filterTypeTraits, filterCmpFactories,
- secondaryBTreeFields, secondaryFilterFields);
+ secondaryBTreeFields, secondaryFilterFields, true);
} else {
// External dataset local resource and dataflow helper
int[] buddyBreeFields = new int[] { numSecondaryKeys };
@@ -111,7 +112,7 @@
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
LSMBTreeWithBuddyIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(), buddyBreeFields,
- ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset));
+ ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset), true);
}
TreeIndexCreateOperatorDescriptor secondaryIndexCreateOp = new TreeIndexCreateOperatorDescriptor(spec,
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
@@ -163,7 +164,7 @@
dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
LSMBTreeWithBuddyIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(), new int[] { numSecondaryKeys },
- ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset));
+ ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset), true);
IOperatorDescriptor root;
if (externalFiles != null) {
// Transaction load
@@ -216,6 +217,7 @@
ExternalSortOperatorDescriptor sortOp = createSortOp(spec, secondaryComparatorFactories, secondaryRecDesc);
AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
+ boolean temp = dataset.getDatasetDetails().isTemp();
// Create secondary BTree bulk load op.
TreeIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp = createTreeIndexBulkLoadOp(
spec,
@@ -226,7 +228,7 @@
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
LSMBTreeIOOperationCallbackFactory.INSTANCE, storageProperties
.getBloomFilterFalsePositiveRate(), false, filterTypeTraits, filterCmpFactories,
- secondaryBTreeFields, secondaryFilterFields), GlobalConfig.DEFAULT_TREE_FILL_FACTOR);
+ secondaryBTreeFields, secondaryFilterFields, !temp), GlobalConfig.DEFAULT_TREE_FILL_FACTOR);
AlgebricksMetaOperatorDescriptor metaOp = new AlgebricksMetaOperatorDescriptor(spec, 1, 0,
new IPushRuntimeFactory[] { new SinkRuntimeFactory() }, new RecordDescriptor[] { secondaryRecDesc });
@@ -247,6 +249,7 @@
}
}
+ @Override
protected int getNumSecondaryKeys() {
return numSecondaryKeys;
}
@@ -256,6 +259,7 @@
JobSpecification spec = JobSpecificationUtils.createJobSpecification();
AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
+ boolean temp = dataset.getDatasetDetails().isTemp();
LSMTreeIndexCompactOperatorDescriptor compactOp;
if (dataset.getDatasetType() == DatasetType.INTERNAL) {
compactOp = new LSMTreeIndexCompactOperatorDescriptor(spec,
@@ -267,7 +271,7 @@
dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
LSMBTreeIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(), false, filterTypeTraits,
- filterCmpFactories, secondaryBTreeFields, secondaryFilterFields),
+ filterCmpFactories, secondaryBTreeFields, secondaryFilterFields, !temp),
NoOpOperationCallbackFactory.INSTANCE);
} else {
// External dataset
@@ -280,7 +284,7 @@
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
LSMBTreeWithBuddyIOOperationCallbackFactory.INSTANCE, storageProperties
.getBloomFilterFalsePositiveRate(), new int[] { numSecondaryKeys },
- ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset)),
+ ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset), true),
NoOpOperationCallbackFactory.INSTANCE);
}
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, compactOp,
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexOperationsHelper.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexOperationsHelper.java
index cccb461..07c8bab 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexOperationsHelper.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexOperationsHelper.java
@@ -148,8 +148,8 @@
}
public static SecondaryIndexOperationsHelper createIndexOperationsHelper(IndexType indexType, String dataverseName,
- String datasetName, String indexName, List<List<String>> secondaryKeyFields, List<IAType> secondaryKeyTypes,
- boolean isEnforced, int gramLength, AqlMetadataProvider metadataProvider,
+ String datasetName, String indexName, List<List<String>> secondaryKeyFields,
+ List<IAType> secondaryKeyTypes, boolean isEnforced, int gramLength, AqlMetadataProvider metadataProvider,
PhysicalOptimizationConfig physOptConf, ARecordType recType, ARecordType enforcedType)
throws AsterixException, AlgebricksException {
IAsterixPropertiesProvider asterixPropertiesProvider = AsterixAppContextInfo.getInstance();
@@ -198,12 +198,13 @@
if (dataset == null) {
throw new AsterixException("Unknown dataset " + datasetName);
}
+ boolean temp = dataset.getDatasetDetails().isTemp();
itemType = aRecType;
enforcedItemType = enforcedType;
payloadSerde = AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(itemType);
numSecondaryKeys = secondaryKeyFields.size();
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> secondarySplitsAndConstraint = metadataProvider
- .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, secondaryIndexName);
+ .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, secondaryIndexName, temp);
secondaryFileSplitProvider = secondarySplitsAndConstraint.first;
secondaryPartitionConstraint = secondarySplitsAndConstraint.second;
@@ -219,7 +220,7 @@
numPrimaryKeys = DatasetUtils.getPartitioningKeys(dataset).size();
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> primarySplitsAndConstraint = metadataProvider
- .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, datasetName);
+ .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, datasetName, temp);
primaryFileSplitProvider = primarySplitsAndConstraint.first;
primaryPartitionConstraint = primarySplitsAndConstraint.second;
setPrimaryRecDescAndComparators();
@@ -291,9 +292,9 @@
primaryRecDesc = new RecordDescriptor(primaryRecFields, primaryTypeTraits);
}
- protected abstract void setSecondaryRecDescAndComparators(IndexType indexType, List<List<String>> secondaryKeyFields,
- List<IAType> secondaryKeyTypes, int gramLength, AqlMetadataProvider metadataProvider)
- throws AlgebricksException, AsterixException;
+ protected abstract void setSecondaryRecDescAndComparators(IndexType indexType,
+ List<List<String>> secondaryKeyFields, List<IAType> secondaryKeyTypes, int gramLength,
+ AqlMetadataProvider metadataProvider) throws AlgebricksException, AsterixException;
protected AbstractOperatorDescriptor createDummyKeyProviderOp(JobSpecification spec) throws AsterixException,
AlgebricksException {
@@ -330,9 +331,10 @@
IJobletEventListenerFactory jobEventListenerFactory = new JobEventListenerFactory(jobId, isWriteTransaction);
spec.setJobletEventListenerFactory(jobEventListenerFactory);
- ISearchOperationCallbackFactory searchCallbackFactory = new PrimaryIndexInstantSearchOperationCallbackFactory(
- jobId, dataset.getDatasetId(), primaryBloomFilterKeyFields, txnSubsystemProvider,
- ResourceType.LSM_BTREE);
+ boolean temp = dataset.getDatasetDetails().isTemp();
+ ISearchOperationCallbackFactory searchCallbackFactory = temp ? NoOpOperationCallbackFactory.INSTANCE
+ : new PrimaryIndexInstantSearchOperationCallbackFactory(jobId, dataset.getDatasetId(),
+ primaryBloomFilterKeyFields, txnSubsystemProvider, ResourceType.LSM_BTREE);
AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
BTreeSearchOperatorDescriptor primarySearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
@@ -343,7 +345,7 @@
dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
LSMBTreeIOOperationCallbackFactory.INSTANCE, storageProperties
.getBloomFilterFalsePositiveRate(), true, filterTypeTraits, filterCmpFactories,
- primaryBTreeFields, primaryFilterFields), false, false, null,
+ primaryBTreeFields, primaryFilterFields, !temp), false, false, null,
searchCallbackFactory, null, null);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, primarySearchOp,
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexOperationsHelper.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexOperationsHelper.java
index 579af2f..74c4256 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexOperationsHelper.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexOperationsHelper.java
@@ -203,6 +203,7 @@
}
+ @Override
protected int getNumSecondaryKeys() {
return numTokenKeyPairFields - numPrimaryKeys;
}
@@ -330,6 +331,7 @@
private IIndexDataflowHelperFactory createDataflowHelperFactory() {
AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
+ boolean temp = dataset.getDatasetDetails().isTemp();
if (!isPartitioned) {
return new LSMInvertedIndexDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(
dataset.getDatasetId()), mergePolicyFactory, mergePolicyFactoryProperties,
@@ -338,7 +340,7 @@
LSMInvertedIndexIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(), invertedIndexFields, filterTypeTraits,
filterCmpFactories, secondaryFilterFields, secondaryFilterFieldsForNonBulkLoadOps,
- invertedIndexFieldsForNonBulkLoadOps);
+ invertedIndexFieldsForNonBulkLoadOps, !temp);
} else {
return new PartitionedLSMInvertedIndexDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(
dataset.getDatasetId()), mergePolicyFactory, mergePolicyFactoryProperties,
@@ -347,7 +349,7 @@
LSMInvertedIndexIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(), invertedIndexFields, filterTypeTraits,
filterCmpFactories, secondaryFilterFields, secondaryFilterFieldsForNonBulkLoadOps,
- invertedIndexFieldsForNonBulkLoadOps);
+ invertedIndexFieldsForNonBulkLoadOps, !temp);
}
}
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeOperationsHelper.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeOperationsHelper.java
index 8be9f59..0e49fb9 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeOperationsHelper.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeOperationsHelper.java
@@ -92,7 +92,7 @@
JobSpecification spec = JobSpecificationUtils.createJobSpecification();
AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
-
+ boolean temp = dataset.getDatasetDetails().isTemp();
IIndexDataflowHelperFactory indexDataflowHelperFactory;
ILocalResourceFactoryProvider localResourceFactoryProvider;
if (dataset.getDatasetType() == DatasetType.INTERNAL) {
@@ -112,7 +112,7 @@
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMRTreeIOOperationCallbackFactory.INSTANCE,
AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
storageProperties.getBloomFilterFalsePositiveRate(), rtreeFields, primaryKeyFields,
- filterTypeTraits, filterCmpFactories, secondaryFilterFields);
+ filterTypeTraits, filterCmpFactories, secondaryFilterFields, !temp);
} else {
// External dataset
// Prepare a LocalResourceMetadata which will be stored in NC's local resource repository
@@ -131,7 +131,7 @@
LSMRTreeIOOperationCallbackFactory.INSTANCE, AqlMetadataProvider.proposeLinearizer(keyType,
secondaryComparatorFactories.length), storageProperties.getBloomFilterFalsePositiveRate(),
new int[] { numNestedSecondaryKeyFields },
- ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset));
+ ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset), true);
}
TreeIndexCreateOperatorDescriptor secondaryIndexCreateOp = new TreeIndexCreateOperatorDescriptor(spec,
@@ -146,6 +146,7 @@
return spec;
}
+ @Override
protected int getNumSecondaryKeys() {
return numNestedSecondaryKeyFields;
}
@@ -233,6 +234,7 @@
@Override
public JobSpecification buildLoadingJobSpec() throws AsterixException, AlgebricksException {
JobSpecification spec = JobSpecificationUtils.createJobSpecification();
+ boolean temp = dataset.getDatasetDetails().isTemp();
if (dataset.getDatasetType() == DatasetType.INTERNAL) {
// Create dummy key provider for feeding the primary index scan.
AbstractOperatorDescriptor keyProviderOp = createDummyKeyProviderOp(spec);
@@ -273,7 +275,7 @@
LSMRTreeIOOperationCallbackFactory.INSTANCE, AqlMetadataProvider.proposeLinearizer(keyType,
secondaryComparatorFactories.length), storageProperties
.getBloomFilterFalsePositiveRate(), rtreeFields, primaryKeyFields,
- filterTypeTraits, filterCmpFactories, secondaryFilterFields),
+ filterTypeTraits, filterCmpFactories, secondaryFilterFields, !temp),
GlobalConfig.DEFAULT_TREE_FILL_FACTOR);
AlgebricksMetaOperatorDescriptor metaOp = new AlgebricksMetaOperatorDescriptor(spec, 1, 0,
new IPushRuntimeFactory[] { new SinkRuntimeFactory() }, new RecordDescriptor[] {});
@@ -325,7 +327,7 @@
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMRTreeIOOperationCallbackFactory.INSTANCE,
AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
storageProperties.getBloomFilterFalsePositiveRate(), new int[] { numNestedSecondaryKeyFields },
- ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset));
+ ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset), true);
// Create secondary RTree bulk load op.
IOperatorDescriptor root;
AbstractTreeIndexOperatorDescriptor secondaryBulkLoadOp;
@@ -364,6 +366,7 @@
JobSpecification spec = JobSpecificationUtils.createJobSpecification();
AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
+ boolean temp = dataset.getDatasetDetails().isTemp();
LSMTreeIndexCompactOperatorDescriptor compactOp;
if (dataset.getDatasetType() == DatasetType.INTERNAL) {
compactOp = new LSMTreeIndexCompactOperatorDescriptor(spec,
@@ -377,7 +380,7 @@
LSMRTreeIOOperationCallbackFactory.INSTANCE, AqlMetadataProvider.proposeLinearizer(keyType,
secondaryComparatorFactories.length),
storageProperties.getBloomFilterFalsePositiveRate(), rtreeFields, primaryKeyFields,
- filterTypeTraits, filterCmpFactories, secondaryFilterFields),
+ filterTypeTraits, filterCmpFactories, secondaryFilterFields, !temp),
NoOpOperationCallbackFactory.INSTANCE);
} else {
// External dataset
@@ -392,7 +395,7 @@
LSMRTreeIOOperationCallbackFactory.INSTANCE, AqlMetadataProvider.proposeLinearizer(keyType,
secondaryComparatorFactories.length), storageProperties
.getBloomFilterFalsePositiveRate(), new int[] { numNestedSecondaryKeyFields },
- ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset)),
+ ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset), true),
NoOpOperationCallbackFactory.INSTANCE);
}
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
index 2f71dcf..89158a0 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
@@ -14,12 +14,17 @@
*/
package edu.uci.ics.asterix.hyracks.bootstrap;
+import java.io.File;
import java.rmi.server.UnicastRemoteObject;
import java.util.HashMap;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
+import org.kohsuke.args4j.CmdLineException;
+import org.kohsuke.args4j.CmdLineParser;
+import org.kohsuke.args4j.Option;
+
import edu.uci.ics.asterix.api.common.AsterixAppRuntimeContext;
import edu.uci.ics.asterix.common.api.AsterixThreadFactory;
import edu.uci.ics.asterix.common.api.IAsterixAppRuntimeContext;
@@ -42,11 +47,6 @@
import edu.uci.ics.hyracks.api.lifecycle.ILifeCycleComponentManager;
import edu.uci.ics.hyracks.api.lifecycle.LifeCycleComponentManager;
-import org.kohsuke.args4j.Argument;
-import org.kohsuke.args4j.CmdLineException;
-import org.kohsuke.args4j.CmdLineParser;
-import org.kohsuke.args4j.Option;
-
public class NCApplicationEntryPoint implements INCApplicationEntryPoint {
private static final Logger LOGGER = Logger.getLogger(NCApplicationEntryPoint.class.getName());
@@ -60,15 +60,14 @@
private boolean stopInitiated = false;
private SystemState systemState = SystemState.NEW_UNIVERSE;
private final long NON_SHARP_CHECKPOINT_TARGET_LSN = -1;
-
+
@Override
public void start(INCApplicationContext ncAppCtx, String[] args) throws Exception {
CmdLineParser parser = new CmdLineParser(this);
try {
parser.parseArgument(args);
- }
- catch (CmdLineException e) {
+ } catch (CmdLineException e) {
System.err.println(e.getMessage());
System.err.println("Usage:");
parser.printUsage(System.err);
@@ -212,8 +211,20 @@
proxy.setMetadataNode(stub);
}
+ // Reclaim storage for temporary datasets.
+ String[] ioDevices = AsterixClusterProperties.INSTANCE.getIODevices(nodeId);
+ String[] nodeStores = metadataProperties.getStores().get(nodeId);
+ int numIoDevices = AsterixClusterProperties.INSTANCE.getNumberOfIODevices(nodeId);
+ for (int j = 0; j < nodeStores.length; j++) {
+ for (int k = 0; k < numIoDevices; k++) {
+ File f = new File(ioDevices[k] + File.separator + nodeStores[j] + File.separator + "temp");
+ f.delete();
+ }
+ }
+
// TODO
// reclaim storage for orphaned index artifacts in NCs.
+
}
private void updateOnNodeJoin() {
diff --git a/asterix-app/src/test/resources/metadata/queries/basic/temp_dataset/temp_dataset.1.ddl.aql b/asterix-app/src/test/resources/metadata/queries/basic/temp_dataset/temp_dataset.1.ddl.aql
new file mode 100644
index 0000000..5a21955
--- /dev/null
+++ b/asterix-app/src/test/resources/metadata/queries/basic/temp_dataset/temp_dataset.1.ddl.aql
@@ -0,0 +1,34 @@
+/*
+ * Description : Create a temporary dataset and verify the dataset does not exist in the metadata store
+ * Expected Res : Success
+ * Date : March 12 2015
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type LineItemType as closed {
+ l_orderkey: int64,
+ l_partkey: int64,
+ l_suppkey: int64,
+ l_linenumber: int64,
+ l_quantity: double,
+ l_extendedprice: double,
+ l_discount: double,
+ l_tax: double,
+ l_returnflag: string,
+ l_linestatus: string,
+ l_shipdate: string,
+ l_commitdate: string,
+ l_receiptdate: string,
+ l_shipinstruct: string,
+ l_shipmode: string,
+ l_comment: string
+}
+
+create temporary dataset LineItem(LineItemType)
+ primary key l_orderkey, l_linenumber;
+
+
diff --git a/asterix-app/src/test/resources/metadata/queries/basic/temp_dataset/temp_dataset.2.update.aql b/asterix-app/src/test/resources/metadata/queries/basic/temp_dataset/temp_dataset.2.update.aql
new file mode 100644
index 0000000..3431e80
--- /dev/null
+++ b/asterix-app/src/test/resources/metadata/queries/basic/temp_dataset/temp_dataset.2.update.aql
@@ -0,0 +1,6 @@
+/*
+ * Description : Create dataverse and drop that dataverse and verify dataverse entries in metadata
+ * Expected Res : Success
+ * Date : Sep 17 2012
+ */
+
diff --git a/asterix-app/src/test/resources/metadata/queries/basic/temp_dataset/temp_dataset.3.query.aql b/asterix-app/src/test/resources/metadata/queries/basic/temp_dataset/temp_dataset.3.query.aql
new file mode 100644
index 0000000..b2e2dbd
--- /dev/null
+++ b/asterix-app/src/test/resources/metadata/queries/basic/temp_dataset/temp_dataset.3.query.aql
@@ -0,0 +1,11 @@
+/*
+ * Description : Create dataverse and drop that dataverse and verify dataverse entries in metadata
+ * Expected Res : Success
+ * Date : Sep 17 2012
+ */
+
+
+for $l in dataset('Metadata.Dataset')
+where $l.DataverseName = 'test'
+return $l;
+
diff --git a/asterix-app/src/test/resources/metadata/results/basic/temp_dataset/temp_dataset.1.adm b/asterix-app/src/test/resources/metadata/results/basic/temp_dataset/temp_dataset.1.adm
new file mode 100644
index 0000000..e3b97f5
--- /dev/null
+++ b/asterix-app/src/test/resources/metadata/results/basic/temp_dataset/temp_dataset.1.adm
@@ -0,0 +1 @@
+[ ]
diff --git a/asterix-app/src/test/resources/metadata/testsuite.xml b/asterix-app/src/test/resources/metadata/testsuite.xml
index c977392..a3c7a38 100644
--- a/asterix-app/src/test/resources/metadata/testsuite.xml
+++ b/asterix-app/src/test/resources/metadata/testsuite.xml
@@ -229,6 +229,11 @@
<output-dir compare="Text">issue_363_temporal_sec_key_4</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="basic">
+ <compilation-unit name="temp_dataset">
+ <output-dir compare="Text">temp_dataset</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="exception">
<test-case FilePath="exception">
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/delete-from-loaded-dataset-with-index/delete-from-loaded-dataset-with-index.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/delete-from-loaded-dataset-with-index/delete-from-loaded-dataset-with-index.1.ddl.aql
new file mode 100644
index 0000000..0d8257b
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/delete-from-loaded-dataset-with-index/delete-from-loaded-dataset-with-index.1.ddl.aql
@@ -0,0 +1,34 @@
+/*
+ * Description : Delete a portion of data from a loaded temporary dataset with indexes.
+ * Expected Result : Success
+ * Date : 27th March 2015
+ *
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type LineItemType as closed {
+ l_orderkey: int64,
+ l_partkey: int64,
+ l_suppkey: int64,
+ l_linenumber: int64,
+ l_quantity: int64,
+ l_extendedprice: double,
+ l_discount: double,
+ l_tax: double,
+ l_returnflag: string,
+ l_linestatus: string,
+ l_shipdate: string,
+ l_commitdate: string,
+ l_receiptdate: string,
+ l_shipinstruct: string,
+ l_shipmode: string,
+ l_comment: string
+}
+
+create temporary dataset LineItem(LineItemType)
+ primary key l_orderkey, l_linenumber;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/delete-from-loaded-dataset-with-index/delete-from-loaded-dataset-with-index.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/delete-from-loaded-dataset-with-index/delete-from-loaded-dataset-with-index.2.update.aql
new file mode 100644
index 0000000..ccdd424
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/delete-from-loaded-dataset-with-index/delete-from-loaded-dataset-with-index.2.update.aql
@@ -0,0 +1,13 @@
+/*
+ * Description : Delete a portion of data from a loaded temporary dataset with indexes.
+ * Expected Result : Success
+ * Date : 27th March 2015
+ *
+ */
+
+use dataverse test;
+
+load dataset LineItem
+using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/tpch0.001/lineitem.tbl"),("format"="delimited-text"),("delimiter"="|")) pre-sorted;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/delete-from-loaded-dataset-with-index/delete-from-loaded-dataset-with-index.3.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/delete-from-loaded-dataset-with-index/delete-from-loaded-dataset-with-index.3.ddl.aql
new file mode 100644
index 0000000..c77ba30
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/delete-from-loaded-dataset-with-index/delete-from-loaded-dataset-with-index.3.ddl.aql
@@ -0,0 +1,12 @@
+/*
+ * Description : Delete a portion of data from a loaded temporary dataset with indexes.
+ * Expected Result : Success
+ * Date : 27th March 2015
+ *
+ */
+
+use dataverse test;
+
+create index idx_LineItem_partkey on LineItem(l_linenumber);
+create index idx_LineItem_suppkey on LineItem(l_suppkey);
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/delete-from-loaded-dataset-with-index/delete-from-loaded-dataset-with-index.4.update.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/delete-from-loaded-dataset-with-index/delete-from-loaded-dataset-with-index.4.update.aql
new file mode 100644
index 0000000..6faf9b0
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/delete-from-loaded-dataset-with-index/delete-from-loaded-dataset-with-index.4.update.aql
@@ -0,0 +1,10 @@
+/*
+ * Description : Delete a portion of data from a loaded temporary dataset with indexes.
+ * Expected Result : Success
+ * Date : 27th March 2015
+ *
+ */
+
+use dataverse test;
+
+delete $l from dataset LineItem where $l.l_suppkey>=2 or $l.l_linenumber>1;
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/delete-from-loaded-dataset-with-index/delete-from-loaded-dataset-with-index.5.query.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/delete-from-loaded-dataset-with-index/delete-from-loaded-dataset-with-index.5.query.aql
new file mode 100644
index 0000000..602406d
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/delete-from-loaded-dataset-with-index/delete-from-loaded-dataset-with-index.5.query.aql
@@ -0,0 +1,13 @@
+/*
+ * Description : Delete a portion of data from a loaded temporary dataset with indexes.
+ * Expected Result : Success
+ * Date : 27th March 2015
+ *
+ */
+
+use dataverse test;
+
+for $c in dataset('LineItem')
+where $c.l_suppkey<150
+order by $c.l_orderkey, $c.l_linenumber
+return $c
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/delete-from-loaded-dataset/delete-from-loaded-dataset.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/delete-from-loaded-dataset/delete-from-loaded-dataset.1.ddl.aql
new file mode 100644
index 0000000..ad408bd
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/delete-from-loaded-dataset/delete-from-loaded-dataset.1.ddl.aql
@@ -0,0 +1,34 @@
+/*
+ * Description : Delete a portion of data from a loaded temporary dataset.
+ * Expected Result : Success
+ * Date : 27th March 2015
+ *
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type LineItemType as closed {
+ l_orderkey: int64,
+ l_partkey: int64,
+ l_suppkey: int64,
+ l_linenumber: int64,
+ l_quantity: int64,
+ l_extendedprice: double,
+ l_discount: double,
+ l_tax: double,
+ l_returnflag: string,
+ l_linestatus: string,
+ l_shipdate: string,
+ l_commitdate: string,
+ l_receiptdate: string,
+ l_shipinstruct: string,
+ l_shipmode: string,
+ l_comment: string
+}
+
+create temporary dataset LineItem(LineItemType)
+ primary key l_orderkey, l_linenumber;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/delete-from-loaded-dataset/delete-from-loaded-dataset.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/delete-from-loaded-dataset/delete-from-loaded-dataset.2.update.aql
new file mode 100644
index 0000000..1a74334
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/delete-from-loaded-dataset/delete-from-loaded-dataset.2.update.aql
@@ -0,0 +1,15 @@
+/*
+ * Description : Delete a portion of data from a loaded temporary dataset.
+ * Expected Result : Success
+ * Date : 27th March 2015
+ *
+ */
+
+use dataverse test;
+
+load dataset LineItem
+using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/tpch0.001/lineitem.tbl"),("format"="delimited-text"),("delimiter"="|")) pre-sorted;
+
+delete $l from dataset LineItem where $l.l_orderkey>=10;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/delete-from-loaded-dataset/delete-from-loaded-dataset.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/delete-from-loaded-dataset/delete-from-loaded-dataset.3.query.aql
new file mode 100644
index 0000000..3f86d13
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/delete-from-loaded-dataset/delete-from-loaded-dataset.3.query.aql
@@ -0,0 +1,12 @@
+/*
+ * Description : Delete a portion of data from a loaded temporary dataset.
+ * Expected Result : Success
+ * Date : 27th March 2015
+ *
+ */
+
+use dataverse test;
+
+for $c in dataset('LineItem')
+order by $c.l_orderkey, $c.l_linenumber
+return $c
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/drop-empty-secondary-indexes/drop-empty-secondary-indexes.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/drop-empty-secondary-indexes/drop-empty-secondary-indexes.1.ddl.aql
new file mode 100644
index 0000000..5e7539c
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/drop-empty-secondary-indexes/drop-empty-secondary-indexes.1.ddl.aql
@@ -0,0 +1,57 @@
+/*
+ * Description : Drop empty secondary indexes for a temporary dataset.
+ * Expected Result : Success
+ * Date : 27th March 2015
+ *
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+create type Name as open {
+first : string,
+last : string
+}
+
+create type Person as open {
+name : Name
+}
+
+create type TestType as open {
+id : int32,
+name : string,
+locn : point,
+zip : string,
+person : Person
+}
+
+create temporary dataset t1(TestType) primary key id;
+
+create index rtree_index_point on t1(locn) type rtree;
+
+create index rtree_index_point_open on t1(open_locn:point) type rtree enforced;
+
+create index keyWD_indx on t1(name) type keyword;
+
+create index keyWD_indx_open on t1(nickname:string) type keyword enforced;
+
+create index secndIndx on t1(zip);
+
+create index nested on t1(person.name.first);
+
+create index secndIndx_open on t1(address:string) enforced;
+
+drop index t1.rtree_index_point;
+
+drop index t1.rtree_index_point_open;
+
+drop index t1.keyWD_indx;
+
+drop index t1.keyWD_indx_open;
+
+drop index t1.secndIndx;
+
+drop index t1.nested;
+
+drop index t1.secndIndx_open;
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/drop-empty-secondary-indexes/drop-empty-secondary-indexes.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/drop-empty-secondary-indexes/drop-empty-secondary-indexes.2.update.aql
new file mode 100644
index 0000000..934e859
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/drop-empty-secondary-indexes/drop-empty-secondary-indexes.2.update.aql
@@ -0,0 +1,7 @@
+/*
+ * Description : Drop empty secondary indexes for a temporary dataset.
+ * Expected Result : Success
+ * Date : 27th March 2015
+ *
+ */
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/drop-empty-secondary-indexes/drop-empty-secondary-indexes.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/drop-empty-secondary-indexes/drop-empty-secondary-indexes.3.query.aql
new file mode 100644
index 0000000..edeb750
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/drop-empty-secondary-indexes/drop-empty-secondary-indexes.3.query.aql
@@ -0,0 +1,10 @@
+/*
+ * Description : Drop empty secondary indexes for a temporary dataset.
+ * Expected Result : Success
+ * Date : 27th March 2015
+ *
+ */
+
+for $l in dataset('Metadata.Index')
+where $l.IsPrimary=false and $l.DataverseName='test'
+return $l;
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/drop-index/drop-index.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/drop-index/drop-index.1.ddl.aql
new file mode 100644
index 0000000..2471d23
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/drop-index/drop-index.1.ddl.aql
@@ -0,0 +1,33 @@
+/*
+ * Description : Drop secondary indexes for a temporary dataset.
+ * Expected Result : Success
+ * Date : 27th March 2015
+ *
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type Schema as closed {
+unique1: int64,
+unique2: int64,
+two: int64,
+four: int64,
+ten: int64,
+twenty: int64,
+onePercent: int64,
+tenPercent: int64,
+twentyPercent: int64,
+fiftyPercent: int64,
+unique3: int64,
+evenOnePercent: int64,
+oddOnePercent: int64,
+stringu1: string,
+stringu2: string,
+string4: string
+}
+
+create temporary dataset t1(Schema) primary key unique2;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/drop-index/drop-index.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/drop-index/drop-index.2.update.aql
new file mode 100644
index 0000000..6f41a30
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/drop-index/drop-index.2.update.aql
@@ -0,0 +1,14 @@
+/*
+ * Description : Drop secondary indexes for a temporary dataset.
+ * Expected Result : Success
+ * Date : 27th March 2015
+ *
+ */
+
+use dataverse test;
+
+// Load data
+load dataset t1
+using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/wisc/onektup.adm"),("format"="delimited-text"),("delimiter"="|")) pre-sorted;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/drop-index/drop-index.3.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/drop-index/drop-index.3.ddl.aql
new file mode 100644
index 0000000..65c6993
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/drop-index/drop-index.3.ddl.aql
@@ -0,0 +1,17 @@
+/*
+ * Description : Drop secondary indexes for a temporary dataset.
+ * Expected Result : Success
+ * Date : 27th March 2015
+ *
+ */
+
+use dataverse test;
+
+// create secondary indexes
+create index idx_t1_str1 on t1(stringu1);
+create index idx_t1_unique1 on t1(unique1);
+
+// drop secondary indexes
+drop index t1.idx_t1_str1;
+drop index t1.idx_t1_unique1;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/drop-index/drop-index.4.query.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/drop-index/drop-index.4.query.aql
new file mode 100644
index 0000000..ef7ee6f
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/drop-index/drop-index.4.query.aql
@@ -0,0 +1,12 @@
+/*
+ * Description : Drop secondary indexes for a temporary dataset.
+ * Expected Result : Success
+ * Date : 27th March 2015
+ *
+ */
+
+use dataverse test;
+
+for $a in dataset('t1')
+where $a.unique1 > 10 and $a.stringu1="DGAAAAXXXXXXXXXXXXXXXXXXX"
+return $a
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/empty-load-with-index/empty-load-with-index.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/empty-load-with-index/empty-load-with-index.1.ddl.aql
new file mode 100644
index 0000000..1b00ea0
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/empty-load-with-index/empty-load-with-index.1.ddl.aql
@@ -0,0 +1,34 @@
+/*
+ * Description : Load a temporary dataset with previously created empty indexes.
+ * Expected Result : Success
+ * Date : March 27 2015
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type LineItemType as closed {
+ l_orderkey: int64,
+ l_partkey: int64,
+ l_suppkey: int64,
+ l_linenumber: int64,
+ l_quantity: double,
+ l_extendedprice: double,
+ l_discount: double,
+ l_tax: double,
+ l_returnflag: string,
+ l_linestatus: string,
+ l_shipdate: string,
+ l_commitdate: string,
+ l_receiptdate: string,
+ l_shipinstruct: string,
+ l_shipmode: string,
+ l_comment: string
+}
+
+create temporary dataset LineItem(LineItemType)
+ primary key l_orderkey, l_linenumber;
+
+create index part_index on LineItem(l_partkey);
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/empty-load-with-index/empty-load-with-index.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/empty-load-with-index/empty-load-with-index.2.update.aql
new file mode 100644
index 0000000..36d9336
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/empty-load-with-index/empty-load-with-index.2.update.aql
@@ -0,0 +1,15 @@
+/*
+ * Description : Load a temporary dataset with previously created empty indexes.
+ * Expected Result : Success
+ * Date : March 27 2015
+ */
+
+use dataverse test;
+
+load dataset LineItem
+using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/empty.adm"),("format"="delimited-text"),("delimiter"="|")) pre-sorted;
+
+load dataset LineItem
+using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/tpch0.001/lineitem.tbl"),("format"="delimited-text"),("delimiter"="|")) pre-sorted;
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/empty-load-with-index/empty-load-with-index.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/empty-load-with-index/empty-load-with-index.3.query.aql
new file mode 100644
index 0000000..776fde6
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/empty-load-with-index/empty-load-with-index.3.query.aql
@@ -0,0 +1,12 @@
+/*
+ * Description : Load a temporary dataset with previously created empty indexes.
+ * Expected Result : Success
+ * Date : March 27 2015
+ */
+
+use dataverse test;
+
+for $c in dataset('LineItem')
+order by $c.l_orderkey, $c.l_linenumber
+limit 1
+return $c
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-index/insert-and-scan-dataset-with-index.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-index/insert-and-scan-dataset-with-index.1.ddl.aql
new file mode 100644
index 0000000..9458135
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-index/insert-and-scan-dataset-with-index.1.ddl.aql
@@ -0,0 +1,21 @@
+/*
+ * Description : This test is intended to test inserting into a temporary dataset that has a secondary index and scan
+ * the data at the same time where we insert a materializing to prevent the possibility of deadlatch.
+ * Expected Result : Success
+ * Date : March 27 2015
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+create type test.Emp as closed {
+id: int64,
+fname: string,
+lname: string,
+age: int64,
+dept: string
+}
+
+create temporary dataset test.employee(Emp) primary key id;
+
+create index idx_employee_first_name on test.employee(fname);
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-index/insert-and-scan-dataset-with-index.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-index/insert-and-scan-dataset-with-index.2.update.aql
new file mode 100644
index 0000000..4249951
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-index/insert-and-scan-dataset-with-index.2.update.aql
@@ -0,0 +1,24 @@
+/*
+ * Description : This test is intended to test inserting into a temporary dataset that has a secondary index and scan
+ * the data at the same time where we insert a materializing to prevent the possibility of deadlatch.
+ * Expected Result : Success
+ * Date : March 28 2015
+ */
+
+use dataverse test;
+
+load dataset test.employee
+using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/names.adm"),("format"="delimited-text"),("delimiter"="|"));
+
+
+insert into dataset test.employee (
+for $x in dataset test.employee
+return {
+ "id": $x.id + 10000,
+ "fname": $x.fname,
+ "lname": $x.lname,
+ "age": $x.age,
+ "dept": $x.dept
+}
+);
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-index/insert-and-scan-dataset-with-index.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-index/insert-and-scan-dataset-with-index.3.query.aql
new file mode 100644
index 0000000..6c0e325
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-index/insert-and-scan-dataset-with-index.3.query.aql
@@ -0,0 +1,13 @@
+/*
+ * Description : This test is intended to test inserting into a temporary dataset that has a secondary index and scan
+ * the data at the same time where we insert a materializing to prevent the possibility of dead latch.
+ * Expected Result : Success
+ * Date : March 27 2015
+ */
+
+
+use dataverse test;
+
+for $l in dataset('test.employee')
+order by $l.id
+return $l
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset/insert-and-scan-dataset.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset/insert-and-scan-dataset.1.ddl.aql
new file mode 100644
index 0000000..2633d33
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset/insert-and-scan-dataset.1.ddl.aql
@@ -0,0 +1,18 @@
+/*
+ * Description : This test is intended to test inserting into a temporary dataset and scan it at the same time
+ * where we insert a materializing to prevent the possibility of deadlatch.
+ * Expected Result : Success
+ * Date : July 11 2013
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type myDataType as open {
+ id: int64
+}
+
+create temporary dataset myData(myDataType)
+ primary key id;
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset/insert-and-scan-dataset.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset/insert-and-scan-dataset.2.update.aql
new file mode 100644
index 0000000..3652be6
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset/insert-and-scan-dataset.2.update.aql
@@ -0,0 +1,21 @@
+/*
+ * Test case Name : insert-and-scan-dataset.aql
+ * Description : This test is intended to test inserting into a temporary dataset and scan it at the same time
+ * where we insert a materializing to prevent the possibility of deadlatch.
+ * Expected Result : Success
+ * Date : March 27 2015
+ */
+
+use dataverse test;
+
+load dataset myData
+using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/odd-numbers.adm"),("format"="adm"))pre-sorted;
+
+
+insert into dataset myData (
+for $x in dataset myData
+return {
+ "id": $x.id + 1
+}
+);
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset/insert-and-scan-dataset.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset/insert-and-scan-dataset.3.query.aql
new file mode 100644
index 0000000..58b5a9c
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset/insert-and-scan-dataset.3.query.aql
@@ -0,0 +1,14 @@
+/*
+ * Test case Name : insert-and-scan-dataset.aql
+ * Description : This test is intended to test inserting into a temporary dataset and scan it at the same time
+ * where we insert a materializing to prevent the possibility of deadlatch.
+ * Expected Result : Success
+ * Date : March 27 2015
+ *
+ */
+
+use dataverse test;
+
+for $c in dataset('myData')
+order by $c.id
+return $c
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/q01_pricing_summary_report_nt/q01_pricing_summary_report_nt.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/q01_pricing_summary_report_nt/q01_pricing_summary_report_nt.1.ddl.aql
new file mode 100644
index 0000000..5c9846c
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/q01_pricing_summary_report_nt/q01_pricing_summary_report_nt.1.ddl.aql
@@ -0,0 +1,33 @@
+/*
+ * Description : Test a read query over a temporary dataset.
+ * Expected Result : Success
+ * Date : March 27 2015
+ */
+
+drop dataverse tpch if exists;
+create dataverse tpch;
+
+use dataverse tpch;
+
+create type LineItemType as closed {
+ l_orderkey: int64,
+ l_partkey: int64,
+ l_suppkey: int64,
+ l_linenumber: int64,
+ l_quantity: double,
+ l_extendedprice: double,
+ l_discount: double,
+ l_tax: double,
+ l_returnflag: string,
+ l_linestatus: string,
+ l_shipdate: string,
+ l_commitdate: string,
+ l_receiptdate: string,
+ l_shipinstruct: string,
+ l_shipmode: string,
+ l_comment: string
+}
+
+create temporary dataset LineItem(LineItemType)
+ primary key l_orderkey, l_linenumber;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/q01_pricing_summary_report_nt/q01_pricing_summary_report_nt.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/q01_pricing_summary_report_nt/q01_pricing_summary_report_nt.2.update.aql
new file mode 100644
index 0000000..2752fc1
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/q01_pricing_summary_report_nt/q01_pricing_summary_report_nt.2.update.aql
@@ -0,0 +1,12 @@
+/*
+ * Description : Test a read query over a temporary dataset.
+ * Expected Result : Success
+ * Date : March 27 2015
+ */
+
+use dataverse tpch;
+
+load dataset LineItem
+using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/tpch0.001/lineitem.tbl"),("format"="delimited-text"),("delimiter"="|")) pre-sorted;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/q01_pricing_summary_report_nt/q01_pricing_summary_report_nt.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/q01_pricing_summary_report_nt/q01_pricing_summary_report_nt.3.query.aql
new file mode 100644
index 0000000..6143a73
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/q01_pricing_summary_report_nt/q01_pricing_summary_report_nt.3.query.aql
@@ -0,0 +1,28 @@
+/*
+ * Description : Test a read query over a temporary dataset.
+ * Expected Result : Success
+ * Date : March 27 2015
+ */
+
+use dataverse tpch;
+set import-private-functions 'true';
+
+for $l in dataset('LineItem')
+where $l.l_shipdate <= '1998-09-02'
+/*+ hash*/
+group by $l_returnflag := $l.l_returnflag,
+ $l_linestatus := $l.l_linestatus
+ with $l
+order by $l_returnflag, $l_linestatus
+return {
+ "l_returnflag": $l_returnflag,
+ "l_linestatus": $l_linestatus,
+ "sum_qty": sum(for $i in $l return $i.l_quantity),
+ "sum_base_price": sum(for $i in $l return $i.l_extendedprice),
+ "sum_disc_price": sum(for $i in $l return $i.l_extendedprice * (1 - $i.l_discount)),
+ "sum_charge": sum(for $i in $l return $i.l_extendedprice * (1 - $i.l_discount) * (1 + $i.l_tax)),
+ "ave_qty": avg(for $i in $l return $i.l_quantity),
+ "ave_price": avg(for $i in $l return $i.l_extendedprice),
+ "ave_disc": avg(for $i in $l return $i.l_discount),
+ "count_order": count($l)
+}
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-delete-rtree-secondary-index/scan-delete-rtree-secondary-index.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-delete-rtree-secondary-index/scan-delete-rtree-secondary-index.1.ddl.aql
new file mode 100644
index 0000000..6513778
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-delete-rtree-secondary-index/scan-delete-rtree-secondary-index.1.ddl.aql
@@ -0,0 +1,26 @@
+/*
+ * Description : Delete tuples from a temporary dataset with a R-tree index.
+ * Expected Result : Success
+ * Date : March 27 2015
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type MyRecord as closed {
+ id: int64,
+ point: point,
+ kwds: string,
+ line1: line,
+ line2: line,
+ poly1: polygon,
+ poly2: polygon,
+ rec: rectangle,
+ circle: circle
+}
+
+create temporary dataset MyData(MyRecord)
+ primary key id;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-delete-rtree-secondary-index/scan-delete-rtree-secondary-index.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-delete-rtree-secondary-index/scan-delete-rtree-secondary-index.2.update.aql
new file mode 100644
index 0000000..6152e42
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-delete-rtree-secondary-index/scan-delete-rtree-secondary-index.2.update.aql
@@ -0,0 +1,12 @@
+/*
+ * Description : Delete tuples from a temporary dataset with a R-tree index.
+ * Expected Result : Success
+ * Date : March 27 2015
+ */
+
+use dataverse test;
+
+load dataset MyData
+using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/spatial/spatialData.json"),("format"="adm")) pre-sorted;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-delete-rtree-secondary-index/scan-delete-rtree-secondary-index.3.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-delete-rtree-secondary-index/scan-delete-rtree-secondary-index.3.ddl.aql
new file mode 100644
index 0000000..ef984ad
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-delete-rtree-secondary-index/scan-delete-rtree-secondary-index.3.ddl.aql
@@ -0,0 +1,10 @@
+/*
+ * Description : Delete tuples from a temporary dataset with a R-tree index.
+ * Expected Result : Success
+ * Date : March 27 2015
+ */
+
+use dataverse test;
+
+create index rtree_index_point on MyData(point) type rtree;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-delete-rtree-secondary-index/scan-delete-rtree-secondary-index.4.update.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-delete-rtree-secondary-index/scan-delete-rtree-secondary-index.4.update.aql
new file mode 100644
index 0000000..583287c
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-delete-rtree-secondary-index/scan-delete-rtree-secondary-index.4.update.aql
@@ -0,0 +1,10 @@
+/*
+ * Description : Delete tuples from a temporary dataset with a R-tree index.
+ * Expected Result : Success
+ * Date : March 27 2015
+ */
+
+use dataverse test;
+
+delete $m from dataset MyData where $m.id>10;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-delete-rtree-secondary-index/scan-delete-rtree-secondary-index.5.query.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-delete-rtree-secondary-index/scan-delete-rtree-secondary-index.5.query.aql
new file mode 100644
index 0000000..0244046
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-delete-rtree-secondary-index/scan-delete-rtree-secondary-index.5.query.aql
@@ -0,0 +1,12 @@
+/*
+ * Description : Delete tuples from a temporary dataset with a R-tree index.
+ * Expected Result : Success
+ * Date : March 27 2015
+ */
+
+use dataverse test;
+
+for $o in dataset('MyData')
+where spatial-intersect($o.point, create-polygon([0.0,1.0,0.0,4.0,12.0,4.0,12.0,1.0]))
+order by $o.id
+return {"id":$o.id}
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-persistent-to-temp/scan-insert-persistent-to-temp.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-persistent-to-temp/scan-insert-persistent-to-temp.1.ddl.aql
new file mode 100644
index 0000000..b58455d
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-persistent-to-temp/scan-insert-persistent-to-temp.1.ddl.aql
@@ -0,0 +1,35 @@
+/*
+ * Description : Port data from a persistent dataset into a temporary dataset.
+ * Both datasets have R-tree indexes.
+ * Expected Result : Success
+ * Date : March 27 2015
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type MyRecord as closed {
+ id: int64,
+ point: point,
+ kwds: string,
+ line1: line,
+ line2: line,
+ poly1: polygon,
+ poly2: polygon,
+ rec: rectangle,
+ circle: circle
+}
+
+create type MyMiniRecord as closed {
+ id: int64,
+ point: point
+}
+
+create dataset MyData(MyRecord)
+ primary key id;
+
+create temporary dataset MyMiniData(MyMiniRecord)
+ primary key id;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-persistent-to-temp/scan-insert-persistent-to-temp.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-persistent-to-temp/scan-insert-persistent-to-temp.2.update.aql
new file mode 100644
index 0000000..e8fd954
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-persistent-to-temp/scan-insert-persistent-to-temp.2.update.aql
@@ -0,0 +1,17 @@
+/*
+ * Description : Port data from a temporary dataset into yet-another temporary dataset.
+ * Both datasets have R-tree indexes.
+ * Expected Result : Success
+ * Date : March 27 2015
+ */
+
+use dataverse test;
+
+load dataset MyData
+using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/spatial/spatialData.json"),("format"="adm")) pre-sorted;
+
+load dataset MyMiniData
+using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/spatial/spatialData0.json"),("format"="adm")) pre-sorted;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-persistent-to-temp/scan-insert-persistent-to-temp.3.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-persistent-to-temp/scan-insert-persistent-to-temp.3.ddl.aql
new file mode 100644
index 0000000..ce6d632
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-persistent-to-temp/scan-insert-persistent-to-temp.3.ddl.aql
@@ -0,0 +1,12 @@
+/*
+ * Description : Port data from a temporary dataset into yet-another temporary dataset.
+ * Both datasets have R-tree indexes.
+ * Expected Result : Success
+ * Date : March 27 2015
+ */
+
+use dataverse test;
+
+create index rtree_index_point_0 on MyData(point) type rtree;
+create index rtree_index_point on MyMiniData(point) type rtree;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-persistent-to-temp/scan-insert-persistent-to-temp.4.update.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-persistent-to-temp/scan-insert-persistent-to-temp.4.update.aql
new file mode 100644
index 0000000..28c6021
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-persistent-to-temp/scan-insert-persistent-to-temp.4.update.aql
@@ -0,0 +1,18 @@
+/*
+ * Description : Port data from a temporary dataset into yet-another temporary dataset.
+ * Both datasets have R-tree indexes.
+ * Expected Result : Success
+ * Date : March 27 2015
+ */
+
+use dataverse test;
+
+insert into dataset MyMiniData
+(
+ for $m in dataset('MyData')
+ return {
+ "id": $m.id,
+ "point": $m.point
+ }
+);
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-persistent-to-temp/scan-insert-persistent-to-temp.5.query.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-persistent-to-temp/scan-insert-persistent-to-temp.5.query.aql
new file mode 100644
index 0000000..aa45997
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-persistent-to-temp/scan-insert-persistent-to-temp.5.query.aql
@@ -0,0 +1,13 @@
+/*
+ * Description : Port data from a temporary dataset into yet-another temporary dataset.
+ * Both datasets have R-tree indexes.
+ * Expected Result : Success
+ * Date : March 27 2015
+ */
+
+use dataverse test;
+
+for $o in dataset('MyMiniData')
+where spatial-intersect($o.point, create-polygon([0.0,1.0,0.0,4.0,12.0,4.0,12.0,1.0]))
+order by $o.id
+return {"id":$o.id}
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-rtree-secondary-index/scan-insert-rtree-secondary-index.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-rtree-secondary-index/scan-insert-rtree-secondary-index.1.ddl.aql
new file mode 100644
index 0000000..958f1fe
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-rtree-secondary-index/scan-insert-rtree-secondary-index.1.ddl.aql
@@ -0,0 +1,35 @@
+/*
+ * Description : Port data from a temporary dataset into yet-another temporary dataset.
+ * Both datasets have R-tree indexes.
+ * Expected Result : Success
+ * Date : March 27 2015
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type MyRecord as closed {
+ id: int64,
+ point: point,
+ kwds: string,
+ line1: line,
+ line2: line,
+ poly1: polygon,
+ poly2: polygon,
+ rec: rectangle,
+ circle: circle
+}
+
+create type MyMiniRecord as closed {
+ id: int64,
+ point: point
+}
+
+create temporary dataset MyData(MyRecord)
+ primary key id;
+
+create temporary dataset MyMiniData(MyMiniRecord)
+ primary key id;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-rtree-secondary-index/scan-insert-rtree-secondary-index.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-rtree-secondary-index/scan-insert-rtree-secondary-index.2.update.aql
new file mode 100644
index 0000000..e8fd954
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-rtree-secondary-index/scan-insert-rtree-secondary-index.2.update.aql
@@ -0,0 +1,17 @@
+/*
+ * Description : Port data from a temporary dataset into yet-another temporary dataset.
+ * Both datasets have R-tree indexes.
+ * Expected Result : Success
+ * Date : March 27 2015
+ */
+
+use dataverse test;
+
+load dataset MyData
+using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/spatial/spatialData.json"),("format"="adm")) pre-sorted;
+
+load dataset MyMiniData
+using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/spatial/spatialData0.json"),("format"="adm")) pre-sorted;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-rtree-secondary-index/scan-insert-rtree-secondary-index.3.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-rtree-secondary-index/scan-insert-rtree-secondary-index.3.ddl.aql
new file mode 100644
index 0000000..ce6d632
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-rtree-secondary-index/scan-insert-rtree-secondary-index.3.ddl.aql
@@ -0,0 +1,12 @@
+/*
+ * Description : Port data from a temporary dataset into yet-another temporary dataset.
+ * Both datasets have R-tree indexes.
+ * Expected Result : Success
+ * Date : March 27 2015
+ */
+
+use dataverse test;
+
+create index rtree_index_point_0 on MyData(point) type rtree;
+create index rtree_index_point on MyMiniData(point) type rtree;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-rtree-secondary-index/scan-insert-rtree-secondary-index.4.update.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-rtree-secondary-index/scan-insert-rtree-secondary-index.4.update.aql
new file mode 100644
index 0000000..28c6021
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-rtree-secondary-index/scan-insert-rtree-secondary-index.4.update.aql
@@ -0,0 +1,18 @@
+/*
+ * Description : Port data from a temporary dataset into yet-another temporary dataset.
+ * Both datasets have R-tree indexes.
+ * Expected Result : Success
+ * Date : March 27 2015
+ */
+
+use dataverse test;
+
+insert into dataset MyMiniData
+(
+ for $m in dataset('MyData')
+ return {
+ "id": $m.id,
+ "point": $m.point
+ }
+);
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-rtree-secondary-index/scan-insert-rtree-secondary-index.5.query.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-rtree-secondary-index/scan-insert-rtree-secondary-index.5.query.aql
new file mode 100644
index 0000000..aa45997
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-rtree-secondary-index/scan-insert-rtree-secondary-index.5.query.aql
@@ -0,0 +1,13 @@
+/*
+ * Description : Port data from a temporary dataset into yet-another temporary dataset.
+ * Both datasets have R-tree indexes.
+ * Expected Result : Success
+ * Date : March 27 2015
+ */
+
+use dataverse test;
+
+for $o in dataset('MyMiniData')
+where spatial-intersect($o.point, create-polygon([0.0,1.0,0.0,4.0,12.0,4.0,12.0,1.0]))
+order by $o.id
+return {"id":$o.id}
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-temp-to-persistent/scan-insert-temp-to-persistent.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-temp-to-persistent/scan-insert-temp-to-persistent.1.ddl.aql
new file mode 100644
index 0000000..87091f2
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-temp-to-persistent/scan-insert-temp-to-persistent.1.ddl.aql
@@ -0,0 +1,35 @@
+/*
+ * Description : Port data from a temporary dataset into a persistent dataset.
+ * Both datasets have R-tree indexes.
+ * Expected Result : Success
+ * Date : March 27 2015
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type MyRecord as closed {
+ id: int64,
+ point: point,
+ kwds: string,
+ line1: line,
+ line2: line,
+ poly1: polygon,
+ poly2: polygon,
+ rec: rectangle,
+ circle: circle
+}
+
+create type MyMiniRecord as closed {
+ id: int64,
+ point: point
+}
+
+create temporary dataset MyData(MyRecord)
+ primary key id;
+
+create dataset MyMiniData(MyMiniRecord)
+ primary key id;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-temp-to-persistent/scan-insert-temp-to-persistent.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-temp-to-persistent/scan-insert-temp-to-persistent.2.update.aql
new file mode 100644
index 0000000..e8fd954
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-temp-to-persistent/scan-insert-temp-to-persistent.2.update.aql
@@ -0,0 +1,17 @@
+/*
+ * Description : Port data from a temporary dataset into yet-another temporary dataset.
+ * Both datasets have R-tree indexes.
+ * Expected Result : Success
+ * Date : March 27 2015
+ */
+
+use dataverse test;
+
+load dataset MyData
+using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/spatial/spatialData.json"),("format"="adm")) pre-sorted;
+
+load dataset MyMiniData
+using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/spatial/spatialData0.json"),("format"="adm")) pre-sorted;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-temp-to-persistent/scan-insert-temp-to-persistent.3.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-temp-to-persistent/scan-insert-temp-to-persistent.3.ddl.aql
new file mode 100644
index 0000000..ce6d632
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-temp-to-persistent/scan-insert-temp-to-persistent.3.ddl.aql
@@ -0,0 +1,12 @@
+/*
+ * Description : Port data from a temporary dataset into yet-another temporary dataset.
+ * Both datasets have R-tree indexes.
+ * Expected Result : Success
+ * Date : March 27 2015
+ */
+
+use dataverse test;
+
+create index rtree_index_point_0 on MyData(point) type rtree;
+create index rtree_index_point on MyMiniData(point) type rtree;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-temp-to-persistent/scan-insert-temp-to-persistent.4.update.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-temp-to-persistent/scan-insert-temp-to-persistent.4.update.aql
new file mode 100644
index 0000000..28c6021
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-temp-to-persistent/scan-insert-temp-to-persistent.4.update.aql
@@ -0,0 +1,18 @@
+/*
+ * Description : Port data from a temporary dataset into yet-another temporary dataset.
+ * Both datasets have R-tree indexes.
+ * Expected Result : Success
+ * Date : March 27 2015
+ */
+
+use dataverse test;
+
+insert into dataset MyMiniData
+(
+ for $m in dataset('MyData')
+ return {
+ "id": $m.id,
+ "point": $m.point
+ }
+);
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-temp-to-persistent/scan-insert-temp-to-persistent.5.query.aql b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-temp-to-persistent/scan-insert-temp-to-persistent.5.query.aql
new file mode 100644
index 0000000..aa45997
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/scan-insert-temp-to-persistent/scan-insert-temp-to-persistent.5.query.aql
@@ -0,0 +1,13 @@
+/*
+ * Description : Port data from a temporary dataset into yet-another temporary dataset.
+ * Both datasets have R-tree indexes.
+ * Expected Result : Success
+ * Date : March 27 2015
+ */
+
+use dataverse test;
+
+for $o in dataset('MyMiniData')
+where spatial-intersect($o.point, create-polygon([0.0,1.0,0.0,4.0,12.0,4.0,12.0,1.0]))
+order by $o.id
+return {"id":$o.id}
diff --git a/asterix-app/src/test/resources/runtimets/results/temp-dataset/compact-dataset-and-its-indexes/compact-dataset-and-its-indexes.1.adm b/asterix-app/src/test/resources/runtimets/results/temp-dataset/compact-dataset-and-its-indexes/compact-dataset-and-its-indexes.1.adm
new file mode 100644
index 0000000..38303d7
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/temp-dataset/compact-dataset-and-its-indexes/compact-dataset-and-its-indexes.1.adm
@@ -0,0 +1,168 @@
+[ { "l_orderkey": 36, "l_partkey": 120, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 42, "l_extendedprice": 42845.04d, "l_discount": 0.09d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-02-03", "l_commitdate": "1996-01-21", "l_receiptdate": "1996-02-23", "l_shipinstruct": "COLLECT COD", "l_shipmode": "SHIP", "l_comment": " careful courts. special " }
+, { "l_orderkey": 68, "l_partkey": 8, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 3, "l_extendedprice": 2724.0d, "l_discount": 0.05d, "l_tax": 0.02d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-07-04", "l_commitdate": "1998-06-05", "l_receiptdate": "1998-07-21", "l_shipinstruct": "NONE", "l_shipmode": "RAIL", "l_comment": "fully special instructions cajole. furious" }
+, { "l_orderkey": 162, "l_partkey": 190, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 2, "l_extendedprice": 2180.38d, "l_discount": 0.02d, "l_tax": 0.01d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-09-02", "l_commitdate": "1995-06-17", "l_receiptdate": "1995-09-08", "l_shipinstruct": "COLLECT COD", "l_shipmode": "FOB", "l_comment": "es! final somas integrate" }
+, { "l_orderkey": 192, "l_partkey": 98, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 23, "l_extendedprice": 22956.07d, "l_discount": 0.0d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-02-05", "l_commitdate": "1998-02-06", "l_receiptdate": "1998-03-07", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "AIR", "l_comment": "ly pending theodolites haggle quickly fluf" }
+, { "l_orderkey": 197, "l_partkey": 99, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 39, "l_extendedprice": 38964.51d, "l_discount": 0.02d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-07-21", "l_commitdate": "1995-07-01", "l_receiptdate": "1995-08-14", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "AIR", "l_comment": "press accounts. daringly sp" }
+, { "l_orderkey": 227, "l_partkey": 166, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 19, "l_extendedprice": 20257.04d, "l_discount": 0.05d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-12-10", "l_commitdate": "1996-01-30", "l_receiptdate": "1995-12-26", "l_shipinstruct": "NONE", "l_shipmode": "RAIL", "l_comment": "s cajole furiously a" }
+, { "l_orderkey": 290, "l_partkey": 6, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 35, "l_extendedprice": 31710.0d, "l_discount": 0.01d, "l_tax": 0.02d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-04-01", "l_commitdate": "1994-02-05", "l_receiptdate": "1994-04-27", "l_shipinstruct": "NONE", "l_shipmode": "MAIL", "l_comment": "ove the final foxes detect slyly fluffily" }
+, { "l_orderkey": 325, "l_partkey": 159, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 34, "l_extendedprice": 36011.1d, "l_discount": 0.09d, "l_tax": 0.04d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1993-10-28", "l_commitdate": "1993-12-13", "l_receiptdate": "1993-11-17", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "MAIL", "l_comment": "ly bold deposits. always iron" }
+, { "l_orderkey": 355, "l_partkey": 114, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 31, "l_extendedprice": 31437.41d, "l_discount": 0.09d, "l_tax": 0.07d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-07-13", "l_commitdate": "1994-08-18", "l_receiptdate": "1994-07-18", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "FOB", "l_comment": "y unusual, ironic" }
+, { "l_orderkey": 389, "l_partkey": 190, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 2, "l_extendedprice": 2180.38d, "l_discount": 0.09d, "l_tax": 0.0d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-04-13", "l_commitdate": "1994-04-10", "l_receiptdate": "1994-04-25", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": "fts. courts eat blithely even dependenc" }
+, { "l_orderkey": 391, "l_partkey": 122, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 14, "l_extendedprice": 14309.68d, "l_discount": 0.09d, "l_tax": 0.02d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1995-02-11", "l_commitdate": "1995-02-03", "l_receiptdate": "1995-02-13", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "TRUCK", "l_comment": " escapades sleep furiously about " }
+, { "l_orderkey": 417, "l_partkey": 40, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 39, "l_extendedprice": 36661.56d, "l_discount": 0.01d, "l_tax": 0.02d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-05-31", "l_commitdate": "1994-05-02", "l_receiptdate": "1994-06-06", "l_shipinstruct": "NONE", "l_shipmode": "SHIP", "l_comment": "y regular requests wake along " }
+, { "l_orderkey": 453, "l_partkey": 198, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 45, "l_extendedprice": 49418.55d, "l_discount": 0.01d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-06-30", "l_commitdate": "1997-08-20", "l_receiptdate": "1997-07-19", "l_shipinstruct": "COLLECT COD", "l_shipmode": "REG AIR", "l_comment": "ifts wake carefully." }
+, { "l_orderkey": 485, "l_partkey": 150, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 50, "l_extendedprice": 52507.5d, "l_discount": 0.01d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-03-28", "l_commitdate": "1997-05-26", "l_receiptdate": "1997-04-18", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "MAIL", "l_comment": "iously quick excuses. carefully final f" }
+, { "l_orderkey": 545, "l_partkey": 170, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 4, "l_extendedprice": 4280.68d, "l_discount": 0.02d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-02-23", "l_commitdate": "1995-12-16", "l_receiptdate": "1996-03-21", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "FOB", "l_comment": ", ironic grouches cajole over" }
+, { "l_orderkey": 581, "l_partkey": 64, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 41, "l_extendedprice": 39526.46d, "l_discount": 0.09d, "l_tax": 0.07d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-05-26", "l_commitdate": "1997-04-06", "l_receiptdate": "1997-06-10", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "MAIL", "l_comment": "nts. quickly" }
+, { "l_orderkey": 647, "l_partkey": 17, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 41, "l_extendedprice": 37597.41d, "l_discount": 0.08d, "l_tax": 0.08d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-11-19", "l_commitdate": "1997-09-24", "l_receiptdate": "1997-12-15", "l_shipinstruct": "COLLECT COD", "l_shipmode": "REG AIR", "l_comment": "r instructions. quickly unusu" }
+, { "l_orderkey": 704, "l_partkey": 190, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 40, "l_extendedprice": 43607.6d, "l_discount": 0.05d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-01-30", "l_commitdate": "1997-01-10", "l_receiptdate": "1997-02-20", "l_shipinstruct": "COLLECT COD", "l_shipmode": "AIR", "l_comment": "ggle quickly. r" }
+, { "l_orderkey": 738, "l_partkey": 198, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 34, "l_extendedprice": 37338.46d, "l_discount": 0.0d, "l_tax": 0.06d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-06-09", "l_commitdate": "1993-04-15", "l_receiptdate": "1993-07-09", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "TRUCK", "l_comment": "s against the ironic exc" }
+, { "l_orderkey": 773, "l_partkey": 100, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 5, "l_extendedprice": 5000.5d, "l_discount": 0.06d, "l_tax": 0.04d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1993-11-21", "l_commitdate": "1993-12-19", "l_receiptdate": "1993-12-21", "l_shipinstruct": "COLLECT COD", "l_shipmode": "MAIL", "l_comment": "ar requests. regular, thin packages u" }
+, { "l_orderkey": 800, "l_partkey": 72, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 38, "l_extendedprice": 36938.66d, "l_discount": 0.0d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-07-21", "l_commitdate": "1998-09-25", "l_receiptdate": "1998-08-07", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "TRUCK", "l_comment": "according to the bold, final dependencies " }
+, { "l_orderkey": 900, "l_partkey": 199, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 44, "l_extendedprice": 48364.36d, "l_discount": 0.01d, "l_tax": 0.06d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-12-15", "l_commitdate": "1994-12-03", "l_receiptdate": "1994-12-27", "l_shipinstruct": "COLLECT COD", "l_shipmode": "MAIL", "l_comment": " detect quick" }
+, { "l_orderkey": 931, "l_partkey": 40, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 18, "l_extendedprice": 16920.72d, "l_discount": 0.0d, "l_tax": 0.05d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1993-04-04", "l_commitdate": "1993-01-11", "l_receiptdate": "1993-04-13", "l_shipinstruct": "NONE", "l_shipmode": "RAIL", "l_comment": "slyly ironic re" }
+, { "l_orderkey": 932, "l_partkey": 44, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 41, "l_extendedprice": 38705.64d, "l_discount": 0.01d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-06-05", "l_commitdate": "1997-07-22", "l_receiptdate": "1997-06-26", "l_shipinstruct": "COLLECT COD", "l_shipmode": "RAIL", "l_comment": "foxes. ironic pl" }
+, { "l_orderkey": 965, "l_partkey": 108, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 20, "l_extendedprice": 20162.0d, "l_discount": 0.04d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "F", "l_shipdate": "1995-06-16", "l_commitdate": "1995-07-20", "l_receiptdate": "1995-07-06", "l_shipinstruct": "COLLECT COD", "l_shipmode": "MAIL", "l_comment": "kly. carefully pending requ" }
+, { "l_orderkey": 995, "l_partkey": 173, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 15, "l_extendedprice": 16097.55d, "l_discount": 0.08d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-06-30", "l_commitdate": "1995-08-04", "l_receiptdate": "1995-07-27", "l_shipinstruct": "NONE", "l_shipmode": "REG AIR", "l_comment": "uses. fluffily fina" }
+, { "l_orderkey": 1025, "l_partkey": 150, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 36, "l_extendedprice": 37805.4d, "l_discount": 0.03d, "l_tax": 0.04d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1995-05-15", "l_commitdate": "1995-07-05", "l_receiptdate": "1995-06-10", "l_shipinstruct": "COLLECT COD", "l_shipmode": "FOB", "l_comment": "e unusual, regular instr" }
+, { "l_orderkey": 1027, "l_partkey": 156, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 43, "l_extendedprice": 45414.45d, "l_discount": 0.07d, "l_tax": 0.08d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1992-06-17", "l_commitdate": "1992-08-28", "l_receiptdate": "1992-07-10", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "MAIL", "l_comment": "oxes. carefully regular deposits" }
+, { "l_orderkey": 1155, "l_partkey": 70, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 4, "l_extendedprice": 3880.28d, "l_discount": 0.09d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-10-19", "l_commitdate": "1997-12-09", "l_receiptdate": "1997-11-02", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "SHIP", "l_comment": "ic foxes according to the carefully final " }
+, { "l_orderkey": 1185, "l_partkey": 72, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 8, "l_extendedprice": 7776.56d, "l_discount": 0.01d, "l_tax": 0.06d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1992-12-05", "l_commitdate": "1992-10-05", "l_receiptdate": "1992-12-28", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "MAIL", "l_comment": "ely according to the furiously regular r" }
+, { "l_orderkey": 1216, "l_partkey": 97, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 8, "l_extendedprice": 7976.72d, "l_discount": 0.03d, "l_tax": 0.04d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-02-01", "l_commitdate": "1993-03-06", "l_receiptdate": "1993-02-08", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "TRUCK", "l_comment": " of the carefully express" }
+, { "l_orderkey": 1223, "l_partkey": 100, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 28, "l_extendedprice": 28002.8d, "l_discount": 0.1d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-08-07", "l_commitdate": "1996-07-24", "l_receiptdate": "1996-08-13", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "MAIL", "l_comment": " quickly ironic requests. furious" }
+, { "l_orderkey": 1381, "l_partkey": 144, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 47, "l_extendedprice": 49074.58d, "l_discount": 0.08d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-09-22", "l_commitdate": "1998-08-12", "l_receiptdate": "1998-10-12", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "AIR", "l_comment": "ly ironic deposits" }
+, { "l_orderkey": 1409, "l_partkey": 99, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 23, "l_extendedprice": 22979.07d, "l_discount": 0.01d, "l_tax": 0.03d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1993-04-18", "l_commitdate": "1993-02-25", "l_receiptdate": "1993-05-06", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "FOB", "l_comment": "ions. slyly ironic packages wake quick" }
+, { "l_orderkey": 1445, "l_partkey": 100, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 24, "l_extendedprice": 24002.4d, "l_discount": 0.01d, "l_tax": 0.0d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1995-02-21", "l_commitdate": "1995-02-22", "l_receiptdate": "1995-03-18", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "SHIP", "l_comment": "al accounts use furiously a" }
+, { "l_orderkey": 1477, "l_partkey": 72, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 31, "l_extendedprice": 30134.17d, "l_discount": 0.0d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-12-16", "l_commitdate": "1997-09-30", "l_receiptdate": "1997-12-17", "l_shipinstruct": "COLLECT COD", "l_shipmode": "RAIL", "l_comment": " requests. fluffily final " }
+, { "l_orderkey": 1540, "l_partkey": 173, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 38, "l_extendedprice": 40780.46d, "l_discount": 0.03d, "l_tax": 0.01d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1992-09-30", "l_commitdate": "1992-10-27", "l_receiptdate": "1992-10-12", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "SHIP", "l_comment": " final grouches bo" }
+, { "l_orderkey": 1568, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 36, "l_extendedprice": 35643.24d, "l_discount": 0.02d, "l_tax": 0.03d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-05-31", "l_commitdate": "1997-04-22", "l_receiptdate": "1997-06-21", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": "platelets-- furiously sly excu" }
+, { "l_orderkey": 1605, "l_partkey": 142, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 47, "l_extendedprice": 48980.58d, "l_discount": 0.0d, "l_tax": 0.01d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-04-29", "l_commitdate": "1998-06-12", "l_receiptdate": "1998-05-20", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "AIR", "l_comment": ". carefully r" }
+, { "l_orderkey": 1607, "l_partkey": 190, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 2, "l_extendedprice": 2180.38d, "l_discount": 0.02d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-01-11", "l_commitdate": "1996-02-15", "l_receiptdate": "1996-01-19", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "MAIL", "l_comment": "packages haggle. regular requests boost s" }
+, { "l_orderkey": 1635, "l_partkey": 71, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 3, "l_extendedprice": 2913.21d, "l_discount": 0.06d, "l_tax": 0.08d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-03-13", "l_commitdate": "1997-03-25", "l_receiptdate": "1997-03-27", "l_shipinstruct": "COLLECT COD", "l_shipmode": "FOB", "l_comment": " quickly ironic r" }
+, { "l_orderkey": 1700, "l_partkey": 140, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 38, "l_extendedprice": 39525.32d, "l_discount": 0.04d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-10-03", "l_commitdate": "1996-07-27", "l_receiptdate": "1996-10-22", "l_shipinstruct": "NONE", "l_shipmode": "RAIL", "l_comment": "ular dependencies engage slyly " }
+, { "l_orderkey": 1796, "l_partkey": 10, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 28, "l_extendedprice": 25480.28d, "l_discount": 0.08d, "l_tax": 0.04d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1992-12-01", "l_commitdate": "1993-01-01", "l_receiptdate": "1992-12-24", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "FOB", "l_comment": "y quickly ironic accounts." }
+, { "l_orderkey": 1825, "l_partkey": 156, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 43, "l_extendedprice": 45414.45d, "l_discount": 0.05d, "l_tax": 0.05d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-02-18", "l_commitdate": "1994-02-19", "l_receiptdate": "1994-03-02", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": " accounts breach fluffily spe" }
+, { "l_orderkey": 1827, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 47, "l_extendedprice": 46534.23d, "l_discount": 0.0d, "l_tax": 0.01d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-08-01", "l_commitdate": "1996-08-07", "l_receiptdate": "1996-08-23", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": ". pending courts about the even e" }
+, { "l_orderkey": 1893, "l_partkey": 99, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 43, "l_extendedprice": 42960.87d, "l_discount": 0.1d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-01-25", "l_commitdate": "1998-01-06", "l_receiptdate": "1998-02-14", "l_shipinstruct": "COLLECT COD", "l_shipmode": "SHIP", "l_comment": "he carefully regular " }
+, { "l_orderkey": 1924, "l_partkey": 73, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 7, "l_extendedprice": 6811.49d, "l_discount": 0.06d, "l_tax": 0.07d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-01-01", "l_commitdate": "1996-12-02", "l_receiptdate": "1997-01-08", "l_shipinstruct": "COLLECT COD", "l_shipmode": "SHIP", "l_comment": "osits. even accounts nag furious" }
+, { "l_orderkey": 1953, "l_partkey": 128, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 25, "l_extendedprice": 25703.0d, "l_discount": 0.07d, "l_tax": 0.06d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-01-07", "l_commitdate": "1994-01-28", "l_receiptdate": "1994-01-29", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": "ular, regular i" }
+, { "l_orderkey": 1985, "l_partkey": 28, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 33, "l_extendedprice": 30624.66d, "l_discount": 0.1d, "l_tax": 0.03d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-12-04", "l_commitdate": "1994-11-01", "l_receiptdate": "1994-12-05", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "FOB", "l_comment": "s are express packages. pendin" }
+, { "l_orderkey": 1988, "l_partkey": 72, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 36, "l_extendedprice": 34994.52d, "l_discount": 0.09d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-01-21", "l_commitdate": "1995-11-24", "l_receiptdate": "1996-01-27", "l_shipinstruct": "NONE", "l_shipmode": "RAIL", "l_comment": "gular theodolites. " }
+, { "l_orderkey": 2048, "l_partkey": 35, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 7, "l_extendedprice": 6545.21d, "l_discount": 0.06d, "l_tax": 0.01d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-12-07", "l_commitdate": "1994-01-31", "l_receiptdate": "1994-01-05", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "REG AIR", "l_comment": "lent platelets boost deposits. carefully sp" }
+, { "l_orderkey": 2086, "l_partkey": 60, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 22, "l_extendedprice": 21121.32d, "l_discount": 0.03d, "l_tax": 0.07d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-12-04", "l_commitdate": "1994-12-16", "l_receiptdate": "1994-12-20", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "RAIL", "l_comment": "idly busy acc" }
+, { "l_orderkey": 2118, "l_partkey": 160, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 24, "l_extendedprice": 25443.84d, "l_discount": 0.1d, "l_tax": 0.03d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-01-06", "l_commitdate": "1996-12-14", "l_receiptdate": "1997-01-14", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": "about the slyly bold depende" }
+, { "l_orderkey": 2183, "l_partkey": 71, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 29, "l_extendedprice": 28161.03d, "l_discount": 0.05d, "l_tax": 0.01d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-07-21", "l_commitdate": "1996-08-24", "l_receiptdate": "1996-08-15", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": "ly unusual deposits sleep carefully" }
+, { "l_orderkey": 2211, "l_partkey": 48, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 25, "l_extendedprice": 23701.0d, "l_discount": 0.04d, "l_tax": 0.01d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-10-09", "l_commitdate": "1994-08-04", "l_receiptdate": "1994-11-03", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": "deas. carefully special theodolites along" }
+, { "l_orderkey": 2215, "l_partkey": 73, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 33, "l_extendedprice": 32111.31d, "l_discount": 0.0d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-07-19", "l_commitdate": "1996-08-10", "l_receiptdate": "1996-07-30", "l_shipinstruct": "COLLECT COD", "l_shipmode": "RAIL", "l_comment": "dolites cajole b" }
+, { "l_orderkey": 2272, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 18, "l_extendedprice": 17821.62d, "l_discount": 0.04d, "l_tax": 0.0d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-08-01", "l_commitdate": "1993-07-06", "l_receiptdate": "1993-08-25", "l_shipinstruct": "NONE", "l_shipmode": "MAIL", "l_comment": "ons along the blithely e" }
+, { "l_orderkey": 2342, "l_partkey": 42, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 12, "l_extendedprice": 11304.48d, "l_discount": 0.0d, "l_tax": 0.08d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-07-31", "l_commitdate": "1996-07-26", "l_receiptdate": "1996-08-14", "l_shipinstruct": "NONE", "l_shipmode": "TRUCK", "l_comment": "print blithely even deposits. carefull" }
+, { "l_orderkey": 2343, "l_partkey": 110, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 27, "l_extendedprice": 27272.97d, "l_discount": 0.0d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-11-10", "l_commitdate": "1995-11-17", "l_receiptdate": "1995-12-10", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "SHIP", "l_comment": "old theodolites." }
+, { "l_orderkey": 2439, "l_partkey": 164, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 2, "l_extendedprice": 2128.32d, "l_discount": 0.09d, "l_tax": 0.03d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-04-14", "l_commitdate": "1997-06-11", "l_receiptdate": "1997-05-09", "l_shipinstruct": "COLLECT COD", "l_shipmode": "MAIL", "l_comment": "courts boos" }
+, { "l_orderkey": 2469, "l_partkey": 166, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 11, "l_extendedprice": 11727.76d, "l_discount": 0.0d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-02-09", "l_commitdate": "1997-01-26", "l_receiptdate": "1997-02-16", "l_shipinstruct": "NONE", "l_shipmode": "TRUCK", "l_comment": "ies wake carefully b" }
+, { "l_orderkey": 2592, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 7, "l_extendedprice": 6930.63d, "l_discount": 0.1d, "l_tax": 0.04d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-03-13", "l_commitdate": "1993-04-25", "l_receiptdate": "1993-04-01", "l_shipinstruct": "NONE", "l_shipmode": "REG AIR", "l_comment": " carefully special theodolites integrate " }
+, { "l_orderkey": 2625, "l_partkey": 20, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 42, "l_extendedprice": 38640.84d, "l_discount": 0.02d, "l_tax": 0.04d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1992-10-18", "l_commitdate": "1992-11-17", "l_receiptdate": "1992-10-23", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "AIR", "l_comment": " even accounts haggle furiously" }
+, { "l_orderkey": 2659, "l_partkey": 42, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 28, "l_extendedprice": 26377.12d, "l_discount": 0.08d, "l_tax": 0.05d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-03-17", "l_commitdate": "1994-01-24", "l_receiptdate": "1994-03-19", "l_shipinstruct": "NONE", "l_shipmode": "FOB", "l_comment": "idle tithes" }
+, { "l_orderkey": 2689, "l_partkey": 6, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 45, "l_extendedprice": 40770.0d, "l_discount": 0.02d, "l_tax": 0.04d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1992-04-29", "l_commitdate": "1992-06-22", "l_receiptdate": "1992-04-30", "l_shipinstruct": "COLLECT COD", "l_shipmode": "SHIP", "l_comment": "e quickly. carefully silent" }
+, { "l_orderkey": 2690, "l_partkey": 140, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 44, "l_extendedprice": 45766.16d, "l_discount": 0.05d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-05-30", "l_commitdate": "1996-05-19", "l_receiptdate": "1996-06-26", "l_shipinstruct": "NONE", "l_shipmode": "REG AIR", "l_comment": "ly alongside of th" }
+, { "l_orderkey": 2692, "l_partkey": 17, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 3, "l_extendedprice": 2751.03d, "l_discount": 0.1d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-02-25", "l_commitdate": "1998-01-29", "l_receiptdate": "1998-03-27", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "MAIL", "l_comment": "equests. bold, even foxes haggle slyl" }
+, { "l_orderkey": 2694, "l_partkey": 153, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 30, "l_extendedprice": 31594.5d, "l_discount": 0.02d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-06-20", "l_commitdate": "1996-06-01", "l_receiptdate": "1996-07-15", "l_shipinstruct": "NONE", "l_shipmode": "TRUCK", "l_comment": "oxes. never iro" }
+, { "l_orderkey": 2759, "l_partkey": 59, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 10, "l_extendedprice": 9590.5d, "l_discount": 0.1d, "l_tax": 0.03d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-12-14", "l_commitdate": "1994-01-08", "l_receiptdate": "1994-01-01", "l_shipinstruct": "COLLECT COD", "l_shipmode": "FOB", "l_comment": "s. busily ironic theodo" }
+, { "l_orderkey": 2819, "l_partkey": 70, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 17, "l_extendedprice": 16491.19d, "l_discount": 0.08d, "l_tax": 0.08d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-07-16", "l_commitdate": "1994-07-15", "l_receiptdate": "1994-07-17", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": "en deposits above the f" }
+, { "l_orderkey": 2850, "l_partkey": 97, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 43, "l_extendedprice": 42874.87d, "l_discount": 0.02d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-01-11", "l_commitdate": "1996-11-03", "l_receiptdate": "1997-02-01", "l_shipinstruct": "COLLECT COD", "l_shipmode": "REG AIR", "l_comment": "unusual accounts" }
+, { "l_orderkey": 2886, "l_partkey": 60, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 1, "l_extendedprice": 960.06d, "l_discount": 0.09d, "l_tax": 0.05d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1995-02-01", "l_commitdate": "1994-12-18", "l_receiptdate": "1995-02-28", "l_shipinstruct": "COLLECT COD", "l_shipmode": "REG AIR", "l_comment": "eposits fr" }
+, { "l_orderkey": 2912, "l_partkey": 122, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 8, "l_extendedprice": 8176.96d, "l_discount": 0.06d, "l_tax": 0.04d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1992-04-09", "l_commitdate": "1992-04-19", "l_receiptdate": "1992-04-26", "l_shipinstruct": "NONE", "l_shipmode": "RAIL", "l_comment": "hs cajole over the slyl" }
+, { "l_orderkey": 2944, "l_partkey": 120, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 44, "l_extendedprice": 44885.28d, "l_discount": 0.08d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-12-25", "l_commitdate": "1997-10-28", "l_receiptdate": "1998-01-21", "l_shipinstruct": "COLLECT COD", "l_shipmode": "AIR", "l_comment": "ickly special theodolit" }
+, { "l_orderkey": 2947, "l_partkey": 10, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 37, "l_extendedprice": 33670.37d, "l_discount": 0.09d, "l_tax": 0.07d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-08-09", "l_commitdate": "1995-07-05", "l_receiptdate": "1995-08-20", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "RAIL", "l_comment": "e accounts: expres" }
+, { "l_orderkey": 2950, "l_partkey": 130, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 32, "l_extendedprice": 32964.16d, "l_discount": 0.01d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-09-21", "l_commitdate": "1997-08-25", "l_receiptdate": "1997-10-08", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "REG AIR", "l_comment": "its wake carefully slyly final ideas." }
+, { "l_orderkey": 2978, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 29, "l_extendedprice": 28712.61d, "l_discount": 0.0d, "l_tax": 0.08d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1995-06-03", "l_commitdate": "1995-07-25", "l_receiptdate": "1995-06-06", "l_shipinstruct": "NONE", "l_shipmode": "SHIP", "l_comment": "ecial ideas promise slyly" }
+, { "l_orderkey": 3143, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 22, "l_extendedprice": 21781.98d, "l_discount": 0.02d, "l_tax": 0.0d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1993-05-11", "l_commitdate": "1993-03-26", "l_receiptdate": "1993-05-20", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "MAIL", "l_comment": "l, special instructions nag " }
+, { "l_orderkey": 3264, "l_partkey": 200, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 39, "l_extendedprice": 42907.8d, "l_discount": 0.06d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-11-07", "l_commitdate": "1996-12-12", "l_receiptdate": "1996-11-20", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "REG AIR", "l_comment": "sleep carefully after the slyly final" }
+, { "l_orderkey": 3266, "l_partkey": 64, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 31, "l_extendedprice": 29885.86d, "l_discount": 0.09d, "l_tax": 0.02d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-06-19", "l_commitdate": "1995-05-04", "l_receiptdate": "1995-07-06", "l_shipinstruct": "COLLECT COD", "l_shipmode": "MAIL", "l_comment": "grate among the quickly express deposits" }
+, { "l_orderkey": 3270, "l_partkey": 35, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 11, "l_extendedprice": 10285.33d, "l_discount": 0.07d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-07-29", "l_commitdate": "1997-08-11", "l_receiptdate": "1997-08-05", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "AIR", "l_comment": " solve at the regular deposits. " }
+, { "l_orderkey": 3364, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 49, "l_extendedprice": 48514.41d, "l_discount": 0.03d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-09-17", "l_commitdate": "1997-08-23", "l_receiptdate": "1997-10-06", "l_shipinstruct": "NONE", "l_shipmode": "SHIP", "l_comment": "d accounts? caref" }
+, { "l_orderkey": 3366, "l_partkey": 40, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 4, "l_extendedprice": 3760.16d, "l_discount": 0.07d, "l_tax": 0.01d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-05-20", "l_commitdate": "1997-06-25", "l_receiptdate": "1997-06-03", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "AIR", "l_comment": " carefully about " }
+, { "l_orderkey": 3425, "l_partkey": 120, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 11, "l_extendedprice": 11221.32d, "l_discount": 0.03d, "l_tax": 0.08d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-04-24", "l_commitdate": "1996-05-29", "l_receiptdate": "1996-05-23", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "FOB", "l_comment": "ckly final deposits use quickly?" }
+, { "l_orderkey": 3460, "l_partkey": 11, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 40, "l_extendedprice": 36440.4d, "l_discount": 0.1d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-12-28", "l_commitdate": "1995-12-14", "l_receiptdate": "1996-01-02", "l_shipinstruct": "NONE", "l_shipmode": "REG AIR", "l_comment": "odolites are slyly bold deposits" }
+, { "l_orderkey": 3494, "l_partkey": 117, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 40, "l_extendedprice": 40684.4d, "l_discount": 0.05d, "l_tax": 0.04d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-07-10", "l_commitdate": "1993-06-01", "l_receiptdate": "1993-07-25", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "TRUCK", "l_comment": "lites haggle furiously about the fin" }
+, { "l_orderkey": 3520, "l_partkey": 28, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 30, "l_extendedprice": 27840.6d, "l_discount": 0.04d, "l_tax": 0.02d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-11-11", "l_commitdate": "1997-10-02", "l_receiptdate": "1997-12-06", "l_shipinstruct": "COLLECT COD", "l_shipmode": "SHIP", "l_comment": "deas should solve blithely among the ironi" }
+, { "l_orderkey": 3559, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 29, "l_extendedprice": 28712.61d, "l_discount": 0.0d, "l_tax": 0.07d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1992-12-10", "l_commitdate": "1992-12-03", "l_receiptdate": "1992-12-20", "l_shipinstruct": "COLLECT COD", "l_shipmode": "REG AIR", "l_comment": "l, regular accounts wake flu" }
+, { "l_orderkey": 3585, "l_partkey": 122, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 21, "l_extendedprice": 21464.52d, "l_discount": 0.05d, "l_tax": 0.04d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-12-04", "l_commitdate": "1994-12-25", "l_receiptdate": "1995-01-01", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "TRUCK", "l_comment": "ounts use. express, final platelets us" }
+, { "l_orderkey": 3618, "l_partkey": 140, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 38, "l_extendedprice": 39525.32d, "l_discount": 0.08d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-12-22", "l_commitdate": "1998-02-23", "l_receiptdate": "1998-01-03", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "TRUCK", "l_comment": "nts haggle fluffily above the regular " }
+, { "l_orderkey": 3715, "l_partkey": 97, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 13, "l_extendedprice": 12962.17d, "l_discount": 0.0d, "l_tax": 0.03d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-05-11", "l_commitdate": "1996-04-25", "l_receiptdate": "1996-06-09", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "SHIP", "l_comment": "e quickly ironic" }
+, { "l_orderkey": 3844, "l_partkey": 135, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 2, "l_extendedprice": 2070.26d, "l_discount": 0.03d, "l_tax": 0.07d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1995-02-24", "l_commitdate": "1995-02-03", "l_receiptdate": "1995-03-18", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "AIR", "l_comment": "es haggle final acco" }
+, { "l_orderkey": 3878, "l_partkey": 200, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 6, "l_extendedprice": 6601.2d, "l_discount": 0.07d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-06-21", "l_commitdate": "1997-05-22", "l_receiptdate": "1997-07-01", "l_shipinstruct": "COLLECT COD", "l_shipmode": "FOB", "l_comment": "s. regular instru" }
+, { "l_orderkey": 3906, "l_partkey": 153, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 42, "l_extendedprice": 44232.3d, "l_discount": 0.0d, "l_tax": 0.04d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1992-09-03", "l_commitdate": "1992-07-22", "l_receiptdate": "1992-09-04", "l_shipinstruct": "COLLECT COD", "l_shipmode": "RAIL", "l_comment": "jole blithely after the furiously regular " }
+, { "l_orderkey": 3974, "l_partkey": 22, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 47, "l_extendedprice": 43334.94d, "l_discount": 0.1d, "l_tax": 0.03d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-06-03", "l_commitdate": "1996-05-08", "l_receiptdate": "1996-06-28", "l_shipinstruct": "NONE", "l_shipmode": "TRUCK", "l_comment": "dencies above the re" }
+, { "l_orderkey": 4001, "l_partkey": 106, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 26, "l_extendedprice": 26158.6d, "l_discount": 0.0d, "l_tax": 0.01d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-07-26", "l_commitdate": "1997-06-18", "l_receiptdate": "1997-08-08", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "RAIL", "l_comment": "tegrate blithely" }
+, { "l_orderkey": 4005, "l_partkey": 4, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 26, "l_extendedprice": 23504.0d, "l_discount": 0.09d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-12-01", "l_commitdate": "1997-02-03", "l_receiptdate": "1996-12-15", "l_shipinstruct": "NONE", "l_shipmode": "REG AIR", "l_comment": " to the quic" }
+, { "l_orderkey": 4033, "l_partkey": 110, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 27, "l_extendedprice": 27272.97d, "l_discount": 0.01d, "l_tax": 0.04d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-08-08", "l_commitdate": "1993-08-14", "l_receiptdate": "1993-08-09", "l_shipinstruct": "NONE", "l_shipmode": "AIR", "l_comment": "pinto beans" }
+, { "l_orderkey": 4034, "l_partkey": 190, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 48, "l_extendedprice": 52329.12d, "l_discount": 0.03d, "l_tax": 0.03d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-03-01", "l_commitdate": "1994-01-16", "l_receiptdate": "1994-03-16", "l_shipinstruct": "NONE", "l_shipmode": "RAIL", "l_comment": " blithely regular requests play carefull" }
+, { "l_orderkey": 4036, "l_partkey": 6, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 46, "l_extendedprice": 41676.0d, "l_discount": 0.09d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-06-21", "l_commitdate": "1997-05-29", "l_receiptdate": "1997-07-18", "l_shipinstruct": "NONE", "l_shipmode": "REG AIR", "l_comment": "usly across the even th" }
+, { "l_orderkey": 4064, "l_partkey": 199, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 3, "l_extendedprice": 3297.57d, "l_discount": 0.1d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-01-04", "l_commitdate": "1997-01-01", "l_receiptdate": "1997-01-23", "l_shipinstruct": "NONE", "l_shipmode": "SHIP", "l_comment": "its! quickly sp" }
+, { "l_orderkey": 4067, "l_partkey": 180, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 18, "l_extendedprice": 19443.24d, "l_discount": 0.03d, "l_tax": 0.08d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1993-01-24", "l_commitdate": "1992-12-23", "l_receiptdate": "1993-02-20", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "FOB", "l_comment": "e the slyly final packages d" }
+, { "l_orderkey": 4068, "l_partkey": 110, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 43, "l_extendedprice": 43434.73d, "l_discount": 0.05d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-11-28", "l_commitdate": "1996-11-16", "l_receiptdate": "1996-12-22", "l_shipinstruct": "NONE", "l_shipmode": "AIR", "l_comment": "ructions. regular, special packag" }
+, { "l_orderkey": 4098, "l_partkey": 200, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 46, "l_extendedprice": 50609.2d, "l_discount": 0.07d, "l_tax": 0.03d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-01-26", "l_commitdate": "1997-01-27", "l_receiptdate": "1997-02-13", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "SHIP", "l_comment": "e slyly blithely silent deposits. fluff" }
+, { "l_orderkey": 4192, "l_partkey": 11, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 36, "l_extendedprice": 32796.36d, "l_discount": 0.06d, "l_tax": 0.08d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-04-25", "l_commitdate": "1998-05-26", "l_receiptdate": "1998-05-03", "l_shipinstruct": "COLLECT COD", "l_shipmode": "TRUCK", "l_comment": "eodolites sleep" }
+, { "l_orderkey": 4194, "l_partkey": 197, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 43, "l_extendedprice": 47179.17d, "l_discount": 0.08d, "l_tax": 0.06d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-11-06", "l_commitdate": "1994-12-09", "l_receiptdate": "1994-11-16", "l_shipinstruct": "NONE", "l_shipmode": "TRUCK", "l_comment": "olites are after the exp" }
+, { "l_orderkey": 4261, "l_partkey": 110, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 12, "l_extendedprice": 12121.32d, "l_discount": 0.05d, "l_tax": 0.01d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1992-11-01", "l_commitdate": "1993-01-01", "l_receiptdate": "1992-11-12", "l_shipinstruct": "NONE", "l_shipmode": "FOB", "l_comment": "into beans " }
+, { "l_orderkey": 4418, "l_partkey": 35, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 32, "l_extendedprice": 29920.96d, "l_discount": 0.02d, "l_tax": 0.06d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1993-05-28", "l_commitdate": "1993-06-02", "l_receiptdate": "1993-05-30", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": "ly. bold pinto b" }
+, { "l_orderkey": 4422, "l_partkey": 135, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 5, "l_extendedprice": 5175.65d, "l_discount": 0.09d, "l_tax": 0.07d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-07-17", "l_commitdate": "1995-08-13", "l_receiptdate": "1995-07-25", "l_shipinstruct": "NONE", "l_shipmode": "SHIP", "l_comment": "e furiously about t" }
+, { "l_orderkey": 4486, "l_partkey": 135, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 46, "l_extendedprice": 47615.98d, "l_discount": 0.08d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-05-02", "l_commitdate": "1998-04-05", "l_receiptdate": "1998-05-08", "l_shipinstruct": "COLLECT COD", "l_shipmode": "MAIL", "l_comment": "ackages. specia" }
+, { "l_orderkey": 4512, "l_partkey": 162, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 30, "l_extendedprice": 31864.8d, "l_discount": 0.07d, "l_tax": 0.07d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-01-28", "l_commitdate": "1995-12-22", "l_receiptdate": "1996-02-22", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "TRUCK", "l_comment": "ly unusual package" }
+, { "l_orderkey": 4513, "l_partkey": 170, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 29, "l_extendedprice": 31034.93d, "l_discount": 0.03d, "l_tax": 0.01d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-05-18", "l_commitdate": "1996-05-23", "l_receiptdate": "1996-06-08", "l_shipinstruct": "NONE", "l_shipmode": "REG AIR", "l_comment": "cajole. regular packages boost. s" }
+, { "l_orderkey": 4545, "l_partkey": 173, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 38, "l_extendedprice": 40780.46d, "l_discount": 0.06d, "l_tax": 0.06d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-01-27", "l_commitdate": "1993-03-01", "l_receiptdate": "1993-02-04", "l_shipinstruct": "NONE", "l_shipmode": "TRUCK", "l_comment": "nts serve according to th" }
+, { "l_orderkey": 4549, "l_partkey": 159, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 44, "l_extendedprice": 46602.6d, "l_discount": 0.08d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-03-13", "l_commitdate": "1998-04-15", "l_receiptdate": "1998-03-27", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "TRUCK", "l_comment": "ding to the regular, silent requests" }
+, { "l_orderkey": 4551, "l_partkey": 11, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 6, "l_extendedprice": 5466.06d, "l_discount": 0.08d, "l_tax": 0.08d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-05-18", "l_commitdate": "1996-04-23", "l_receiptdate": "1996-06-13", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "TRUCK", "l_comment": "fily silent fo" }
+, { "l_orderkey": 4576, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 5, "l_extendedprice": 4950.45d, "l_discount": 0.09d, "l_tax": 0.03d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-08-23", "l_commitdate": "1996-11-08", "l_receiptdate": "1996-09-20", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "AIR", "l_comment": "ly express, special asymptote" }
+, { "l_orderkey": 4608, "l_partkey": 173, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 30, "l_extendedprice": 32195.1d, "l_discount": 0.08d, "l_tax": 0.05d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-10-08", "l_commitdate": "1994-07-18", "l_receiptdate": "1994-10-25", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "SHIP", "l_comment": "s cajole. slyly " }
+, { "l_orderkey": 4641, "l_partkey": 190, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 45, "l_extendedprice": 49058.55d, "l_discount": 0.07d, "l_tax": 0.03d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-05-11", "l_commitdate": "1993-04-19", "l_receiptdate": "1993-05-21", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "MAIL", "l_comment": " about the close " }
+, { "l_orderkey": 4679, "l_partkey": 190, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 7, "l_extendedprice": 7631.33d, "l_discount": 0.1d, "l_tax": 0.05d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-05-11", "l_commitdate": "1993-04-11", "l_receiptdate": "1993-05-16", "l_shipinstruct": "NONE", "l_shipmode": "TRUCK", "l_comment": "kages. bold, regular packa" }
+, { "l_orderkey": 4769, "l_partkey": 35, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 16, "l_extendedprice": 14960.48d, "l_discount": 0.08d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-07-16", "l_commitdate": "1995-07-05", "l_receiptdate": "1995-07-22", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "FOB", "l_comment": " deposits. slyly even asymptote" }
+, { "l_orderkey": 4802, "l_partkey": 40, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 6, "l_extendedprice": 5640.24d, "l_discount": 0.0d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-04-16", "l_commitdate": "1997-03-25", "l_receiptdate": "1997-04-21", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "SHIP", "l_comment": "unusual accounts wake blithely. b" }
+, { "l_orderkey": 4804, "l_partkey": 128, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 44, "l_extendedprice": 45237.28d, "l_discount": 0.06d, "l_tax": 0.08d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1992-05-02", "l_commitdate": "1992-03-24", "l_receiptdate": "1992-05-28", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "AIR", "l_comment": "aggle quickly among the slyly fi" }
+, { "l_orderkey": 4805, "l_partkey": 150, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 7, "l_extendedprice": 7351.05d, "l_discount": 0.09d, "l_tax": 0.03d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1992-05-01", "l_commitdate": "1992-07-09", "l_receiptdate": "1992-05-09", "l_shipinstruct": "NONE", "l_shipmode": "FOB", "l_comment": " requests. regular deposit" }
+, { "l_orderkey": 4807, "l_partkey": 122, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 9, "l_extendedprice": 9199.08d, "l_discount": 0.04d, "l_tax": 0.08d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-04-23", "l_commitdate": "1997-03-01", "l_receiptdate": "1997-05-15", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "TRUCK", "l_comment": "may are blithely. carefully even pinto b" }
+, { "l_orderkey": 4836, "l_partkey": 162, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 22, "l_extendedprice": 23367.52d, "l_discount": 0.01d, "l_tax": 0.03d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-03-03", "l_commitdate": "1997-02-23", "l_receiptdate": "1997-03-04", "l_shipinstruct": "NONE", "l_shipmode": "SHIP", "l_comment": "al pinto beans. care" }
+, { "l_orderkey": 4837, "l_partkey": 42, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 16, "l_extendedprice": 15072.64d, "l_discount": 0.09d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-08-12", "l_commitdate": "1998-06-06", "l_receiptdate": "1998-08-26", "l_shipinstruct": "COLLECT COD", "l_shipmode": "FOB", "l_comment": "ing requests are blithely regular instructi" }
+, { "l_orderkey": 4898, "l_partkey": 72, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 44, "l_extendedprice": 42771.08d, "l_discount": 0.07d, "l_tax": 0.02d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-09-13", "l_commitdate": "1994-08-18", "l_receiptdate": "1994-09-16", "l_shipinstruct": "NONE", "l_shipmode": "FOB", "l_comment": "y regular grouches about" }
+, { "l_orderkey": 4928, "l_partkey": 100, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 4, "l_extendedprice": 4000.4d, "l_discount": 0.04d, "l_tax": 0.02d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-10-25", "l_commitdate": "1993-12-24", "l_receiptdate": "1993-11-16", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "REG AIR", "l_comment": "bout the slyly final accounts. carefull" }
+, { "l_orderkey": 4929, "l_partkey": 14, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 20, "l_extendedprice": 18280.2d, "l_discount": 0.0d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-03-12", "l_commitdate": "1996-05-23", "l_receiptdate": "1996-03-20", "l_shipinstruct": "COLLECT COD", "l_shipmode": "REG AIR", "l_comment": " final pinto beans detect. final," }
+, { "l_orderkey": 4967, "l_partkey": 71, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 50, "l_extendedprice": 48553.5d, "l_discount": 0.07d, "l_tax": 0.01d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-05-27", "l_commitdate": "1997-05-13", "l_receiptdate": "1997-06-12", "l_shipinstruct": "NONE", "l_shipmode": "REG AIR", "l_comment": "kages. final, unusual accounts c" }
+, { "l_orderkey": 4996, "l_partkey": 56, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 35, "l_extendedprice": 33461.75d, "l_discount": 0.07d, "l_tax": 0.01d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1992-10-30", "l_commitdate": "1992-10-27", "l_receiptdate": "1992-11-05", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "SHIP", "l_comment": "s. unusual, regular dolphins integrate care" }
+, { "l_orderkey": 5028, "l_partkey": 14, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 15, "l_extendedprice": 13710.15d, "l_discount": 0.07d, "l_tax": 0.07d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1992-07-17", "l_commitdate": "1992-07-16", "l_receiptdate": "1992-08-05", "l_shipinstruct": "COLLECT COD", "l_shipmode": "REG AIR", "l_comment": "es are quickly final pains. furiously pend" }
+, { "l_orderkey": 5031, "l_partkey": 50, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 15, "l_extendedprice": 14250.75d, "l_discount": 0.02d, "l_tax": 0.05d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1995-04-01", "l_commitdate": "1995-02-24", "l_receiptdate": "1995-04-12", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "AIR", "l_comment": "yly pending theodolites." }
+, { "l_orderkey": 5092, "l_partkey": 164, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 30, "l_extendedprice": 31924.8d, "l_discount": 0.06d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-12-27", "l_commitdate": "1995-12-08", "l_receiptdate": "1996-01-09", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "MAIL", "l_comment": "ss, ironic deposits. furiously stea" }
+, { "l_orderkey": 5153, "l_partkey": 35, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 42, "l_extendedprice": 39271.26d, "l_discount": 0.03d, "l_tax": 0.01d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-10-03", "l_commitdate": "1995-11-09", "l_receiptdate": "1995-10-11", "l_shipinstruct": "COLLECT COD", "l_shipmode": "RAIL", "l_comment": "re thinly. ironic" }
+, { "l_orderkey": 5154, "l_partkey": 190, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 11, "l_extendedprice": 11992.09d, "l_discount": 0.02d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-08-06", "l_commitdate": "1997-06-30", "l_receiptdate": "1997-09-04", "l_shipinstruct": "NONE", "l_shipmode": "RAIL", "l_comment": "luffily bold foxes. final" }
+, { "l_orderkey": 5185, "l_partkey": 197, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 37, "l_extendedprice": 40596.03d, "l_discount": 0.0d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-08-08", "l_commitdate": "1997-09-08", "l_receiptdate": "1997-08-14", "l_shipinstruct": "COLLECT COD", "l_shipmode": "SHIP", "l_comment": "gainst the courts dazzle care" }
+, { "l_orderkey": 5187, "l_partkey": 11, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 49, "l_extendedprice": 44639.49d, "l_discount": 0.04d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-10-20", "l_commitdate": "1997-10-12", "l_receiptdate": "1997-10-26", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "AIR", "l_comment": "l, regular platelets instead of the foxes w" }
+, { "l_orderkey": 5190, "l_partkey": 56, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 43, "l_extendedprice": 41110.15d, "l_discount": 0.09d, "l_tax": 0.06d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1992-08-19", "l_commitdate": "1992-06-10", "l_receiptdate": "1992-09-01", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "FOB", "l_comment": "encies use fluffily unusual requests? hoc" }
+, { "l_orderkey": 5217, "l_partkey": 80, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 50, "l_extendedprice": 49004.0d, "l_discount": 0.05d, "l_tax": 0.02d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-12-26", "l_commitdate": "1995-11-21", "l_receiptdate": "1996-01-24", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "MAIL", "l_comment": "s. express, express accounts c" }
+, { "l_orderkey": 5281, "l_partkey": 114, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 37, "l_extendedprice": 37522.07d, "l_discount": 0.05d, "l_tax": 0.02d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-11-10", "l_commitdate": "1996-01-31", "l_receiptdate": "1995-11-22", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "MAIL", "l_comment": "ronic dependencies. fluffily final p" }
+, { "l_orderkey": 5284, "l_partkey": 173, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 16, "l_extendedprice": 17170.72d, "l_discount": 0.04d, "l_tax": 0.02d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-08-17", "l_commitdate": "1995-08-23", "l_receiptdate": "1995-08-26", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "TRUCK", "l_comment": "unts detect furiously even d" }
+, { "l_orderkey": 5286, "l_partkey": 199, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 1, "l_extendedprice": 1099.19d, "l_discount": 0.01d, "l_tax": 0.07d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-11-25", "l_commitdate": "1997-11-07", "l_receiptdate": "1997-12-17", "l_shipinstruct": "COLLECT COD", "l_shipmode": "REG AIR", "l_comment": "ly! furiously final pack" }
+, { "l_orderkey": 5313, "l_partkey": 17, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 34, "l_extendedprice": 31178.34d, "l_discount": 0.1d, "l_tax": 0.02d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-08-07", "l_commitdate": "1997-08-12", "l_receiptdate": "1997-08-24", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "FOB", "l_comment": "ccording to the blithely final account" }
+, { "l_orderkey": 5315, "l_partkey": 35, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 12, "l_extendedprice": 11220.36d, "l_discount": 0.08d, "l_tax": 0.06d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1992-12-18", "l_commitdate": "1993-01-16", "l_receiptdate": "1993-01-10", "l_shipinstruct": "NONE", "l_shipmode": "AIR", "l_comment": "ccounts. furiously ironi" }
+, { "l_orderkey": 5316, "l_partkey": 108, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 29, "l_extendedprice": 29234.9d, "l_discount": 0.1d, "l_tax": 0.05d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-03-28", "l_commitdate": "1994-04-29", "l_receiptdate": "1994-04-09", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "REG AIR", "l_comment": "ckly unusual foxes bo" }
+, { "l_orderkey": 5379, "l_partkey": 199, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 40, "l_extendedprice": 43967.6d, "l_discount": 0.01d, "l_tax": 0.08d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-10-01", "l_commitdate": "1995-10-19", "l_receiptdate": "1995-10-30", "l_shipinstruct": "COLLECT COD", "l_shipmode": "MAIL", "l_comment": "carefully final accounts haggle blithely. " }
+, { "l_orderkey": 5441, "l_partkey": 164, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 3, "l_extendedprice": 3192.48d, "l_discount": 0.0d, "l_tax": 0.02d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-08-12", "l_commitdate": "1994-10-14", "l_receiptdate": "1994-09-01", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "REG AIR", "l_comment": "are. unusual, " }
+, { "l_orderkey": 5445, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 33, "l_extendedprice": 32672.97d, "l_discount": 0.08d, "l_tax": 0.07d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1993-10-21", "l_commitdate": "1993-10-14", "l_receiptdate": "1993-10-28", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "REG AIR", "l_comment": "ests. final instructions" }
+, { "l_orderkey": 5446, "l_partkey": 190, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 27, "l_extendedprice": 29435.13d, "l_discount": 0.05d, "l_tax": 0.07d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-07-21", "l_commitdate": "1994-08-25", "l_receiptdate": "1994-08-17", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": "ously across the quic" }
+, { "l_orderkey": 5447, "l_partkey": 99, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 31, "l_extendedprice": 30971.79d, "l_discount": 0.09d, "l_tax": 0.03d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-07-14", "l_commitdate": "1996-05-07", "l_receiptdate": "1996-07-17", "l_shipinstruct": "COLLECT COD", "l_shipmode": "SHIP", "l_comment": " foxes sleep. blithely unusual accounts det" }
+, { "l_orderkey": 5476, "l_partkey": 48, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 13, "l_extendedprice": 12324.52d, "l_discount": 0.01d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-12-27", "l_commitdate": "1997-12-08", "l_receiptdate": "1997-12-29", "l_shipinstruct": "COLLECT COD", "l_shipmode": "TRUCK", "l_comment": "iously special ac" }
+, { "l_orderkey": 5506, "l_partkey": 140, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 2, "l_extendedprice": 2080.28d, "l_discount": 0.0d, "l_tax": 0.03d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-02-04", "l_commitdate": "1994-01-13", "l_receiptdate": "1994-02-17", "l_shipinstruct": "COLLECT COD", "l_shipmode": "MAIL", "l_comment": "onic theodolites are fluffil" }
+, { "l_orderkey": 5536, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 14, "l_extendedprice": 13861.26d, "l_discount": 0.08d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-05-18", "l_commitdate": "1998-05-08", "l_receiptdate": "1998-06-05", "l_shipinstruct": "COLLECT COD", "l_shipmode": "MAIL", "l_comment": "instructions sleep " }
+, { "l_orderkey": 5572, "l_partkey": 22, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 24, "l_extendedprice": 22128.48d, "l_discount": 0.08d, "l_tax": 0.08d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-10-30", "l_commitdate": "1994-10-02", "l_receiptdate": "1994-11-27", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "MAIL", "l_comment": "ests cajole. evenly ironic exc" }
+, { "l_orderkey": 5664, "l_partkey": 122, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 25, "l_extendedprice": 25553.0d, "l_discount": 0.0d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-10-29", "l_commitdate": "1998-09-23", "l_receiptdate": "1998-11-25", "l_shipinstruct": "COLLECT COD", "l_shipmode": "FOB", "l_comment": "eposits: furiously ironic grouch" }
+, { "l_orderkey": 5670, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 27, "l_extendedprice": 26732.43d, "l_discount": 0.1d, "l_tax": 0.06d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-05-09", "l_commitdate": "1993-05-30", "l_receiptdate": "1993-06-06", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "REG AIR", "l_comment": " ideas promise bli" }
+, { "l_orderkey": 5728, "l_partkey": 44, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 47, "l_extendedprice": 44369.88d, "l_discount": 0.1d, "l_tax": 0.05d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-12-13", "l_commitdate": "1995-01-25", "l_receiptdate": "1994-12-25", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "MAIL", "l_comment": "nd the bravely final deposits. final ideas" }
+, { "l_orderkey": 5735, "l_partkey": 60, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 41, "l_extendedprice": 39362.46d, "l_discount": 0.01d, "l_tax": 0.01d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-12-23", "l_commitdate": "1995-02-10", "l_receiptdate": "1995-01-22", "l_shipinstruct": "COLLECT COD", "l_shipmode": "MAIL", "l_comment": "lthily ruthless i" }
+, { "l_orderkey": 5826, "l_partkey": 144, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 4, "l_extendedprice": 4176.56d, "l_discount": 0.03d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-07-31", "l_commitdate": "1998-09-10", "l_receiptdate": "1998-08-27", "l_shipinstruct": "NONE", "l_shipmode": "AIR", "l_comment": " packages across the fluffily spec" }
+, { "l_orderkey": 5829, "l_partkey": 40, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 4, "l_extendedprice": 3760.16d, "l_discount": 0.01d, "l_tax": 0.02d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-03-01", "l_commitdate": "1997-02-17", "l_receiptdate": "1997-03-22", "l_shipinstruct": "NONE", "l_shipmode": "TRUCK", "l_comment": "ithely; accounts cajole ideas. regular foxe" }
+, { "l_orderkey": 5856, "l_partkey": 4, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 1, "l_extendedprice": 904.0d, "l_discount": 0.03d, "l_tax": 0.02d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-12-29", "l_commitdate": "1995-01-07", "l_receiptdate": "1995-01-10", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "MAIL", "l_comment": "tly. special deposits wake blithely even" }
+, { "l_orderkey": 5926, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 8, "l_extendedprice": 7920.72d, "l_discount": 0.02d, "l_tax": 0.0d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-07-17", "l_commitdate": "1994-07-20", "l_receiptdate": "1994-08-11", "l_shipinstruct": "COLLECT COD", "l_shipmode": "MAIL", "l_comment": "gle furiously express foxes. bo" }
+, { "l_orderkey": 5927, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 44, "l_extendedprice": 43563.96d, "l_discount": 0.04d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-11-29", "l_commitdate": "1997-11-21", "l_receiptdate": "1997-12-13", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "TRUCK", "l_comment": "rding to the special, final decoy" }
+, { "l_orderkey": 5955, "l_partkey": 140, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 14, "l_extendedprice": 14561.96d, "l_discount": 0.08d, "l_tax": 0.08d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-06-22", "l_commitdate": "1995-05-23", "l_receiptdate": "1995-06-24", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "TRUCK", "l_comment": " unusual, bold theodolit" }
+, { "l_orderkey": 5959, "l_partkey": 135, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 49, "l_extendedprice": 50721.37d, "l_discount": 0.07d, "l_tax": 0.03d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1992-07-16", "l_commitdate": "1992-08-09", "l_receiptdate": "1992-08-14", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "SHIP", "l_comment": "usual packages haggle slyly pi" }
+, { "l_orderkey": 5988, "l_partkey": 172, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 41, "l_extendedprice": 43958.97d, "l_discount": 0.08d, "l_tax": 0.03d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-01-20", "l_commitdate": "1994-02-06", "l_receiptdate": "1994-02-10", "l_shipinstruct": "COLLECT COD", "l_shipmode": "AIR", "l_comment": "the pending, express reque" }
+ ]
diff --git a/asterix-app/src/test/resources/runtimets/results/temp-dataset/delete-from-loaded-dataset-with-index/delete-from-loaded-dataset-with-index.1.adm b/asterix-app/src/test/resources/runtimets/results/temp-dataset/delete-from-loaded-dataset-with-index/delete-from-loaded-dataset-with-index.1.adm
new file mode 100644
index 0000000..38303d7
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/temp-dataset/delete-from-loaded-dataset-with-index/delete-from-loaded-dataset-with-index.1.adm
@@ -0,0 +1,168 @@
+[ { "l_orderkey": 36, "l_partkey": 120, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 42, "l_extendedprice": 42845.04d, "l_discount": 0.09d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-02-03", "l_commitdate": "1996-01-21", "l_receiptdate": "1996-02-23", "l_shipinstruct": "COLLECT COD", "l_shipmode": "SHIP", "l_comment": " careful courts. special " }
+, { "l_orderkey": 68, "l_partkey": 8, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 3, "l_extendedprice": 2724.0d, "l_discount": 0.05d, "l_tax": 0.02d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-07-04", "l_commitdate": "1998-06-05", "l_receiptdate": "1998-07-21", "l_shipinstruct": "NONE", "l_shipmode": "RAIL", "l_comment": "fully special instructions cajole. furious" }
+, { "l_orderkey": 162, "l_partkey": 190, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 2, "l_extendedprice": 2180.38d, "l_discount": 0.02d, "l_tax": 0.01d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-09-02", "l_commitdate": "1995-06-17", "l_receiptdate": "1995-09-08", "l_shipinstruct": "COLLECT COD", "l_shipmode": "FOB", "l_comment": "es! final somas integrate" }
+, { "l_orderkey": 192, "l_partkey": 98, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 23, "l_extendedprice": 22956.07d, "l_discount": 0.0d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-02-05", "l_commitdate": "1998-02-06", "l_receiptdate": "1998-03-07", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "AIR", "l_comment": "ly pending theodolites haggle quickly fluf" }
+, { "l_orderkey": 197, "l_partkey": 99, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 39, "l_extendedprice": 38964.51d, "l_discount": 0.02d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-07-21", "l_commitdate": "1995-07-01", "l_receiptdate": "1995-08-14", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "AIR", "l_comment": "press accounts. daringly sp" }
+, { "l_orderkey": 227, "l_partkey": 166, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 19, "l_extendedprice": 20257.04d, "l_discount": 0.05d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-12-10", "l_commitdate": "1996-01-30", "l_receiptdate": "1995-12-26", "l_shipinstruct": "NONE", "l_shipmode": "RAIL", "l_comment": "s cajole furiously a" }
+, { "l_orderkey": 290, "l_partkey": 6, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 35, "l_extendedprice": 31710.0d, "l_discount": 0.01d, "l_tax": 0.02d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-04-01", "l_commitdate": "1994-02-05", "l_receiptdate": "1994-04-27", "l_shipinstruct": "NONE", "l_shipmode": "MAIL", "l_comment": "ove the final foxes detect slyly fluffily" }
+, { "l_orderkey": 325, "l_partkey": 159, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 34, "l_extendedprice": 36011.1d, "l_discount": 0.09d, "l_tax": 0.04d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1993-10-28", "l_commitdate": "1993-12-13", "l_receiptdate": "1993-11-17", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "MAIL", "l_comment": "ly bold deposits. always iron" }
+, { "l_orderkey": 355, "l_partkey": 114, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 31, "l_extendedprice": 31437.41d, "l_discount": 0.09d, "l_tax": 0.07d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-07-13", "l_commitdate": "1994-08-18", "l_receiptdate": "1994-07-18", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "FOB", "l_comment": "y unusual, ironic" }
+, { "l_orderkey": 389, "l_partkey": 190, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 2, "l_extendedprice": 2180.38d, "l_discount": 0.09d, "l_tax": 0.0d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-04-13", "l_commitdate": "1994-04-10", "l_receiptdate": "1994-04-25", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": "fts. courts eat blithely even dependenc" }
+, { "l_orderkey": 391, "l_partkey": 122, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 14, "l_extendedprice": 14309.68d, "l_discount": 0.09d, "l_tax": 0.02d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1995-02-11", "l_commitdate": "1995-02-03", "l_receiptdate": "1995-02-13", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "TRUCK", "l_comment": " escapades sleep furiously about " }
+, { "l_orderkey": 417, "l_partkey": 40, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 39, "l_extendedprice": 36661.56d, "l_discount": 0.01d, "l_tax": 0.02d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-05-31", "l_commitdate": "1994-05-02", "l_receiptdate": "1994-06-06", "l_shipinstruct": "NONE", "l_shipmode": "SHIP", "l_comment": "y regular requests wake along " }
+, { "l_orderkey": 453, "l_partkey": 198, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 45, "l_extendedprice": 49418.55d, "l_discount": 0.01d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-06-30", "l_commitdate": "1997-08-20", "l_receiptdate": "1997-07-19", "l_shipinstruct": "COLLECT COD", "l_shipmode": "REG AIR", "l_comment": "ifts wake carefully." }
+, { "l_orderkey": 485, "l_partkey": 150, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 50, "l_extendedprice": 52507.5d, "l_discount": 0.01d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-03-28", "l_commitdate": "1997-05-26", "l_receiptdate": "1997-04-18", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "MAIL", "l_comment": "iously quick excuses. carefully final f" }
+, { "l_orderkey": 545, "l_partkey": 170, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 4, "l_extendedprice": 4280.68d, "l_discount": 0.02d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-02-23", "l_commitdate": "1995-12-16", "l_receiptdate": "1996-03-21", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "FOB", "l_comment": ", ironic grouches cajole over" }
+, { "l_orderkey": 581, "l_partkey": 64, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 41, "l_extendedprice": 39526.46d, "l_discount": 0.09d, "l_tax": 0.07d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-05-26", "l_commitdate": "1997-04-06", "l_receiptdate": "1997-06-10", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "MAIL", "l_comment": "nts. quickly" }
+, { "l_orderkey": 647, "l_partkey": 17, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 41, "l_extendedprice": 37597.41d, "l_discount": 0.08d, "l_tax": 0.08d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-11-19", "l_commitdate": "1997-09-24", "l_receiptdate": "1997-12-15", "l_shipinstruct": "COLLECT COD", "l_shipmode": "REG AIR", "l_comment": "r instructions. quickly unusu" }
+, { "l_orderkey": 704, "l_partkey": 190, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 40, "l_extendedprice": 43607.6d, "l_discount": 0.05d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-01-30", "l_commitdate": "1997-01-10", "l_receiptdate": "1997-02-20", "l_shipinstruct": "COLLECT COD", "l_shipmode": "AIR", "l_comment": "ggle quickly. r" }
+, { "l_orderkey": 738, "l_partkey": 198, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 34, "l_extendedprice": 37338.46d, "l_discount": 0.0d, "l_tax": 0.06d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-06-09", "l_commitdate": "1993-04-15", "l_receiptdate": "1993-07-09", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "TRUCK", "l_comment": "s against the ironic exc" }
+, { "l_orderkey": 773, "l_partkey": 100, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 5, "l_extendedprice": 5000.5d, "l_discount": 0.06d, "l_tax": 0.04d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1993-11-21", "l_commitdate": "1993-12-19", "l_receiptdate": "1993-12-21", "l_shipinstruct": "COLLECT COD", "l_shipmode": "MAIL", "l_comment": "ar requests. regular, thin packages u" }
+, { "l_orderkey": 800, "l_partkey": 72, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 38, "l_extendedprice": 36938.66d, "l_discount": 0.0d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-07-21", "l_commitdate": "1998-09-25", "l_receiptdate": "1998-08-07", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "TRUCK", "l_comment": "according to the bold, final dependencies " }
+, { "l_orderkey": 900, "l_partkey": 199, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 44, "l_extendedprice": 48364.36d, "l_discount": 0.01d, "l_tax": 0.06d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-12-15", "l_commitdate": "1994-12-03", "l_receiptdate": "1994-12-27", "l_shipinstruct": "COLLECT COD", "l_shipmode": "MAIL", "l_comment": " detect quick" }
+, { "l_orderkey": 931, "l_partkey": 40, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 18, "l_extendedprice": 16920.72d, "l_discount": 0.0d, "l_tax": 0.05d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1993-04-04", "l_commitdate": "1993-01-11", "l_receiptdate": "1993-04-13", "l_shipinstruct": "NONE", "l_shipmode": "RAIL", "l_comment": "slyly ironic re" }
+, { "l_orderkey": 932, "l_partkey": 44, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 41, "l_extendedprice": 38705.64d, "l_discount": 0.01d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-06-05", "l_commitdate": "1997-07-22", "l_receiptdate": "1997-06-26", "l_shipinstruct": "COLLECT COD", "l_shipmode": "RAIL", "l_comment": "foxes. ironic pl" }
+, { "l_orderkey": 965, "l_partkey": 108, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 20, "l_extendedprice": 20162.0d, "l_discount": 0.04d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "F", "l_shipdate": "1995-06-16", "l_commitdate": "1995-07-20", "l_receiptdate": "1995-07-06", "l_shipinstruct": "COLLECT COD", "l_shipmode": "MAIL", "l_comment": "kly. carefully pending requ" }
+, { "l_orderkey": 995, "l_partkey": 173, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 15, "l_extendedprice": 16097.55d, "l_discount": 0.08d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-06-30", "l_commitdate": "1995-08-04", "l_receiptdate": "1995-07-27", "l_shipinstruct": "NONE", "l_shipmode": "REG AIR", "l_comment": "uses. fluffily fina" }
+, { "l_orderkey": 1025, "l_partkey": 150, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 36, "l_extendedprice": 37805.4d, "l_discount": 0.03d, "l_tax": 0.04d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1995-05-15", "l_commitdate": "1995-07-05", "l_receiptdate": "1995-06-10", "l_shipinstruct": "COLLECT COD", "l_shipmode": "FOB", "l_comment": "e unusual, regular instr" }
+, { "l_orderkey": 1027, "l_partkey": 156, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 43, "l_extendedprice": 45414.45d, "l_discount": 0.07d, "l_tax": 0.08d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1992-06-17", "l_commitdate": "1992-08-28", "l_receiptdate": "1992-07-10", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "MAIL", "l_comment": "oxes. carefully regular deposits" }
+, { "l_orderkey": 1155, "l_partkey": 70, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 4, "l_extendedprice": 3880.28d, "l_discount": 0.09d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-10-19", "l_commitdate": "1997-12-09", "l_receiptdate": "1997-11-02", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "SHIP", "l_comment": "ic foxes according to the carefully final " }
+, { "l_orderkey": 1185, "l_partkey": 72, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 8, "l_extendedprice": 7776.56d, "l_discount": 0.01d, "l_tax": 0.06d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1992-12-05", "l_commitdate": "1992-10-05", "l_receiptdate": "1992-12-28", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "MAIL", "l_comment": "ely according to the furiously regular r" }
+, { "l_orderkey": 1216, "l_partkey": 97, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 8, "l_extendedprice": 7976.72d, "l_discount": 0.03d, "l_tax": 0.04d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-02-01", "l_commitdate": "1993-03-06", "l_receiptdate": "1993-02-08", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "TRUCK", "l_comment": " of the carefully express" }
+, { "l_orderkey": 1223, "l_partkey": 100, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 28, "l_extendedprice": 28002.8d, "l_discount": 0.1d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-08-07", "l_commitdate": "1996-07-24", "l_receiptdate": "1996-08-13", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "MAIL", "l_comment": " quickly ironic requests. furious" }
+, { "l_orderkey": 1381, "l_partkey": 144, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 47, "l_extendedprice": 49074.58d, "l_discount": 0.08d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-09-22", "l_commitdate": "1998-08-12", "l_receiptdate": "1998-10-12", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "AIR", "l_comment": "ly ironic deposits" }
+, { "l_orderkey": 1409, "l_partkey": 99, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 23, "l_extendedprice": 22979.07d, "l_discount": 0.01d, "l_tax": 0.03d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1993-04-18", "l_commitdate": "1993-02-25", "l_receiptdate": "1993-05-06", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "FOB", "l_comment": "ions. slyly ironic packages wake quick" }
+, { "l_orderkey": 1445, "l_partkey": 100, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 24, "l_extendedprice": 24002.4d, "l_discount": 0.01d, "l_tax": 0.0d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1995-02-21", "l_commitdate": "1995-02-22", "l_receiptdate": "1995-03-18", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "SHIP", "l_comment": "al accounts use furiously a" }
+, { "l_orderkey": 1477, "l_partkey": 72, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 31, "l_extendedprice": 30134.17d, "l_discount": 0.0d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-12-16", "l_commitdate": "1997-09-30", "l_receiptdate": "1997-12-17", "l_shipinstruct": "COLLECT COD", "l_shipmode": "RAIL", "l_comment": " requests. fluffily final " }
+, { "l_orderkey": 1540, "l_partkey": 173, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 38, "l_extendedprice": 40780.46d, "l_discount": 0.03d, "l_tax": 0.01d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1992-09-30", "l_commitdate": "1992-10-27", "l_receiptdate": "1992-10-12", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "SHIP", "l_comment": " final grouches bo" }
+, { "l_orderkey": 1568, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 36, "l_extendedprice": 35643.24d, "l_discount": 0.02d, "l_tax": 0.03d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-05-31", "l_commitdate": "1997-04-22", "l_receiptdate": "1997-06-21", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": "platelets-- furiously sly excu" }
+, { "l_orderkey": 1605, "l_partkey": 142, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 47, "l_extendedprice": 48980.58d, "l_discount": 0.0d, "l_tax": 0.01d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-04-29", "l_commitdate": "1998-06-12", "l_receiptdate": "1998-05-20", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "AIR", "l_comment": ". carefully r" }
+, { "l_orderkey": 1607, "l_partkey": 190, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 2, "l_extendedprice": 2180.38d, "l_discount": 0.02d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-01-11", "l_commitdate": "1996-02-15", "l_receiptdate": "1996-01-19", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "MAIL", "l_comment": "packages haggle. regular requests boost s" }
+, { "l_orderkey": 1635, "l_partkey": 71, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 3, "l_extendedprice": 2913.21d, "l_discount": 0.06d, "l_tax": 0.08d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-03-13", "l_commitdate": "1997-03-25", "l_receiptdate": "1997-03-27", "l_shipinstruct": "COLLECT COD", "l_shipmode": "FOB", "l_comment": " quickly ironic r" }
+, { "l_orderkey": 1700, "l_partkey": 140, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 38, "l_extendedprice": 39525.32d, "l_discount": 0.04d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-10-03", "l_commitdate": "1996-07-27", "l_receiptdate": "1996-10-22", "l_shipinstruct": "NONE", "l_shipmode": "RAIL", "l_comment": "ular dependencies engage slyly " }
+, { "l_orderkey": 1796, "l_partkey": 10, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 28, "l_extendedprice": 25480.28d, "l_discount": 0.08d, "l_tax": 0.04d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1992-12-01", "l_commitdate": "1993-01-01", "l_receiptdate": "1992-12-24", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "FOB", "l_comment": "y quickly ironic accounts." }
+, { "l_orderkey": 1825, "l_partkey": 156, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 43, "l_extendedprice": 45414.45d, "l_discount": 0.05d, "l_tax": 0.05d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-02-18", "l_commitdate": "1994-02-19", "l_receiptdate": "1994-03-02", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": " accounts breach fluffily spe" }
+, { "l_orderkey": 1827, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 47, "l_extendedprice": 46534.23d, "l_discount": 0.0d, "l_tax": 0.01d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-08-01", "l_commitdate": "1996-08-07", "l_receiptdate": "1996-08-23", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": ". pending courts about the even e" }
+, { "l_orderkey": 1893, "l_partkey": 99, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 43, "l_extendedprice": 42960.87d, "l_discount": 0.1d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-01-25", "l_commitdate": "1998-01-06", "l_receiptdate": "1998-02-14", "l_shipinstruct": "COLLECT COD", "l_shipmode": "SHIP", "l_comment": "he carefully regular " }
+, { "l_orderkey": 1924, "l_partkey": 73, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 7, "l_extendedprice": 6811.49d, "l_discount": 0.06d, "l_tax": 0.07d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-01-01", "l_commitdate": "1996-12-02", "l_receiptdate": "1997-01-08", "l_shipinstruct": "COLLECT COD", "l_shipmode": "SHIP", "l_comment": "osits. even accounts nag furious" }
+, { "l_orderkey": 1953, "l_partkey": 128, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 25, "l_extendedprice": 25703.0d, "l_discount": 0.07d, "l_tax": 0.06d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-01-07", "l_commitdate": "1994-01-28", "l_receiptdate": "1994-01-29", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": "ular, regular i" }
+, { "l_orderkey": 1985, "l_partkey": 28, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 33, "l_extendedprice": 30624.66d, "l_discount": 0.1d, "l_tax": 0.03d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-12-04", "l_commitdate": "1994-11-01", "l_receiptdate": "1994-12-05", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "FOB", "l_comment": "s are express packages. pendin" }
+, { "l_orderkey": 1988, "l_partkey": 72, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 36, "l_extendedprice": 34994.52d, "l_discount": 0.09d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-01-21", "l_commitdate": "1995-11-24", "l_receiptdate": "1996-01-27", "l_shipinstruct": "NONE", "l_shipmode": "RAIL", "l_comment": "gular theodolites. " }
+, { "l_orderkey": 2048, "l_partkey": 35, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 7, "l_extendedprice": 6545.21d, "l_discount": 0.06d, "l_tax": 0.01d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-12-07", "l_commitdate": "1994-01-31", "l_receiptdate": "1994-01-05", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "REG AIR", "l_comment": "lent platelets boost deposits. carefully sp" }
+, { "l_orderkey": 2086, "l_partkey": 60, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 22, "l_extendedprice": 21121.32d, "l_discount": 0.03d, "l_tax": 0.07d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-12-04", "l_commitdate": "1994-12-16", "l_receiptdate": "1994-12-20", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "RAIL", "l_comment": "idly busy acc" }
+, { "l_orderkey": 2118, "l_partkey": 160, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 24, "l_extendedprice": 25443.84d, "l_discount": 0.1d, "l_tax": 0.03d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-01-06", "l_commitdate": "1996-12-14", "l_receiptdate": "1997-01-14", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": "about the slyly bold depende" }
+, { "l_orderkey": 2183, "l_partkey": 71, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 29, "l_extendedprice": 28161.03d, "l_discount": 0.05d, "l_tax": 0.01d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-07-21", "l_commitdate": "1996-08-24", "l_receiptdate": "1996-08-15", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": "ly unusual deposits sleep carefully" }
+, { "l_orderkey": 2211, "l_partkey": 48, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 25, "l_extendedprice": 23701.0d, "l_discount": 0.04d, "l_tax": 0.01d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-10-09", "l_commitdate": "1994-08-04", "l_receiptdate": "1994-11-03", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": "deas. carefully special theodolites along" }
+, { "l_orderkey": 2215, "l_partkey": 73, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 33, "l_extendedprice": 32111.31d, "l_discount": 0.0d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-07-19", "l_commitdate": "1996-08-10", "l_receiptdate": "1996-07-30", "l_shipinstruct": "COLLECT COD", "l_shipmode": "RAIL", "l_comment": "dolites cajole b" }
+, { "l_orderkey": 2272, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 18, "l_extendedprice": 17821.62d, "l_discount": 0.04d, "l_tax": 0.0d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-08-01", "l_commitdate": "1993-07-06", "l_receiptdate": "1993-08-25", "l_shipinstruct": "NONE", "l_shipmode": "MAIL", "l_comment": "ons along the blithely e" }
+, { "l_orderkey": 2342, "l_partkey": 42, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 12, "l_extendedprice": 11304.48d, "l_discount": 0.0d, "l_tax": 0.08d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-07-31", "l_commitdate": "1996-07-26", "l_receiptdate": "1996-08-14", "l_shipinstruct": "NONE", "l_shipmode": "TRUCK", "l_comment": "print blithely even deposits. carefull" }
+, { "l_orderkey": 2343, "l_partkey": 110, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 27, "l_extendedprice": 27272.97d, "l_discount": 0.0d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-11-10", "l_commitdate": "1995-11-17", "l_receiptdate": "1995-12-10", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "SHIP", "l_comment": "old theodolites." }
+, { "l_orderkey": 2439, "l_partkey": 164, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 2, "l_extendedprice": 2128.32d, "l_discount": 0.09d, "l_tax": 0.03d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-04-14", "l_commitdate": "1997-06-11", "l_receiptdate": "1997-05-09", "l_shipinstruct": "COLLECT COD", "l_shipmode": "MAIL", "l_comment": "courts boos" }
+, { "l_orderkey": 2469, "l_partkey": 166, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 11, "l_extendedprice": 11727.76d, "l_discount": 0.0d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-02-09", "l_commitdate": "1997-01-26", "l_receiptdate": "1997-02-16", "l_shipinstruct": "NONE", "l_shipmode": "TRUCK", "l_comment": "ies wake carefully b" }
+, { "l_orderkey": 2592, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 7, "l_extendedprice": 6930.63d, "l_discount": 0.1d, "l_tax": 0.04d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-03-13", "l_commitdate": "1993-04-25", "l_receiptdate": "1993-04-01", "l_shipinstruct": "NONE", "l_shipmode": "REG AIR", "l_comment": " carefully special theodolites integrate " }
+, { "l_orderkey": 2625, "l_partkey": 20, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 42, "l_extendedprice": 38640.84d, "l_discount": 0.02d, "l_tax": 0.04d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1992-10-18", "l_commitdate": "1992-11-17", "l_receiptdate": "1992-10-23", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "AIR", "l_comment": " even accounts haggle furiously" }
+, { "l_orderkey": 2659, "l_partkey": 42, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 28, "l_extendedprice": 26377.12d, "l_discount": 0.08d, "l_tax": 0.05d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-03-17", "l_commitdate": "1994-01-24", "l_receiptdate": "1994-03-19", "l_shipinstruct": "NONE", "l_shipmode": "FOB", "l_comment": "idle tithes" }
+, { "l_orderkey": 2689, "l_partkey": 6, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 45, "l_extendedprice": 40770.0d, "l_discount": 0.02d, "l_tax": 0.04d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1992-04-29", "l_commitdate": "1992-06-22", "l_receiptdate": "1992-04-30", "l_shipinstruct": "COLLECT COD", "l_shipmode": "SHIP", "l_comment": "e quickly. carefully silent" }
+, { "l_orderkey": 2690, "l_partkey": 140, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 44, "l_extendedprice": 45766.16d, "l_discount": 0.05d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-05-30", "l_commitdate": "1996-05-19", "l_receiptdate": "1996-06-26", "l_shipinstruct": "NONE", "l_shipmode": "REG AIR", "l_comment": "ly alongside of th" }
+, { "l_orderkey": 2692, "l_partkey": 17, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 3, "l_extendedprice": 2751.03d, "l_discount": 0.1d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-02-25", "l_commitdate": "1998-01-29", "l_receiptdate": "1998-03-27", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "MAIL", "l_comment": "equests. bold, even foxes haggle slyl" }
+, { "l_orderkey": 2694, "l_partkey": 153, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 30, "l_extendedprice": 31594.5d, "l_discount": 0.02d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-06-20", "l_commitdate": "1996-06-01", "l_receiptdate": "1996-07-15", "l_shipinstruct": "NONE", "l_shipmode": "TRUCK", "l_comment": "oxes. never iro" }
+, { "l_orderkey": 2759, "l_partkey": 59, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 10, "l_extendedprice": 9590.5d, "l_discount": 0.1d, "l_tax": 0.03d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-12-14", "l_commitdate": "1994-01-08", "l_receiptdate": "1994-01-01", "l_shipinstruct": "COLLECT COD", "l_shipmode": "FOB", "l_comment": "s. busily ironic theodo" }
+, { "l_orderkey": 2819, "l_partkey": 70, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 17, "l_extendedprice": 16491.19d, "l_discount": 0.08d, "l_tax": 0.08d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-07-16", "l_commitdate": "1994-07-15", "l_receiptdate": "1994-07-17", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": "en deposits above the f" }
+, { "l_orderkey": 2850, "l_partkey": 97, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 43, "l_extendedprice": 42874.87d, "l_discount": 0.02d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-01-11", "l_commitdate": "1996-11-03", "l_receiptdate": "1997-02-01", "l_shipinstruct": "COLLECT COD", "l_shipmode": "REG AIR", "l_comment": "unusual accounts" }
+, { "l_orderkey": 2886, "l_partkey": 60, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 1, "l_extendedprice": 960.06d, "l_discount": 0.09d, "l_tax": 0.05d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1995-02-01", "l_commitdate": "1994-12-18", "l_receiptdate": "1995-02-28", "l_shipinstruct": "COLLECT COD", "l_shipmode": "REG AIR", "l_comment": "eposits fr" }
+, { "l_orderkey": 2912, "l_partkey": 122, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 8, "l_extendedprice": 8176.96d, "l_discount": 0.06d, "l_tax": 0.04d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1992-04-09", "l_commitdate": "1992-04-19", "l_receiptdate": "1992-04-26", "l_shipinstruct": "NONE", "l_shipmode": "RAIL", "l_comment": "hs cajole over the slyl" }
+, { "l_orderkey": 2944, "l_partkey": 120, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 44, "l_extendedprice": 44885.28d, "l_discount": 0.08d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-12-25", "l_commitdate": "1997-10-28", "l_receiptdate": "1998-01-21", "l_shipinstruct": "COLLECT COD", "l_shipmode": "AIR", "l_comment": "ickly special theodolit" }
+, { "l_orderkey": 2947, "l_partkey": 10, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 37, "l_extendedprice": 33670.37d, "l_discount": 0.09d, "l_tax": 0.07d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-08-09", "l_commitdate": "1995-07-05", "l_receiptdate": "1995-08-20", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "RAIL", "l_comment": "e accounts: expres" }
+, { "l_orderkey": 2950, "l_partkey": 130, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 32, "l_extendedprice": 32964.16d, "l_discount": 0.01d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-09-21", "l_commitdate": "1997-08-25", "l_receiptdate": "1997-10-08", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "REG AIR", "l_comment": "its wake carefully slyly final ideas." }
+, { "l_orderkey": 2978, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 29, "l_extendedprice": 28712.61d, "l_discount": 0.0d, "l_tax": 0.08d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1995-06-03", "l_commitdate": "1995-07-25", "l_receiptdate": "1995-06-06", "l_shipinstruct": "NONE", "l_shipmode": "SHIP", "l_comment": "ecial ideas promise slyly" }
+, { "l_orderkey": 3143, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 22, "l_extendedprice": 21781.98d, "l_discount": 0.02d, "l_tax": 0.0d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1993-05-11", "l_commitdate": "1993-03-26", "l_receiptdate": "1993-05-20", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "MAIL", "l_comment": "l, special instructions nag " }
+, { "l_orderkey": 3264, "l_partkey": 200, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 39, "l_extendedprice": 42907.8d, "l_discount": 0.06d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-11-07", "l_commitdate": "1996-12-12", "l_receiptdate": "1996-11-20", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "REG AIR", "l_comment": "sleep carefully after the slyly final" }
+, { "l_orderkey": 3266, "l_partkey": 64, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 31, "l_extendedprice": 29885.86d, "l_discount": 0.09d, "l_tax": 0.02d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-06-19", "l_commitdate": "1995-05-04", "l_receiptdate": "1995-07-06", "l_shipinstruct": "COLLECT COD", "l_shipmode": "MAIL", "l_comment": "grate among the quickly express deposits" }
+, { "l_orderkey": 3270, "l_partkey": 35, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 11, "l_extendedprice": 10285.33d, "l_discount": 0.07d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-07-29", "l_commitdate": "1997-08-11", "l_receiptdate": "1997-08-05", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "AIR", "l_comment": " solve at the regular deposits. " }
+, { "l_orderkey": 3364, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 49, "l_extendedprice": 48514.41d, "l_discount": 0.03d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-09-17", "l_commitdate": "1997-08-23", "l_receiptdate": "1997-10-06", "l_shipinstruct": "NONE", "l_shipmode": "SHIP", "l_comment": "d accounts? caref" }
+, { "l_orderkey": 3366, "l_partkey": 40, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 4, "l_extendedprice": 3760.16d, "l_discount": 0.07d, "l_tax": 0.01d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-05-20", "l_commitdate": "1997-06-25", "l_receiptdate": "1997-06-03", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "AIR", "l_comment": " carefully about " }
+, { "l_orderkey": 3425, "l_partkey": 120, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 11, "l_extendedprice": 11221.32d, "l_discount": 0.03d, "l_tax": 0.08d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-04-24", "l_commitdate": "1996-05-29", "l_receiptdate": "1996-05-23", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "FOB", "l_comment": "ckly final deposits use quickly?" }
+, { "l_orderkey": 3460, "l_partkey": 11, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 40, "l_extendedprice": 36440.4d, "l_discount": 0.1d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-12-28", "l_commitdate": "1995-12-14", "l_receiptdate": "1996-01-02", "l_shipinstruct": "NONE", "l_shipmode": "REG AIR", "l_comment": "odolites are slyly bold deposits" }
+, { "l_orderkey": 3494, "l_partkey": 117, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 40, "l_extendedprice": 40684.4d, "l_discount": 0.05d, "l_tax": 0.04d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-07-10", "l_commitdate": "1993-06-01", "l_receiptdate": "1993-07-25", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "TRUCK", "l_comment": "lites haggle furiously about the fin" }
+, { "l_orderkey": 3520, "l_partkey": 28, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 30, "l_extendedprice": 27840.6d, "l_discount": 0.04d, "l_tax": 0.02d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-11-11", "l_commitdate": "1997-10-02", "l_receiptdate": "1997-12-06", "l_shipinstruct": "COLLECT COD", "l_shipmode": "SHIP", "l_comment": "deas should solve blithely among the ironi" }
+, { "l_orderkey": 3559, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 29, "l_extendedprice": 28712.61d, "l_discount": 0.0d, "l_tax": 0.07d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1992-12-10", "l_commitdate": "1992-12-03", "l_receiptdate": "1992-12-20", "l_shipinstruct": "COLLECT COD", "l_shipmode": "REG AIR", "l_comment": "l, regular accounts wake flu" }
+, { "l_orderkey": 3585, "l_partkey": 122, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 21, "l_extendedprice": 21464.52d, "l_discount": 0.05d, "l_tax": 0.04d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-12-04", "l_commitdate": "1994-12-25", "l_receiptdate": "1995-01-01", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "TRUCK", "l_comment": "ounts use. express, final platelets us" }
+, { "l_orderkey": 3618, "l_partkey": 140, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 38, "l_extendedprice": 39525.32d, "l_discount": 0.08d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-12-22", "l_commitdate": "1998-02-23", "l_receiptdate": "1998-01-03", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "TRUCK", "l_comment": "nts haggle fluffily above the regular " }
+, { "l_orderkey": 3715, "l_partkey": 97, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 13, "l_extendedprice": 12962.17d, "l_discount": 0.0d, "l_tax": 0.03d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-05-11", "l_commitdate": "1996-04-25", "l_receiptdate": "1996-06-09", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "SHIP", "l_comment": "e quickly ironic" }
+, { "l_orderkey": 3844, "l_partkey": 135, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 2, "l_extendedprice": 2070.26d, "l_discount": 0.03d, "l_tax": 0.07d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1995-02-24", "l_commitdate": "1995-02-03", "l_receiptdate": "1995-03-18", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "AIR", "l_comment": "es haggle final acco" }
+, { "l_orderkey": 3878, "l_partkey": 200, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 6, "l_extendedprice": 6601.2d, "l_discount": 0.07d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-06-21", "l_commitdate": "1997-05-22", "l_receiptdate": "1997-07-01", "l_shipinstruct": "COLLECT COD", "l_shipmode": "FOB", "l_comment": "s. regular instru" }
+, { "l_orderkey": 3906, "l_partkey": 153, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 42, "l_extendedprice": 44232.3d, "l_discount": 0.0d, "l_tax": 0.04d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1992-09-03", "l_commitdate": "1992-07-22", "l_receiptdate": "1992-09-04", "l_shipinstruct": "COLLECT COD", "l_shipmode": "RAIL", "l_comment": "jole blithely after the furiously regular " }
+, { "l_orderkey": 3974, "l_partkey": 22, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 47, "l_extendedprice": 43334.94d, "l_discount": 0.1d, "l_tax": 0.03d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-06-03", "l_commitdate": "1996-05-08", "l_receiptdate": "1996-06-28", "l_shipinstruct": "NONE", "l_shipmode": "TRUCK", "l_comment": "dencies above the re" }
+, { "l_orderkey": 4001, "l_partkey": 106, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 26, "l_extendedprice": 26158.6d, "l_discount": 0.0d, "l_tax": 0.01d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-07-26", "l_commitdate": "1997-06-18", "l_receiptdate": "1997-08-08", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "RAIL", "l_comment": "tegrate blithely" }
+, { "l_orderkey": 4005, "l_partkey": 4, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 26, "l_extendedprice": 23504.0d, "l_discount": 0.09d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-12-01", "l_commitdate": "1997-02-03", "l_receiptdate": "1996-12-15", "l_shipinstruct": "NONE", "l_shipmode": "REG AIR", "l_comment": " to the quic" }
+, { "l_orderkey": 4033, "l_partkey": 110, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 27, "l_extendedprice": 27272.97d, "l_discount": 0.01d, "l_tax": 0.04d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-08-08", "l_commitdate": "1993-08-14", "l_receiptdate": "1993-08-09", "l_shipinstruct": "NONE", "l_shipmode": "AIR", "l_comment": "pinto beans" }
+, { "l_orderkey": 4034, "l_partkey": 190, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 48, "l_extendedprice": 52329.12d, "l_discount": 0.03d, "l_tax": 0.03d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-03-01", "l_commitdate": "1994-01-16", "l_receiptdate": "1994-03-16", "l_shipinstruct": "NONE", "l_shipmode": "RAIL", "l_comment": " blithely regular requests play carefull" }
+, { "l_orderkey": 4036, "l_partkey": 6, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 46, "l_extendedprice": 41676.0d, "l_discount": 0.09d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-06-21", "l_commitdate": "1997-05-29", "l_receiptdate": "1997-07-18", "l_shipinstruct": "NONE", "l_shipmode": "REG AIR", "l_comment": "usly across the even th" }
+, { "l_orderkey": 4064, "l_partkey": 199, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 3, "l_extendedprice": 3297.57d, "l_discount": 0.1d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-01-04", "l_commitdate": "1997-01-01", "l_receiptdate": "1997-01-23", "l_shipinstruct": "NONE", "l_shipmode": "SHIP", "l_comment": "its! quickly sp" }
+, { "l_orderkey": 4067, "l_partkey": 180, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 18, "l_extendedprice": 19443.24d, "l_discount": 0.03d, "l_tax": 0.08d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1993-01-24", "l_commitdate": "1992-12-23", "l_receiptdate": "1993-02-20", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "FOB", "l_comment": "e the slyly final packages d" }
+, { "l_orderkey": 4068, "l_partkey": 110, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 43, "l_extendedprice": 43434.73d, "l_discount": 0.05d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-11-28", "l_commitdate": "1996-11-16", "l_receiptdate": "1996-12-22", "l_shipinstruct": "NONE", "l_shipmode": "AIR", "l_comment": "ructions. regular, special packag" }
+, { "l_orderkey": 4098, "l_partkey": 200, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 46, "l_extendedprice": 50609.2d, "l_discount": 0.07d, "l_tax": 0.03d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-01-26", "l_commitdate": "1997-01-27", "l_receiptdate": "1997-02-13", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "SHIP", "l_comment": "e slyly blithely silent deposits. fluff" }
+, { "l_orderkey": 4192, "l_partkey": 11, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 36, "l_extendedprice": 32796.36d, "l_discount": 0.06d, "l_tax": 0.08d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-04-25", "l_commitdate": "1998-05-26", "l_receiptdate": "1998-05-03", "l_shipinstruct": "COLLECT COD", "l_shipmode": "TRUCK", "l_comment": "eodolites sleep" }
+, { "l_orderkey": 4194, "l_partkey": 197, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 43, "l_extendedprice": 47179.17d, "l_discount": 0.08d, "l_tax": 0.06d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-11-06", "l_commitdate": "1994-12-09", "l_receiptdate": "1994-11-16", "l_shipinstruct": "NONE", "l_shipmode": "TRUCK", "l_comment": "olites are after the exp" }
+, { "l_orderkey": 4261, "l_partkey": 110, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 12, "l_extendedprice": 12121.32d, "l_discount": 0.05d, "l_tax": 0.01d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1992-11-01", "l_commitdate": "1993-01-01", "l_receiptdate": "1992-11-12", "l_shipinstruct": "NONE", "l_shipmode": "FOB", "l_comment": "into beans " }
+, { "l_orderkey": 4418, "l_partkey": 35, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 32, "l_extendedprice": 29920.96d, "l_discount": 0.02d, "l_tax": 0.06d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1993-05-28", "l_commitdate": "1993-06-02", "l_receiptdate": "1993-05-30", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": "ly. bold pinto b" }
+, { "l_orderkey": 4422, "l_partkey": 135, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 5, "l_extendedprice": 5175.65d, "l_discount": 0.09d, "l_tax": 0.07d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-07-17", "l_commitdate": "1995-08-13", "l_receiptdate": "1995-07-25", "l_shipinstruct": "NONE", "l_shipmode": "SHIP", "l_comment": "e furiously about t" }
+, { "l_orderkey": 4486, "l_partkey": 135, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 46, "l_extendedprice": 47615.98d, "l_discount": 0.08d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-05-02", "l_commitdate": "1998-04-05", "l_receiptdate": "1998-05-08", "l_shipinstruct": "COLLECT COD", "l_shipmode": "MAIL", "l_comment": "ackages. specia" }
+, { "l_orderkey": 4512, "l_partkey": 162, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 30, "l_extendedprice": 31864.8d, "l_discount": 0.07d, "l_tax": 0.07d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-01-28", "l_commitdate": "1995-12-22", "l_receiptdate": "1996-02-22", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "TRUCK", "l_comment": "ly unusual package" }
+, { "l_orderkey": 4513, "l_partkey": 170, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 29, "l_extendedprice": 31034.93d, "l_discount": 0.03d, "l_tax": 0.01d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-05-18", "l_commitdate": "1996-05-23", "l_receiptdate": "1996-06-08", "l_shipinstruct": "NONE", "l_shipmode": "REG AIR", "l_comment": "cajole. regular packages boost. s" }
+, { "l_orderkey": 4545, "l_partkey": 173, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 38, "l_extendedprice": 40780.46d, "l_discount": 0.06d, "l_tax": 0.06d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-01-27", "l_commitdate": "1993-03-01", "l_receiptdate": "1993-02-04", "l_shipinstruct": "NONE", "l_shipmode": "TRUCK", "l_comment": "nts serve according to th" }
+, { "l_orderkey": 4549, "l_partkey": 159, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 44, "l_extendedprice": 46602.6d, "l_discount": 0.08d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-03-13", "l_commitdate": "1998-04-15", "l_receiptdate": "1998-03-27", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "TRUCK", "l_comment": "ding to the regular, silent requests" }
+, { "l_orderkey": 4551, "l_partkey": 11, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 6, "l_extendedprice": 5466.06d, "l_discount": 0.08d, "l_tax": 0.08d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-05-18", "l_commitdate": "1996-04-23", "l_receiptdate": "1996-06-13", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "TRUCK", "l_comment": "fily silent fo" }
+, { "l_orderkey": 4576, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 5, "l_extendedprice": 4950.45d, "l_discount": 0.09d, "l_tax": 0.03d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-08-23", "l_commitdate": "1996-11-08", "l_receiptdate": "1996-09-20", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "AIR", "l_comment": "ly express, special asymptote" }
+, { "l_orderkey": 4608, "l_partkey": 173, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 30, "l_extendedprice": 32195.1d, "l_discount": 0.08d, "l_tax": 0.05d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-10-08", "l_commitdate": "1994-07-18", "l_receiptdate": "1994-10-25", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "SHIP", "l_comment": "s cajole. slyly " }
+, { "l_orderkey": 4641, "l_partkey": 190, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 45, "l_extendedprice": 49058.55d, "l_discount": 0.07d, "l_tax": 0.03d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-05-11", "l_commitdate": "1993-04-19", "l_receiptdate": "1993-05-21", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "MAIL", "l_comment": " about the close " }
+, { "l_orderkey": 4679, "l_partkey": 190, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 7, "l_extendedprice": 7631.33d, "l_discount": 0.1d, "l_tax": 0.05d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-05-11", "l_commitdate": "1993-04-11", "l_receiptdate": "1993-05-16", "l_shipinstruct": "NONE", "l_shipmode": "TRUCK", "l_comment": "kages. bold, regular packa" }
+, { "l_orderkey": 4769, "l_partkey": 35, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 16, "l_extendedprice": 14960.48d, "l_discount": 0.08d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-07-16", "l_commitdate": "1995-07-05", "l_receiptdate": "1995-07-22", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "FOB", "l_comment": " deposits. slyly even asymptote" }
+, { "l_orderkey": 4802, "l_partkey": 40, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 6, "l_extendedprice": 5640.24d, "l_discount": 0.0d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-04-16", "l_commitdate": "1997-03-25", "l_receiptdate": "1997-04-21", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "SHIP", "l_comment": "unusual accounts wake blithely. b" }
+, { "l_orderkey": 4804, "l_partkey": 128, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 44, "l_extendedprice": 45237.28d, "l_discount": 0.06d, "l_tax": 0.08d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1992-05-02", "l_commitdate": "1992-03-24", "l_receiptdate": "1992-05-28", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "AIR", "l_comment": "aggle quickly among the slyly fi" }
+, { "l_orderkey": 4805, "l_partkey": 150, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 7, "l_extendedprice": 7351.05d, "l_discount": 0.09d, "l_tax": 0.03d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1992-05-01", "l_commitdate": "1992-07-09", "l_receiptdate": "1992-05-09", "l_shipinstruct": "NONE", "l_shipmode": "FOB", "l_comment": " requests. regular deposit" }
+, { "l_orderkey": 4807, "l_partkey": 122, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 9, "l_extendedprice": 9199.08d, "l_discount": 0.04d, "l_tax": 0.08d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-04-23", "l_commitdate": "1997-03-01", "l_receiptdate": "1997-05-15", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "TRUCK", "l_comment": "may are blithely. carefully even pinto b" }
+, { "l_orderkey": 4836, "l_partkey": 162, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 22, "l_extendedprice": 23367.52d, "l_discount": 0.01d, "l_tax": 0.03d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-03-03", "l_commitdate": "1997-02-23", "l_receiptdate": "1997-03-04", "l_shipinstruct": "NONE", "l_shipmode": "SHIP", "l_comment": "al pinto beans. care" }
+, { "l_orderkey": 4837, "l_partkey": 42, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 16, "l_extendedprice": 15072.64d, "l_discount": 0.09d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-08-12", "l_commitdate": "1998-06-06", "l_receiptdate": "1998-08-26", "l_shipinstruct": "COLLECT COD", "l_shipmode": "FOB", "l_comment": "ing requests are blithely regular instructi" }
+, { "l_orderkey": 4898, "l_partkey": 72, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 44, "l_extendedprice": 42771.08d, "l_discount": 0.07d, "l_tax": 0.02d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-09-13", "l_commitdate": "1994-08-18", "l_receiptdate": "1994-09-16", "l_shipinstruct": "NONE", "l_shipmode": "FOB", "l_comment": "y regular grouches about" }
+, { "l_orderkey": 4928, "l_partkey": 100, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 4, "l_extendedprice": 4000.4d, "l_discount": 0.04d, "l_tax": 0.02d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-10-25", "l_commitdate": "1993-12-24", "l_receiptdate": "1993-11-16", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "REG AIR", "l_comment": "bout the slyly final accounts. carefull" }
+, { "l_orderkey": 4929, "l_partkey": 14, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 20, "l_extendedprice": 18280.2d, "l_discount": 0.0d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-03-12", "l_commitdate": "1996-05-23", "l_receiptdate": "1996-03-20", "l_shipinstruct": "COLLECT COD", "l_shipmode": "REG AIR", "l_comment": " final pinto beans detect. final," }
+, { "l_orderkey": 4967, "l_partkey": 71, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 50, "l_extendedprice": 48553.5d, "l_discount": 0.07d, "l_tax": 0.01d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-05-27", "l_commitdate": "1997-05-13", "l_receiptdate": "1997-06-12", "l_shipinstruct": "NONE", "l_shipmode": "REG AIR", "l_comment": "kages. final, unusual accounts c" }
+, { "l_orderkey": 4996, "l_partkey": 56, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 35, "l_extendedprice": 33461.75d, "l_discount": 0.07d, "l_tax": 0.01d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1992-10-30", "l_commitdate": "1992-10-27", "l_receiptdate": "1992-11-05", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "SHIP", "l_comment": "s. unusual, regular dolphins integrate care" }
+, { "l_orderkey": 5028, "l_partkey": 14, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 15, "l_extendedprice": 13710.15d, "l_discount": 0.07d, "l_tax": 0.07d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1992-07-17", "l_commitdate": "1992-07-16", "l_receiptdate": "1992-08-05", "l_shipinstruct": "COLLECT COD", "l_shipmode": "REG AIR", "l_comment": "es are quickly final pains. furiously pend" }
+, { "l_orderkey": 5031, "l_partkey": 50, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 15, "l_extendedprice": 14250.75d, "l_discount": 0.02d, "l_tax": 0.05d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1995-04-01", "l_commitdate": "1995-02-24", "l_receiptdate": "1995-04-12", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "AIR", "l_comment": "yly pending theodolites." }
+, { "l_orderkey": 5092, "l_partkey": 164, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 30, "l_extendedprice": 31924.8d, "l_discount": 0.06d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-12-27", "l_commitdate": "1995-12-08", "l_receiptdate": "1996-01-09", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "MAIL", "l_comment": "ss, ironic deposits. furiously stea" }
+, { "l_orderkey": 5153, "l_partkey": 35, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 42, "l_extendedprice": 39271.26d, "l_discount": 0.03d, "l_tax": 0.01d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-10-03", "l_commitdate": "1995-11-09", "l_receiptdate": "1995-10-11", "l_shipinstruct": "COLLECT COD", "l_shipmode": "RAIL", "l_comment": "re thinly. ironic" }
+, { "l_orderkey": 5154, "l_partkey": 190, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 11, "l_extendedprice": 11992.09d, "l_discount": 0.02d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-08-06", "l_commitdate": "1997-06-30", "l_receiptdate": "1997-09-04", "l_shipinstruct": "NONE", "l_shipmode": "RAIL", "l_comment": "luffily bold foxes. final" }
+, { "l_orderkey": 5185, "l_partkey": 197, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 37, "l_extendedprice": 40596.03d, "l_discount": 0.0d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-08-08", "l_commitdate": "1997-09-08", "l_receiptdate": "1997-08-14", "l_shipinstruct": "COLLECT COD", "l_shipmode": "SHIP", "l_comment": "gainst the courts dazzle care" }
+, { "l_orderkey": 5187, "l_partkey": 11, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 49, "l_extendedprice": 44639.49d, "l_discount": 0.04d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-10-20", "l_commitdate": "1997-10-12", "l_receiptdate": "1997-10-26", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "AIR", "l_comment": "l, regular platelets instead of the foxes w" }
+, { "l_orderkey": 5190, "l_partkey": 56, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 43, "l_extendedprice": 41110.15d, "l_discount": 0.09d, "l_tax": 0.06d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1992-08-19", "l_commitdate": "1992-06-10", "l_receiptdate": "1992-09-01", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "FOB", "l_comment": "encies use fluffily unusual requests? hoc" }
+, { "l_orderkey": 5217, "l_partkey": 80, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 50, "l_extendedprice": 49004.0d, "l_discount": 0.05d, "l_tax": 0.02d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-12-26", "l_commitdate": "1995-11-21", "l_receiptdate": "1996-01-24", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "MAIL", "l_comment": "s. express, express accounts c" }
+, { "l_orderkey": 5281, "l_partkey": 114, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 37, "l_extendedprice": 37522.07d, "l_discount": 0.05d, "l_tax": 0.02d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-11-10", "l_commitdate": "1996-01-31", "l_receiptdate": "1995-11-22", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "MAIL", "l_comment": "ronic dependencies. fluffily final p" }
+, { "l_orderkey": 5284, "l_partkey": 173, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 16, "l_extendedprice": 17170.72d, "l_discount": 0.04d, "l_tax": 0.02d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-08-17", "l_commitdate": "1995-08-23", "l_receiptdate": "1995-08-26", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "TRUCK", "l_comment": "unts detect furiously even d" }
+, { "l_orderkey": 5286, "l_partkey": 199, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 1, "l_extendedprice": 1099.19d, "l_discount": 0.01d, "l_tax": 0.07d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-11-25", "l_commitdate": "1997-11-07", "l_receiptdate": "1997-12-17", "l_shipinstruct": "COLLECT COD", "l_shipmode": "REG AIR", "l_comment": "ly! furiously final pack" }
+, { "l_orderkey": 5313, "l_partkey": 17, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 34, "l_extendedprice": 31178.34d, "l_discount": 0.1d, "l_tax": 0.02d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-08-07", "l_commitdate": "1997-08-12", "l_receiptdate": "1997-08-24", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "FOB", "l_comment": "ccording to the blithely final account" }
+, { "l_orderkey": 5315, "l_partkey": 35, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 12, "l_extendedprice": 11220.36d, "l_discount": 0.08d, "l_tax": 0.06d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1992-12-18", "l_commitdate": "1993-01-16", "l_receiptdate": "1993-01-10", "l_shipinstruct": "NONE", "l_shipmode": "AIR", "l_comment": "ccounts. furiously ironi" }
+, { "l_orderkey": 5316, "l_partkey": 108, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 29, "l_extendedprice": 29234.9d, "l_discount": 0.1d, "l_tax": 0.05d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-03-28", "l_commitdate": "1994-04-29", "l_receiptdate": "1994-04-09", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "REG AIR", "l_comment": "ckly unusual foxes bo" }
+, { "l_orderkey": 5379, "l_partkey": 199, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 40, "l_extendedprice": 43967.6d, "l_discount": 0.01d, "l_tax": 0.08d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-10-01", "l_commitdate": "1995-10-19", "l_receiptdate": "1995-10-30", "l_shipinstruct": "COLLECT COD", "l_shipmode": "MAIL", "l_comment": "carefully final accounts haggle blithely. " }
+, { "l_orderkey": 5441, "l_partkey": 164, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 3, "l_extendedprice": 3192.48d, "l_discount": 0.0d, "l_tax": 0.02d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-08-12", "l_commitdate": "1994-10-14", "l_receiptdate": "1994-09-01", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "REG AIR", "l_comment": "are. unusual, " }
+, { "l_orderkey": 5445, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 33, "l_extendedprice": 32672.97d, "l_discount": 0.08d, "l_tax": 0.07d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1993-10-21", "l_commitdate": "1993-10-14", "l_receiptdate": "1993-10-28", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "REG AIR", "l_comment": "ests. final instructions" }
+, { "l_orderkey": 5446, "l_partkey": 190, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 27, "l_extendedprice": 29435.13d, "l_discount": 0.05d, "l_tax": 0.07d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-07-21", "l_commitdate": "1994-08-25", "l_receiptdate": "1994-08-17", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": "ously across the quic" }
+, { "l_orderkey": 5447, "l_partkey": 99, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 31, "l_extendedprice": 30971.79d, "l_discount": 0.09d, "l_tax": 0.03d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-07-14", "l_commitdate": "1996-05-07", "l_receiptdate": "1996-07-17", "l_shipinstruct": "COLLECT COD", "l_shipmode": "SHIP", "l_comment": " foxes sleep. blithely unusual accounts det" }
+, { "l_orderkey": 5476, "l_partkey": 48, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 13, "l_extendedprice": 12324.52d, "l_discount": 0.01d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-12-27", "l_commitdate": "1997-12-08", "l_receiptdate": "1997-12-29", "l_shipinstruct": "COLLECT COD", "l_shipmode": "TRUCK", "l_comment": "iously special ac" }
+, { "l_orderkey": 5506, "l_partkey": 140, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 2, "l_extendedprice": 2080.28d, "l_discount": 0.0d, "l_tax": 0.03d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-02-04", "l_commitdate": "1994-01-13", "l_receiptdate": "1994-02-17", "l_shipinstruct": "COLLECT COD", "l_shipmode": "MAIL", "l_comment": "onic theodolites are fluffil" }
+, { "l_orderkey": 5536, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 14, "l_extendedprice": 13861.26d, "l_discount": 0.08d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-05-18", "l_commitdate": "1998-05-08", "l_receiptdate": "1998-06-05", "l_shipinstruct": "COLLECT COD", "l_shipmode": "MAIL", "l_comment": "instructions sleep " }
+, { "l_orderkey": 5572, "l_partkey": 22, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 24, "l_extendedprice": 22128.48d, "l_discount": 0.08d, "l_tax": 0.08d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-10-30", "l_commitdate": "1994-10-02", "l_receiptdate": "1994-11-27", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "MAIL", "l_comment": "ests cajole. evenly ironic exc" }
+, { "l_orderkey": 5664, "l_partkey": 122, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 25, "l_extendedprice": 25553.0d, "l_discount": 0.0d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-10-29", "l_commitdate": "1998-09-23", "l_receiptdate": "1998-11-25", "l_shipinstruct": "COLLECT COD", "l_shipmode": "FOB", "l_comment": "eposits: furiously ironic grouch" }
+, { "l_orderkey": 5670, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 27, "l_extendedprice": 26732.43d, "l_discount": 0.1d, "l_tax": 0.06d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-05-09", "l_commitdate": "1993-05-30", "l_receiptdate": "1993-06-06", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "REG AIR", "l_comment": " ideas promise bli" }
+, { "l_orderkey": 5728, "l_partkey": 44, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 47, "l_extendedprice": 44369.88d, "l_discount": 0.1d, "l_tax": 0.05d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-12-13", "l_commitdate": "1995-01-25", "l_receiptdate": "1994-12-25", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "MAIL", "l_comment": "nd the bravely final deposits. final ideas" }
+, { "l_orderkey": 5735, "l_partkey": 60, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 41, "l_extendedprice": 39362.46d, "l_discount": 0.01d, "l_tax": 0.01d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-12-23", "l_commitdate": "1995-02-10", "l_receiptdate": "1995-01-22", "l_shipinstruct": "COLLECT COD", "l_shipmode": "MAIL", "l_comment": "lthily ruthless i" }
+, { "l_orderkey": 5826, "l_partkey": 144, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 4, "l_extendedprice": 4176.56d, "l_discount": 0.03d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-07-31", "l_commitdate": "1998-09-10", "l_receiptdate": "1998-08-27", "l_shipinstruct": "NONE", "l_shipmode": "AIR", "l_comment": " packages across the fluffily spec" }
+, { "l_orderkey": 5829, "l_partkey": 40, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 4, "l_extendedprice": 3760.16d, "l_discount": 0.01d, "l_tax": 0.02d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-03-01", "l_commitdate": "1997-02-17", "l_receiptdate": "1997-03-22", "l_shipinstruct": "NONE", "l_shipmode": "TRUCK", "l_comment": "ithely; accounts cajole ideas. regular foxe" }
+, { "l_orderkey": 5856, "l_partkey": 4, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 1, "l_extendedprice": 904.0d, "l_discount": 0.03d, "l_tax": 0.02d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-12-29", "l_commitdate": "1995-01-07", "l_receiptdate": "1995-01-10", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "MAIL", "l_comment": "tly. special deposits wake blithely even" }
+, { "l_orderkey": 5926, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 8, "l_extendedprice": 7920.72d, "l_discount": 0.02d, "l_tax": 0.0d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-07-17", "l_commitdate": "1994-07-20", "l_receiptdate": "1994-08-11", "l_shipinstruct": "COLLECT COD", "l_shipmode": "MAIL", "l_comment": "gle furiously express foxes. bo" }
+, { "l_orderkey": 5927, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 44, "l_extendedprice": 43563.96d, "l_discount": 0.04d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-11-29", "l_commitdate": "1997-11-21", "l_receiptdate": "1997-12-13", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "TRUCK", "l_comment": "rding to the special, final decoy" }
+, { "l_orderkey": 5955, "l_partkey": 140, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 14, "l_extendedprice": 14561.96d, "l_discount": 0.08d, "l_tax": 0.08d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-06-22", "l_commitdate": "1995-05-23", "l_receiptdate": "1995-06-24", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "TRUCK", "l_comment": " unusual, bold theodolit" }
+, { "l_orderkey": 5959, "l_partkey": 135, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 49, "l_extendedprice": 50721.37d, "l_discount": 0.07d, "l_tax": 0.03d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1992-07-16", "l_commitdate": "1992-08-09", "l_receiptdate": "1992-08-14", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "SHIP", "l_comment": "usual packages haggle slyly pi" }
+, { "l_orderkey": 5988, "l_partkey": 172, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 41, "l_extendedprice": 43958.97d, "l_discount": 0.08d, "l_tax": 0.03d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-01-20", "l_commitdate": "1994-02-06", "l_receiptdate": "1994-02-10", "l_shipinstruct": "COLLECT COD", "l_shipmode": "AIR", "l_comment": "the pending, express reque" }
+ ]
diff --git a/asterix-app/src/test/resources/runtimets/results/temp-dataset/delete-from-loaded-dataset/delete-from-loaded-dataset.1.adm b/asterix-app/src/test/resources/runtimets/results/temp-dataset/delete-from-loaded-dataset/delete-from-loaded-dataset.1.adm
new file mode 100644
index 0000000..f4c601d
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/temp-dataset/delete-from-loaded-dataset/delete-from-loaded-dataset.1.adm
@@ -0,0 +1,26 @@
+[ { "l_orderkey": 1, "l_partkey": 156, "l_suppkey": 4, "l_linenumber": 1, "l_quantity": 17, "l_extendedprice": 17954.55d, "l_discount": 0.04d, "l_tax": 0.02d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-03-13", "l_commitdate": "1996-02-12", "l_receiptdate": "1996-03-22", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "TRUCK", "l_comment": "egular courts above the" }
+, { "l_orderkey": 1, "l_partkey": 68, "l_suppkey": 9, "l_linenumber": 2, "l_quantity": 36, "l_extendedprice": 34850.16d, "l_discount": 0.09d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-04-12", "l_commitdate": "1996-02-28", "l_receiptdate": "1996-04-20", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "MAIL", "l_comment": "ly final dependencies: slyly bold " }
+, { "l_orderkey": 1, "l_partkey": 64, "l_suppkey": 5, "l_linenumber": 3, "l_quantity": 8, "l_extendedprice": 7712.48d, "l_discount": 0.1d, "l_tax": 0.02d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-01-29", "l_commitdate": "1996-03-05", "l_receiptdate": "1996-01-31", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "REG AIR", "l_comment": "riously. regular, express dep" }
+, { "l_orderkey": 1, "l_partkey": 3, "l_suppkey": 6, "l_linenumber": 4, "l_quantity": 28, "l_extendedprice": 25284.0d, "l_discount": 0.09d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-04-21", "l_commitdate": "1996-03-30", "l_receiptdate": "1996-05-16", "l_shipinstruct": "NONE", "l_shipmode": "AIR", "l_comment": "lites. fluffily even de" }
+, { "l_orderkey": 1, "l_partkey": 25, "l_suppkey": 8, "l_linenumber": 5, "l_quantity": 24, "l_extendedprice": 22200.48d, "l_discount": 0.1d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-03-30", "l_commitdate": "1996-03-14", "l_receiptdate": "1996-04-01", "l_shipinstruct": "NONE", "l_shipmode": "FOB", "l_comment": " pending foxes. slyly re" }
+, { "l_orderkey": 1, "l_partkey": 16, "l_suppkey": 3, "l_linenumber": 6, "l_quantity": 32, "l_extendedprice": 29312.32d, "l_discount": 0.07d, "l_tax": 0.02d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-01-30", "l_commitdate": "1996-02-07", "l_receiptdate": "1996-02-03", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "MAIL", "l_comment": "arefully slyly ex" }
+, { "l_orderkey": 2, "l_partkey": 107, "l_suppkey": 2, "l_linenumber": 1, "l_quantity": 38, "l_extendedprice": 38269.8d, "l_discount": 0.0d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-01-28", "l_commitdate": "1997-01-14", "l_receiptdate": "1997-02-02", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": "ven requests. deposits breach a" }
+, { "l_orderkey": 3, "l_partkey": 5, "l_suppkey": 2, "l_linenumber": 1, "l_quantity": 45, "l_extendedprice": 40725.0d, "l_discount": 0.06d, "l_tax": 0.0d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-02-02", "l_commitdate": "1994-01-04", "l_receiptdate": "1994-02-23", "l_shipinstruct": "NONE", "l_shipmode": "AIR", "l_comment": "ongside of the furiously brave acco" }
+, { "l_orderkey": 3, "l_partkey": 20, "l_suppkey": 10, "l_linenumber": 2, "l_quantity": 49, "l_extendedprice": 45080.98d, "l_discount": 0.1d, "l_tax": 0.0d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-11-09", "l_commitdate": "1993-12-20", "l_receiptdate": "1993-11-24", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": " unusual accounts. eve" }
+, { "l_orderkey": 3, "l_partkey": 129, "l_suppkey": 8, "l_linenumber": 3, "l_quantity": 27, "l_extendedprice": 27786.24d, "l_discount": 0.06d, "l_tax": 0.07d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-01-16", "l_commitdate": "1993-11-22", "l_receiptdate": "1994-01-23", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "SHIP", "l_comment": "nal foxes wake. " }
+, { "l_orderkey": 3, "l_partkey": 30, "l_suppkey": 5, "l_linenumber": 4, "l_quantity": 2, "l_extendedprice": 1860.06d, "l_discount": 0.01d, "l_tax": 0.06d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1993-12-04", "l_commitdate": "1994-01-07", "l_receiptdate": "1994-01-01", "l_shipinstruct": "NONE", "l_shipmode": "TRUCK", "l_comment": "y. fluffily pending d" }
+, { "l_orderkey": 3, "l_partkey": 184, "l_suppkey": 5, "l_linenumber": 5, "l_quantity": 28, "l_extendedprice": 30357.04d, "l_discount": 0.04d, "l_tax": 0.0d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-12-14", "l_commitdate": "1994-01-10", "l_receiptdate": "1994-01-01", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "FOB", "l_comment": "ages nag slyly pending" }
+, { "l_orderkey": 3, "l_partkey": 63, "l_suppkey": 8, "l_linenumber": 6, "l_quantity": 26, "l_extendedprice": 25039.56d, "l_discount": 0.1d, "l_tax": 0.02d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1993-10-29", "l_commitdate": "1993-12-18", "l_receiptdate": "1993-11-04", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": "ges sleep after the caref" }
+, { "l_orderkey": 4, "l_partkey": 89, "l_suppkey": 10, "l_linenumber": 1, "l_quantity": 30, "l_extendedprice": 29672.4d, "l_discount": 0.03d, "l_tax": 0.08d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-01-10", "l_commitdate": "1995-12-14", "l_receiptdate": "1996-01-18", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "REG AIR", "l_comment": "- quickly regular packages sleep. idly" }
+, { "l_orderkey": 5, "l_partkey": 109, "l_suppkey": 10, "l_linenumber": 1, "l_quantity": 15, "l_extendedprice": 15136.5d, "l_discount": 0.02d, "l_tax": 0.04d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-10-31", "l_commitdate": "1994-08-31", "l_receiptdate": "1994-11-20", "l_shipinstruct": "NONE", "l_shipmode": "AIR", "l_comment": "ts wake furiously " }
+, { "l_orderkey": 5, "l_partkey": 124, "l_suppkey": 5, "l_linenumber": 2, "l_quantity": 26, "l_extendedprice": 26627.12d, "l_discount": 0.07d, "l_tax": 0.08d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-10-16", "l_commitdate": "1994-09-25", "l_receiptdate": "1994-10-19", "l_shipinstruct": "NONE", "l_shipmode": "FOB", "l_comment": "sts use slyly quickly special instruc" }
+, { "l_orderkey": 5, "l_partkey": 38, "l_suppkey": 4, "l_linenumber": 3, "l_quantity": 50, "l_extendedprice": 46901.5d, "l_discount": 0.08d, "l_tax": 0.03d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-08-08", "l_commitdate": "1994-10-13", "l_receiptdate": "1994-08-26", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "AIR", "l_comment": "eodolites. fluffily unusual" }
+, { "l_orderkey": 6, "l_partkey": 140, "l_suppkey": 6, "l_linenumber": 1, "l_quantity": 37, "l_extendedprice": 38485.18d, "l_discount": 0.08d, "l_tax": 0.03d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1992-04-27", "l_commitdate": "1992-05-15", "l_receiptdate": "1992-05-02", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "TRUCK", "l_comment": "p furiously special foxes" }
+, { "l_orderkey": 7, "l_partkey": 183, "l_suppkey": 4, "l_linenumber": 1, "l_quantity": 12, "l_extendedprice": 12998.16d, "l_discount": 0.07d, "l_tax": 0.03d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-05-07", "l_commitdate": "1996-03-13", "l_receiptdate": "1996-06-03", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "FOB", "l_comment": "ss pinto beans wake against th" }
+, { "l_orderkey": 7, "l_partkey": 146, "l_suppkey": 3, "l_linenumber": 2, "l_quantity": 9, "l_extendedprice": 9415.26d, "l_discount": 0.08d, "l_tax": 0.08d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-02-01", "l_commitdate": "1996-03-02", "l_receiptdate": "1996-02-19", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "SHIP", "l_comment": "es. instructions" }
+, { "l_orderkey": 7, "l_partkey": 95, "l_suppkey": 8, "l_linenumber": 3, "l_quantity": 46, "l_extendedprice": 45774.14d, "l_discount": 0.1d, "l_tax": 0.07d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-01-15", "l_commitdate": "1996-03-27", "l_receiptdate": "1996-02-03", "l_shipinstruct": "COLLECT COD", "l_shipmode": "MAIL", "l_comment": " unusual reques" }
+, { "l_orderkey": 7, "l_partkey": 164, "l_suppkey": 5, "l_linenumber": 4, "l_quantity": 28, "l_extendedprice": 29796.48d, "l_discount": 0.03d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-03-21", "l_commitdate": "1996-04-08", "l_receiptdate": "1996-04-20", "l_shipinstruct": "NONE", "l_shipmode": "FOB", "l_comment": ". slyly special requests haggl" }
+, { "l_orderkey": 7, "l_partkey": 152, "l_suppkey": 4, "l_linenumber": 5, "l_quantity": 38, "l_extendedprice": 39981.7d, "l_discount": 0.08d, "l_tax": 0.01d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-02-11", "l_commitdate": "1996-02-24", "l_receiptdate": "1996-02-18", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "TRUCK", "l_comment": "ns haggle carefully ironic deposits. bl" }
+, { "l_orderkey": 7, "l_partkey": 80, "l_suppkey": 10, "l_linenumber": 6, "l_quantity": 35, "l_extendedprice": 34302.8d, "l_discount": 0.06d, "l_tax": 0.03d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-01-16", "l_commitdate": "1996-02-23", "l_receiptdate": "1996-01-22", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "FOB", "l_comment": "jole. excuses wake carefully alongside of " }
+, { "l_orderkey": 7, "l_partkey": 158, "l_suppkey": 3, "l_linenumber": 7, "l_quantity": 5, "l_extendedprice": 5290.75d, "l_discount": 0.04d, "l_tax": 0.02d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-02-10", "l_commitdate": "1996-03-26", "l_receiptdate": "1996-02-13", "l_shipinstruct": "NONE", "l_shipmode": "FOB", "l_comment": "ithely regula" }
+ ]
diff --git a/asterix-app/src/test/resources/runtimets/results/temp-dataset/drop-empty-secondary-indexes/drop-empty-secondary-indexes.1.adm b/asterix-app/src/test/resources/runtimets/results/temp-dataset/drop-empty-secondary-indexes/drop-empty-secondary-indexes.1.adm
new file mode 100644
index 0000000..f6ffad5
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/temp-dataset/drop-empty-secondary-indexes/drop-empty-secondary-indexes.1.adm
@@ -0,0 +1 @@
+[ ]
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/temp-dataset/drop-index/drop-index.1.adm b/asterix-app/src/test/resources/runtimets/results/temp-dataset/drop-index/drop-index.1.adm
new file mode 100644
index 0000000..4ce95bd
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/temp-dataset/drop-index/drop-index.1.adm
@@ -0,0 +1,2 @@
+[ { "unique1": 84, "unique2": 10, "two": 0, "four": 0, "ten": 4, "twenty": 4, "onePercent": 84, "tenPercent": 4, "twentyPercent": 4, "fiftyPercent": 0, "unique3": 84, "evenOnePercent": 168, "oddOnePercent": 169, "stringu1": "DGAAAAXXXXXXXXXXXXXXXXXXX", "stringu2": "KAAAAAXXXXXXXXXXXXXXXXXXX", "string4": "OOOOXXXXXXXXXXXXXXXXXXXXXX" }
+ ]
diff --git a/asterix-app/src/test/resources/runtimets/results/temp-dataset/empty-load-with-index/empty-load-with-index.1.adm b/asterix-app/src/test/resources/runtimets/results/temp-dataset/empty-load-with-index/empty-load-with-index.1.adm
new file mode 100644
index 0000000..d4d073a
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/temp-dataset/empty-load-with-index/empty-load-with-index.1.adm
@@ -0,0 +1,2 @@
+[ { "l_orderkey": 1, "l_partkey": 156, "l_suppkey": 4, "l_linenumber": 1, "l_quantity": 17.0d, "l_extendedprice": 17954.55d, "l_discount": 0.04d, "l_tax": 0.02d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-03-13", "l_commitdate": "1996-02-12", "l_receiptdate": "1996-03-22", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "TRUCK", "l_comment": "egular courts above the" }
+ ]
diff --git a/asterix-app/src/test/resources/runtimets/results/temp-dataset/insert-and-scan-dataset-with-index/insert-and-scan-dataset-with-index.1.adm b/asterix-app/src/test/resources/runtimets/results/temp-dataset/insert-and-scan-dataset-with-index/insert-and-scan-dataset-with-index.1.adm
new file mode 100644
index 0000000..f889b2d
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/temp-dataset/insert-and-scan-dataset-with-index/insert-and-scan-dataset-with-index.1.adm
@@ -0,0 +1,241 @@
+[ { "id": 101, "fname": "Javier", "lname": "Makuch", "age": 28, "dept": "IT" }
+, { "id": 110, "fname": "Allan", "lname": "Piland", "age": 29, "dept": "HR" }
+, { "id": 112, "fname": "Pearlie", "lname": "Aumann", "age": 31, "dept": "Payroll" }
+, { "id": 113, "fname": "Chandra", "lname": "Hase", "age": 34, "dept": "Sales" }
+, { "id": 114, "fname": "Christian", "lname": "Convery", "age": 28, "dept": "HR" }
+, { "id": 115, "fname": "Panther", "lname": "Ritch", "age": 26, "dept": "IT" }
+, { "id": 116, "fname": "Ted", "lname": "Elsea", "age": 26, "dept": "IT" }
+, { "id": 117, "fname": "Tabatha", "lname": "Bladen", "age": 25, "dept": "HR" }
+, { "id": 118, "fname": "Clayton", "lname": "Oltman", "age": 42, "dept": "Sales" }
+, { "id": 119, "fname": "Sharron", "lname": "Darwin", "age": 32, "dept": "Payroll" }
+, { "id": 210, "fname": "Clayton", "lname": "Durgin", "age": 52, "dept": "HR" }
+, { "id": 212, "fname": "Emilia", "lname": "Chenail", "age": 26, "dept": "Sales" }
+, { "id": 213, "fname": "Kenya", "lname": "Almquist", "age": 43, "dept": "Payroll" }
+, { "id": 214, "fname": "Alejandra", "lname": "Lacefield", "age": 41, "dept": "HR" }
+, { "id": 215, "fname": "Karina", "lname": "Michelsen", "age": 46, "dept": "IT" }
+, { "id": 216, "fname": "Katy", "lname": "Delillo", "age": 36, "dept": "IT" }
+, { "id": 217, "fname": "Benita", "lname": "Kleist", "age": 37, "dept": "HR" }
+, { "id": 218, "fname": "Earlene", "lname": "Paluch", "age": 31, "dept": "IT" }
+, { "id": 219, "fname": "Kurt", "lname": "Petermann", "age": 27, "dept": "Payroll" }
+, { "id": 299, "fname": "Julio", "lname": "Iorio", "age": 37, "dept": "IT" }
+, { "id": 363, "fname": "Cody", "lname": "Rodreguez", "age": 26, "dept": "IT" }
+, { "id": 404, "fname": "Emilia", "lname": "Square", "age": 32, "dept": "IT" }
+, { "id": 414, "fname": "Mathew", "lname": "Fuschetto", "age": 34, "dept": "HR" }
+, { "id": 424, "fname": "Allyson", "lname": "Remus", "age": 32, "dept": "IT" }
+, { "id": 434, "fname": "Earlene", "lname": "Linebarger", "age": 26, "dept": "Payroll" }
+, { "id": 444, "fname": "Clinton", "lname": "Sick", "age": 29, "dept": "IT" }
+, { "id": 454, "fname": "Ted", "lname": "Caba", "age": 28, "dept": "HR" }
+, { "id": 463, "fname": "Marcie", "lname": "States", "age": 28, "dept": "IT" }
+, { "id": 464, "fname": "Fernando", "lname": "Engelke", "age": 39, "dept": "IT" }
+, { "id": 474, "fname": "Mathew", "lname": "Courchesne", "age": 31, "dept": "IT" }
+, { "id": 484, "fname": "Cody", "lname": "Vinyard", "age": 36, "dept": "Payroll" }
+, { "id": 494, "fname": "Benita", "lname": "Fravel", "age": 33, "dept": "Sales" }
+, { "id": 504, "fname": "Erik", "lname": "Dobek", "age": 29, "dept": "IT" }
+, { "id": 514, "fname": "Julio", "lname": "Ruben", "age": 41, "dept": "IT" }
+, { "id": 524, "fname": "Benita", "lname": "Maltos", "age": 33, "dept": "IT" }
+, { "id": 534, "fname": "Kurt", "lname": "Biscoe", "age": 36, "dept": "HR" }
+, { "id": 538, "fname": "Milagros", "lname": "Forkey", "age": 34, "dept": "Sales" }
+, { "id": 544, "fname": "Loraine", "lname": "Housel", "age": 30, "dept": "Sales" }
+, { "id": 554, "fname": "Jamie", "lname": "Rachal", "age": 30, "dept": "IT" }
+, { "id": 564, "fname": "Liza", "lname": "Fredenburg", "age": 37, "dept": "IT" }
+, { "id": 574, "fname": "Ericka", "lname": "Feldmann", "age": 29, "dept": "Sales" }
+, { "id": 584, "fname": "Dollie", "lname": "Dattilo", "age": 32, "dept": "Payroll" }
+, { "id": 589, "fname": "Lorrie", "lname": "Sharon", "age": 27, "dept": "IT" }
+, { "id": 594, "fname": "Roxie", "lname": "Houghtaling", "age": 40, "dept": "Payroll" }
+, { "id": 601, "fname": "Neil", "lname": "Deforge", "age": 26, "dept": "HR" }
+, { "id": 611, "fname": "Earlene", "lname": "Marcy", "age": 32, "dept": "IT" }
+, { "id": 621, "fname": "Erik", "lname": "Lechuga", "age": 42, "dept": "Payroll" }
+, { "id": 631, "fname": "Tyrone", "lname": "Holtzclaw", "age": 34, "dept": "Sales" }
+, { "id": 641, "fname": "Lance", "lname": "Hankey", "age": 35, "dept": "Sales" }
+, { "id": 651, "fname": "Mallory", "lname": "Gladding", "age": 31, "dept": "HR" }
+, { "id": 661, "fname": "Tia", "lname": "Braaten", "age": 40, "dept": "IT" }
+, { "id": 671, "fname": "Julio", "lname": "Vanpatten", "age": 30, "dept": "Payroll" }
+, { "id": 681, "fname": "Max", "lname": "Teachout", "age": 34, "dept": "IT" }
+, { "id": 691, "fname": "Karina", "lname": "Wingerter", "age": 31, "dept": "IT" }
+, { "id": 711, "fname": "Hugh", "lname": "Lema", "age": 25, "dept": "HR" }
+, { "id": 721, "fname": "Schwan", "lname": "Phil", "age": 34, "dept": "Payroll" }
+, { "id": 732, "fname": "Noemi", "lname": "Eacret", "age": 56, "dept": "HR" }
+, { "id": 741, "fname": "Julio", "lname": "Mattocks", "age": 38, "dept": "Sales" }
+, { "id": 751, "fname": "Lance", "lname": "Kottke", "age": 34, "dept": "IT" }
+, { "id": 761, "fname": "Kurt", "lname": "Liz", "age": 32, "dept": "HR" }
+, { "id": 771, "fname": "Neva", "lname": "Barbeau", "age": 45, "dept": "Sales" }
+, { "id": 781, "fname": "Karina", "lname": "Tuthill", "age": 46, "dept": "Payroll" }
+, { "id": 791, "fname": "Maricela", "lname": "Cambron", "age": 36, "dept": "IT" }
+, { "id": 809, "fname": "Clayton", "lname": "Delany", "age": 23, "dept": "IT" }
+, { "id": 811, "fname": "Kubik", "lname": "Kuhn", "age": 27, "dept": "HR" }
+, { "id": 821, "fname": "Allan", "lname": "Tomes", "age": 29, "dept": "Payroll" }
+, { "id": 831, "fname": "Lonnie", "lname": "Aller", "age": 33, "dept": "Sales" }
+, { "id": 841, "fname": "Neil", "lname": "Hurrell", "age": 26, "dept": "IT" }
+, { "id": 851, "fname": "Clayton", "lname": "Engles", "age": 41, "dept": "HR" }
+, { "id": 861, "fname": "Javier", "lname": "Gabrielson", "age": 39, "dept": "Payroll" }
+, { "id": 871, "fname": "Allan", "lname": "Alejandre", "age": 48, "dept": "IT" }
+, { "id": 881, "fname": "Julio", "lname": "Isa", "age": 38, "dept": "Sales" }
+, { "id": 891, "fname": "Roslyn", "lname": "Simmerman", "age": 31, "dept": "IT" }
+, { "id": 915, "fname": "Starner", "lname": "Stuart", "age": 25, "dept": "Sales" }
+, { "id": 925, "fname": "Sofia", "lname": "Cuff", "age": 30, "dept": "HR" }
+, { "id": 935, "fname": "Milagros", "lname": "Murguia", "age": 31, "dept": "IT" }
+, { "id": 945, "fname": "Margery", "lname": "Haldeman", "age": 32, "dept": "IT" }
+, { "id": 955, "fname": "Max", "lname": "Mell", "age": 33, "dept": "HR" }
+, { "id": 965, "fname": "Micco", "lname": "Mercy", "age": 31, "dept": "Payroll" }
+, { "id": 975, "fname": "Clare", "lname": "Vangieson", "age": 34, "dept": "IT" }
+, { "id": 985, "fname": "Elnora", "lname": "Dimauro", "age": 35, "dept": "Sales" }
+, { "id": 995, "fname": "Pearlie", "lname": "Kocian", "age": 38, "dept": "HR" }
+, { "id": 1007, "fname": "Yingyi", "lname": "Bu", "age": 27, "dept": "IT" }
+, { "id": 1263, "fname": "Tania", "lname": "Loffredo", "age": 25, "dept": "IT" }
+, { "id": 1410, "fname": "Clinton", "lname": "Fredricks", "age": 34, "dept": "IT" }
+, { "id": 1411, "fname": "Lance", "lname": "Farquhar", "age": 32, "dept": "HR" }
+, { "id": 1412, "fname": "Tabatha", "lname": "Crisler", "age": 33, "dept": "IT" }
+, { "id": 1413, "fname": "Max", "lname": "Durney", "age": 29, "dept": "IT" }
+, { "id": 1414, "fname": "Carmella", "lname": "Strauser", "age": 30, "dept": "Payroll" }
+, { "id": 1415, "fname": "Kelly", "lname": "Carrales", "age": 40, "dept": "IT" }
+, { "id": 1416, "fname": "Guy", "lname": "Merten", "age": 29, "dept": "Sales" }
+, { "id": 1417, "fname": "Noreen", "lname": "Ruhland", "age": 29, "dept": "IT" }
+, { "id": 1418, "fname": "Julio", "lname": "Damore", "age": 27, "dept": "Sales" }
+, { "id": 1419, "fname": "Selena", "lname": "Truby", "age": 25, "dept": "HR" }
+, { "id": 1420, "fname": "Alejandra", "lname": "Commons", "age": 30, "dept": "Sales" }
+, { "id": 1421, "fname": "Allyson", "lname": "Balk", "age": 30, "dept": "IT" }
+, { "id": 1422, "fname": "Nelson", "lname": "Byun", "age": 40, "dept": "Sales" }
+, { "id": 1423, "fname": "Christian", "lname": "Reidhead", "age": 40, "dept": "IT" }
+, { "id": 1424, "fname": "Pearlie", "lname": "Hopkin", "age": 48, "dept": "Payroll" }
+, { "id": 1425, "fname": "Nelson", "lname": "Wohlers", "age": 41, "dept": "HR" }
+, { "id": 1426, "fname": "Marcie", "lname": "Rasnake", "age": 42, "dept": "Sales" }
+, { "id": 1427, "fname": "Hugh", "lname": "Marshburn", "age": 43, "dept": "Payroll" }
+, { "id": 1428, "fname": "Mathew", "lname": "Marasco", "age": 45, "dept": "Sales" }
+, { "id": 1429, "fname": "Kurt", "lname": "Veres", "age": 32, "dept": "IT" }
+, { "id": 1430, "fname": "Julio", "lname": "Barkett", "age": 39, "dept": "Sales" }
+, { "id": 1863, "fname": "Darren", "lname": "Thorington", "age": 32, "dept": "Sales" }
+, { "id": 1999, "fname": "Susan", "lname": "Malaika", "age": 42, "dept": "HR" }
+, { "id": 2333, "fname": "Chen", "lname": "Li", "age": 42, "dept": "HR" }
+, { "id": 2963, "fname": "Neil", "lname": "Gunnerson", "age": 34, "dept": "IT" }
+, { "id": 3563, "fname": "Hazeltine", "lname": "Susan", "age": 29, "dept": "Sales" }
+, { "id": 3666, "fname": "Young Seok", "lname": "Kim", "age": 35, "dept": "Payroll" }
+, { "id": 4727, "fname": "Michael", "lname": "Carey", "age": 50, "dept": "Payroll" }
+, { "id": 5438, "fname": "Lakisha", "lname": "Quashie", "age": 29, "dept": "HR" }
+, { "id": 7444, "fname": "Sharad", "lname": "Mehrotra", "age": 42, "dept": "Sales" }
+, { "id": 7663, "fname": "Annabelle", "lname": "Nimmo", "age": 30, "dept": "Payroll" }
+, { "id": 8301, "fname": "Earlene", "lname": "Wallick", "age": 26, "dept": "HR" }
+, { "id": 8338, "fname": "Julio", "lname": "Bosket", "age": 28, "dept": "Payroll" }
+, { "id": 9555, "fname": "Tony", "lname": "Givargis", "age": 40, "dept": "Sales" }
+, { "id": 9763, "fname": "Ted", "lname": "Saini", "age": 31, "dept": "IT" }
+, { "id": 9941, "fname": "Khurram Faraaz", "lname": "Mohammed", "age": 30, "dept": "HR" }
+, { "id": 10101, "fname": "Javier", "lname": "Makuch", "age": 28, "dept": "IT" }
+, { "id": 10110, "fname": "Allan", "lname": "Piland", "age": 29, "dept": "HR" }
+, { "id": 10112, "fname": "Pearlie", "lname": "Aumann", "age": 31, "dept": "Payroll" }
+, { "id": 10113, "fname": "Chandra", "lname": "Hase", "age": 34, "dept": "Sales" }
+, { "id": 10114, "fname": "Christian", "lname": "Convery", "age": 28, "dept": "HR" }
+, { "id": 10115, "fname": "Panther", "lname": "Ritch", "age": 26, "dept": "IT" }
+, { "id": 10116, "fname": "Ted", "lname": "Elsea", "age": 26, "dept": "IT" }
+, { "id": 10117, "fname": "Tabatha", "lname": "Bladen", "age": 25, "dept": "HR" }
+, { "id": 10118, "fname": "Clayton", "lname": "Oltman", "age": 42, "dept": "Sales" }
+, { "id": 10119, "fname": "Sharron", "lname": "Darwin", "age": 32, "dept": "Payroll" }
+, { "id": 10210, "fname": "Clayton", "lname": "Durgin", "age": 52, "dept": "HR" }
+, { "id": 10212, "fname": "Emilia", "lname": "Chenail", "age": 26, "dept": "Sales" }
+, { "id": 10213, "fname": "Kenya", "lname": "Almquist", "age": 43, "dept": "Payroll" }
+, { "id": 10214, "fname": "Alejandra", "lname": "Lacefield", "age": 41, "dept": "HR" }
+, { "id": 10215, "fname": "Karina", "lname": "Michelsen", "age": 46, "dept": "IT" }
+, { "id": 10216, "fname": "Katy", "lname": "Delillo", "age": 36, "dept": "IT" }
+, { "id": 10217, "fname": "Benita", "lname": "Kleist", "age": 37, "dept": "HR" }
+, { "id": 10218, "fname": "Earlene", "lname": "Paluch", "age": 31, "dept": "IT" }
+, { "id": 10219, "fname": "Kurt", "lname": "Petermann", "age": 27, "dept": "Payroll" }
+, { "id": 10299, "fname": "Julio", "lname": "Iorio", "age": 37, "dept": "IT" }
+, { "id": 10363, "fname": "Cody", "lname": "Rodreguez", "age": 26, "dept": "IT" }
+, { "id": 10404, "fname": "Emilia", "lname": "Square", "age": 32, "dept": "IT" }
+, { "id": 10414, "fname": "Mathew", "lname": "Fuschetto", "age": 34, "dept": "HR" }
+, { "id": 10424, "fname": "Allyson", "lname": "Remus", "age": 32, "dept": "IT" }
+, { "id": 10434, "fname": "Earlene", "lname": "Linebarger", "age": 26, "dept": "Payroll" }
+, { "id": 10444, "fname": "Clinton", "lname": "Sick", "age": 29, "dept": "IT" }
+, { "id": 10454, "fname": "Ted", "lname": "Caba", "age": 28, "dept": "HR" }
+, { "id": 10463, "fname": "Marcie", "lname": "States", "age": 28, "dept": "IT" }
+, { "id": 10464, "fname": "Fernando", "lname": "Engelke", "age": 39, "dept": "IT" }
+, { "id": 10474, "fname": "Mathew", "lname": "Courchesne", "age": 31, "dept": "IT" }
+, { "id": 10484, "fname": "Cody", "lname": "Vinyard", "age": 36, "dept": "Payroll" }
+, { "id": 10494, "fname": "Benita", "lname": "Fravel", "age": 33, "dept": "Sales" }
+, { "id": 10504, "fname": "Erik", "lname": "Dobek", "age": 29, "dept": "IT" }
+, { "id": 10514, "fname": "Julio", "lname": "Ruben", "age": 41, "dept": "IT" }
+, { "id": 10524, "fname": "Benita", "lname": "Maltos", "age": 33, "dept": "IT" }
+, { "id": 10534, "fname": "Kurt", "lname": "Biscoe", "age": 36, "dept": "HR" }
+, { "id": 10538, "fname": "Milagros", "lname": "Forkey", "age": 34, "dept": "Sales" }
+, { "id": 10544, "fname": "Loraine", "lname": "Housel", "age": 30, "dept": "Sales" }
+, { "id": 10554, "fname": "Jamie", "lname": "Rachal", "age": 30, "dept": "IT" }
+, { "id": 10564, "fname": "Liza", "lname": "Fredenburg", "age": 37, "dept": "IT" }
+, { "id": 10574, "fname": "Ericka", "lname": "Feldmann", "age": 29, "dept": "Sales" }
+, { "id": 10584, "fname": "Dollie", "lname": "Dattilo", "age": 32, "dept": "Payroll" }
+, { "id": 10589, "fname": "Lorrie", "lname": "Sharon", "age": 27, "dept": "IT" }
+, { "id": 10594, "fname": "Roxie", "lname": "Houghtaling", "age": 40, "dept": "Payroll" }
+, { "id": 10601, "fname": "Neil", "lname": "Deforge", "age": 26, "dept": "HR" }
+, { "id": 10611, "fname": "Earlene", "lname": "Marcy", "age": 32, "dept": "IT" }
+, { "id": 10621, "fname": "Erik", "lname": "Lechuga", "age": 42, "dept": "Payroll" }
+, { "id": 10631, "fname": "Tyrone", "lname": "Holtzclaw", "age": 34, "dept": "Sales" }
+, { "id": 10641, "fname": "Lance", "lname": "Hankey", "age": 35, "dept": "Sales" }
+, { "id": 10651, "fname": "Mallory", "lname": "Gladding", "age": 31, "dept": "HR" }
+, { "id": 10661, "fname": "Tia", "lname": "Braaten", "age": 40, "dept": "IT" }
+, { "id": 10671, "fname": "Julio", "lname": "Vanpatten", "age": 30, "dept": "Payroll" }
+, { "id": 10681, "fname": "Max", "lname": "Teachout", "age": 34, "dept": "IT" }
+, { "id": 10691, "fname": "Karina", "lname": "Wingerter", "age": 31, "dept": "IT" }
+, { "id": 10711, "fname": "Hugh", "lname": "Lema", "age": 25, "dept": "HR" }
+, { "id": 10721, "fname": "Schwan", "lname": "Phil", "age": 34, "dept": "Payroll" }
+, { "id": 10732, "fname": "Noemi", "lname": "Eacret", "age": 56, "dept": "HR" }
+, { "id": 10741, "fname": "Julio", "lname": "Mattocks", "age": 38, "dept": "Sales" }
+, { "id": 10751, "fname": "Lance", "lname": "Kottke", "age": 34, "dept": "IT" }
+, { "id": 10761, "fname": "Kurt", "lname": "Liz", "age": 32, "dept": "HR" }
+, { "id": 10771, "fname": "Neva", "lname": "Barbeau", "age": 45, "dept": "Sales" }
+, { "id": 10781, "fname": "Karina", "lname": "Tuthill", "age": 46, "dept": "Payroll" }
+, { "id": 10791, "fname": "Maricela", "lname": "Cambron", "age": 36, "dept": "IT" }
+, { "id": 10809, "fname": "Clayton", "lname": "Delany", "age": 23, "dept": "IT" }
+, { "id": 10811, "fname": "Kubik", "lname": "Kuhn", "age": 27, "dept": "HR" }
+, { "id": 10821, "fname": "Allan", "lname": "Tomes", "age": 29, "dept": "Payroll" }
+, { "id": 10831, "fname": "Lonnie", "lname": "Aller", "age": 33, "dept": "Sales" }
+, { "id": 10841, "fname": "Neil", "lname": "Hurrell", "age": 26, "dept": "IT" }
+, { "id": 10851, "fname": "Clayton", "lname": "Engles", "age": 41, "dept": "HR" }
+, { "id": 10861, "fname": "Javier", "lname": "Gabrielson", "age": 39, "dept": "Payroll" }
+, { "id": 10871, "fname": "Allan", "lname": "Alejandre", "age": 48, "dept": "IT" }
+, { "id": 10881, "fname": "Julio", "lname": "Isa", "age": 38, "dept": "Sales" }
+, { "id": 10891, "fname": "Roslyn", "lname": "Simmerman", "age": 31, "dept": "IT" }
+, { "id": 10915, "fname": "Starner", "lname": "Stuart", "age": 25, "dept": "Sales" }
+, { "id": 10925, "fname": "Sofia", "lname": "Cuff", "age": 30, "dept": "HR" }
+, { "id": 10935, "fname": "Milagros", "lname": "Murguia", "age": 31, "dept": "IT" }
+, { "id": 10945, "fname": "Margery", "lname": "Haldeman", "age": 32, "dept": "IT" }
+, { "id": 10955, "fname": "Max", "lname": "Mell", "age": 33, "dept": "HR" }
+, { "id": 10965, "fname": "Micco", "lname": "Mercy", "age": 31, "dept": "Payroll" }
+, { "id": 10975, "fname": "Clare", "lname": "Vangieson", "age": 34, "dept": "IT" }
+, { "id": 10985, "fname": "Elnora", "lname": "Dimauro", "age": 35, "dept": "Sales" }
+, { "id": 10995, "fname": "Pearlie", "lname": "Kocian", "age": 38, "dept": "HR" }
+, { "id": 11007, "fname": "Yingyi", "lname": "Bu", "age": 27, "dept": "IT" }
+, { "id": 11263, "fname": "Tania", "lname": "Loffredo", "age": 25, "dept": "IT" }
+, { "id": 11410, "fname": "Clinton", "lname": "Fredricks", "age": 34, "dept": "IT" }
+, { "id": 11411, "fname": "Lance", "lname": "Farquhar", "age": 32, "dept": "HR" }
+, { "id": 11412, "fname": "Tabatha", "lname": "Crisler", "age": 33, "dept": "IT" }
+, { "id": 11413, "fname": "Max", "lname": "Durney", "age": 29, "dept": "IT" }
+, { "id": 11414, "fname": "Carmella", "lname": "Strauser", "age": 30, "dept": "Payroll" }
+, { "id": 11415, "fname": "Kelly", "lname": "Carrales", "age": 40, "dept": "IT" }
+, { "id": 11416, "fname": "Guy", "lname": "Merten", "age": 29, "dept": "Sales" }
+, { "id": 11417, "fname": "Noreen", "lname": "Ruhland", "age": 29, "dept": "IT" }
+, { "id": 11418, "fname": "Julio", "lname": "Damore", "age": 27, "dept": "Sales" }
+, { "id": 11419, "fname": "Selena", "lname": "Truby", "age": 25, "dept": "HR" }
+, { "id": 11420, "fname": "Alejandra", "lname": "Commons", "age": 30, "dept": "Sales" }
+, { "id": 11421, "fname": "Allyson", "lname": "Balk", "age": 30, "dept": "IT" }
+, { "id": 11422, "fname": "Nelson", "lname": "Byun", "age": 40, "dept": "Sales" }
+, { "id": 11423, "fname": "Christian", "lname": "Reidhead", "age": 40, "dept": "IT" }
+, { "id": 11424, "fname": "Pearlie", "lname": "Hopkin", "age": 48, "dept": "Payroll" }
+, { "id": 11425, "fname": "Nelson", "lname": "Wohlers", "age": 41, "dept": "HR" }
+, { "id": 11426, "fname": "Marcie", "lname": "Rasnake", "age": 42, "dept": "Sales" }
+, { "id": 11427, "fname": "Hugh", "lname": "Marshburn", "age": 43, "dept": "Payroll" }
+, { "id": 11428, "fname": "Mathew", "lname": "Marasco", "age": 45, "dept": "Sales" }
+, { "id": 11429, "fname": "Kurt", "lname": "Veres", "age": 32, "dept": "IT" }
+, { "id": 11430, "fname": "Julio", "lname": "Barkett", "age": 39, "dept": "Sales" }
+, { "id": 11863, "fname": "Darren", "lname": "Thorington", "age": 32, "dept": "Sales" }
+, { "id": 11999, "fname": "Susan", "lname": "Malaika", "age": 42, "dept": "HR" }
+, { "id": 12333, "fname": "Chen", "lname": "Li", "age": 42, "dept": "HR" }
+, { "id": 12963, "fname": "Neil", "lname": "Gunnerson", "age": 34, "dept": "IT" }
+, { "id": 13563, "fname": "Hazeltine", "lname": "Susan", "age": 29, "dept": "Sales" }
+, { "id": 13666, "fname": "Young Seok", "lname": "Kim", "age": 35, "dept": "Payroll" }
+, { "id": 14727, "fname": "Michael", "lname": "Carey", "age": 50, "dept": "Payroll" }
+, { "id": 15438, "fname": "Lakisha", "lname": "Quashie", "age": 29, "dept": "HR" }
+, { "id": 17444, "fname": "Sharad", "lname": "Mehrotra", "age": 42, "dept": "Sales" }
+, { "id": 17663, "fname": "Annabelle", "lname": "Nimmo", "age": 30, "dept": "Payroll" }
+, { "id": 18301, "fname": "Earlene", "lname": "Wallick", "age": 26, "dept": "HR" }
+, { "id": 18338, "fname": "Julio", "lname": "Bosket", "age": 28, "dept": "Payroll" }
+, { "id": 19555, "fname": "Tony", "lname": "Givargis", "age": 40, "dept": "Sales" }
+, { "id": 19763, "fname": "Ted", "lname": "Saini", "age": 31, "dept": "IT" }
+, { "id": 19941, "fname": "Khurram Faraaz", "lname": "Mohammed", "age": 30, "dept": "HR" }
+ ]
diff --git a/asterix-app/src/test/resources/runtimets/results/temp-dataset/insert-and-scan-dataset/insert-and-scan-dataset.1.adm b/asterix-app/src/test/resources/runtimets/results/temp-dataset/insert-and-scan-dataset/insert-and-scan-dataset.1.adm
new file mode 100644
index 0000000..185e642
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/temp-dataset/insert-and-scan-dataset/insert-and-scan-dataset.1.adm
@@ -0,0 +1,21 @@
+[ { "id": 1 }
+, { "id": 2 }
+, { "id": 3 }
+, { "id": 4 }
+, { "id": 5 }
+, { "id": 6 }
+, { "id": 7 }
+, { "id": 8 }
+, { "id": 9 }
+, { "id": 10 }
+, { "id": 11 }
+, { "id": 12 }
+, { "id": 13 }
+, { "id": 14 }
+, { "id": 15 }
+, { "id": 16 }
+, { "id": 17 }
+, { "id": 18 }
+, { "id": 19 }
+, { "id": 20 }
+ ]
diff --git a/asterix-app/src/test/resources/runtimets/results/temp-dataset/q01_pricing_summary_report_nt/q01_pricing_summary_report_nt.1.adm b/asterix-app/src/test/resources/runtimets/results/temp-dataset/q01_pricing_summary_report_nt/q01_pricing_summary_report_nt.1.adm
new file mode 100644
index 0000000..774e16e
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/temp-dataset/q01_pricing_summary_report_nt/q01_pricing_summary_report_nt.1.adm
@@ -0,0 +1,5 @@
+[ { "l_returnflag": "A", "l_linestatus": "F", "sum_qty": 37474.0d, "sum_base_price": 3.7569624640000015E7d, "sum_disc_price": 3.567619209699997E7d, "sum_charge": 3.7101416222424E7d, "ave_qty": 25.354533152909337d, "ave_price": 25419.231826792973d, "ave_disc": 0.05086603518267936d, "count_order": 1478 }
+, { "l_returnflag": "N", "l_linestatus": "F", "sum_qty": 1041.0d, "sum_base_price": 1041301.0700000001d, "sum_disc_price": 999060.898d, "sum_charge": 1036450.8022800002d, "ave_qty": 27.394736842105264d, "ave_price": 27402.659736842106d, "ave_disc": 0.04289473684210526d, "count_order": 38 }
+, { "l_returnflag": "N", "l_linestatus": "O", "sum_qty": 75168.0d, "sum_base_price": 7.538495537000003E7d, "sum_disc_price": 7.165316630340004E7d, "sum_charge": 7.449879813307303E7d, "ave_qty": 25.558653519211152d, "ave_price": 25632.422771166282d, "ave_disc": 0.04969738184291074d, "count_order": 2941 }
+, { "l_returnflag": "R", "l_linestatus": "F", "sum_qty": 36511.0d, "sum_base_price": 3.657084124000002E7d, "sum_disc_price": 3.473847287579997E7d, "sum_charge": 3.616906011219299E7d, "ave_qty": 25.059025394646532d, "ave_price": 25100.09693891559d, "ave_disc": 0.05002745367192867d, "count_order": 1457 }
+ ]
diff --git a/asterix-app/src/test/resources/runtimets/results/temp-dataset/scan-delete-rtree-secondary-index/scan-delete-rtree-secondary-index.1.adm b/asterix-app/src/test/resources/runtimets/results/temp-dataset/scan-delete-rtree-secondary-index/scan-delete-rtree-secondary-index.1.adm
new file mode 100644
index 0000000..5b54383
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/temp-dataset/scan-delete-rtree-secondary-index/scan-delete-rtree-secondary-index.1.adm
@@ -0,0 +1,2 @@
+[ { "id": 10 }
+ ]
diff --git a/asterix-app/src/test/resources/runtimets/results/temp-dataset/scan-insert-persistent-to-temp/scan-insert-persistent-to-temp.1.adm b/asterix-app/src/test/resources/runtimets/results/temp-dataset/scan-insert-persistent-to-temp/scan-insert-persistent-to-temp.1.adm
new file mode 100644
index 0000000..6fa2dc2
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/temp-dataset/scan-insert-persistent-to-temp/scan-insert-persistent-to-temp.1.adm
@@ -0,0 +1,4 @@
+[ { "id": 10 }
+, { "id": 12 }
+, { "id": 20 }
+ ]
diff --git a/asterix-app/src/test/resources/runtimets/results/temp-dataset/scan-insert-rtree-secondary-index/scan-insert-rtree-secondary-index.1.adm b/asterix-app/src/test/resources/runtimets/results/temp-dataset/scan-insert-rtree-secondary-index/scan-insert-rtree-secondary-index.1.adm
new file mode 100644
index 0000000..6fa2dc2
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/temp-dataset/scan-insert-rtree-secondary-index/scan-insert-rtree-secondary-index.1.adm
@@ -0,0 +1,4 @@
+[ { "id": 10 }
+, { "id": 12 }
+, { "id": 20 }
+ ]
diff --git a/asterix-app/src/test/resources/runtimets/results/temp-dataset/scan-insert-temp-to-persistent/scan-insert-persistent-to-temp.1.adm b/asterix-app/src/test/resources/runtimets/results/temp-dataset/scan-insert-temp-to-persistent/scan-insert-persistent-to-temp.1.adm
new file mode 100644
index 0000000..6fa2dc2
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/temp-dataset/scan-insert-temp-to-persistent/scan-insert-persistent-to-temp.1.adm
@@ -0,0 +1,4 @@
+[ { "id": 10 }
+, { "id": 12 }
+, { "id": 20 }
+ ]
diff --git a/asterix-app/src/test/resources/runtimets/testsuite.xml b/asterix-app/src/test/resources/runtimets/testsuite.xml
index f97f793..a954f7e 100644
--- a/asterix-app/src/test/resources/runtimets/testsuite.xml
+++ b/asterix-app/src/test/resources/runtimets/testsuite.xml
@@ -6691,4 +6691,66 @@
</compilation-unit>
</test-case>
</test-group>
+ <test-group name="temp-dataset">
+ <test-case FilePath="temp-dataset">
+ <compilation-unit name="delete-from-loaded-dataset">
+ <output-dir compare="Text">delete-from-loaded-dataset</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="temp-dataset">
+ <compilation-unit name="delete-from-loaded-dataset-with-index">
+ <output-dir compare="Text">delete-from-loaded-dataset-with-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="temp-dataset">
+ <compilation-unit name="drop-empty-secondary-indexes">
+ <output-dir compare="Text">drop-empty-secondary-indexes</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="temp-dataset">
+ <compilation-unit name="drop-index">
+ <output-dir compare="Text">drop-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="temp-dataset">
+ <compilation-unit name="empty-load-with-index">
+ <output-dir compare="Text">empty-load-with-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="temp-dataset">
+ <compilation-unit name="insert-and-scan-dataset">
+ <output-dir compare="Text">insert-and-scan-dataset</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="temp-dataset">
+ <compilation-unit name="q01_pricing_summary_report_nt">
+ <output-dir compare="Text">q01_pricing_summary_report_nt</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="temp-dataset">
+ <compilation-unit name="scan-delete-rtree-secondary-index">
+ <output-dir compare="Text">scan-delete-rtree-secondary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="temp-dataset">
+ <compilation-unit name="scan-insert-rtree-secondary-index">
+ <output-dir compare="Text">scan-insert-rtree-secondary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="temp-dataset">
+ <compilation-unit name="scan-insert-persistent-to-temp">
+ <output-dir compare="Text">scan-insert-persistent-to-temp</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="temp-dataset">
+ <compilation-unit name="scan-insert-temp-to-persistent">
+ <output-dir compare="Text">scan-insert-temp-to-persistent</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="temp-dataset">
+ <compilation-unit name="insert-and-scan-dataset-with-index">
+ <output-dir compare="Text">insert-and-scan-dataset-with-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
</test-suite>
diff --git a/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/ExternalDetailsDecl.java b/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/ExternalDetailsDecl.java
index a22d4b4..dca9291 100644
--- a/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/ExternalDetailsDecl.java
+++ b/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/ExternalDetailsDecl.java
@@ -62,6 +62,11 @@
return compactionPolicyProperties;
}
+ @Override
+ public boolean isTemp() {
+ return false;
+ }
+
public void setCompactionPolicyProperties(Map<String, String> compactionPolicyProperties) {
this.compactionPolicyProperties = compactionPolicyProperties;
}
diff --git a/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/FeedDetailsDecl.java b/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/FeedDetailsDecl.java
index 4a3661b..e8eca29 100644
--- a/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/FeedDetailsDecl.java
+++ b/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/FeedDetailsDecl.java
@@ -27,7 +27,7 @@
public FeedDetailsDecl(String adapterFactoryClassname, Map<String, String> configuration,
FunctionSignature signature, Identifier nodeGroupName, List<List<String>> partitioningExpr,
String compactionPolicy, Map<String, String> compactionPolicyProperties, List<String> filterField) {
- super(nodeGroupName, partitioningExpr, false, compactionPolicy, compactionPolicyProperties, filterField);
+ super(nodeGroupName, partitioningExpr, false, compactionPolicy, compactionPolicyProperties, filterField, false);
this.adapterFactoryClassname = adapterFactoryClassname;
this.configuration = configuration;
this.functionSignature = signature;
diff --git a/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/IDatasetDetailsDecl.java b/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/IDatasetDetailsDecl.java
index 51f2a1b..e89ef0d 100644
--- a/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/IDatasetDetailsDecl.java
+++ b/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/IDatasetDetailsDecl.java
@@ -24,4 +24,6 @@
public Map<String, String> getCompactionPolicyProperties();
+ public boolean isTemp();
+
}
diff --git a/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/InternalDetailsDecl.java b/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/InternalDetailsDecl.java
index 8fe482e..92618fa 100644
--- a/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/InternalDetailsDecl.java
+++ b/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/InternalDetailsDecl.java
@@ -25,10 +25,12 @@
private final boolean autogenerated;
private final String compactionPolicy;
private final Map<String, String> compactionPolicyProperties;
+ private final boolean temp;
private final List<String> filterField;
public InternalDetailsDecl(Identifier nodeGroupName, List<List<String>> partitioningExpr, boolean autogenerated,
- String compactionPolicy, Map<String, String> compactionPolicyProperties, List<String> filterField) {
+ String compactionPolicy, Map<String, String> compactionPolicyProperties, List<String> filterField,
+ boolean temp) {
this.nodegroupName = nodeGroupName == null ? new Identifier(MetadataConstants.METADATA_DEFAULT_NODEGROUP_NAME)
: nodeGroupName;
this.partitioningExprs = partitioningExpr;
@@ -36,6 +38,7 @@
this.compactionPolicy = compactionPolicy;
this.compactionPolicyProperties = compactionPolicyProperties;
this.filterField = filterField;
+ this.temp = temp;
}
public List<List<String>> getPartitioningExprs() {
@@ -61,6 +64,11 @@
return compactionPolicyProperties;
}
+ @Override
+ public boolean isTemp() {
+ return temp;
+ }
+
public List<String> getFilterField() {
return filterField;
}
diff --git a/asterix-aql/src/main/javacc/AQL.html b/asterix-aql/src/main/javacc/AQL.html
index 994b1c7..b423f6d 100644
--- a/asterix-aql/src/main/javacc/AQL.html
+++ b/asterix-aql/src/main/javacc/AQL.html
@@ -38,6 +38,7 @@
| <UNION: "union">
| <WHERE: "where">
| <WITH: "with">
+| <KEEPING: "keeping">
}
</PRE>
@@ -316,132 +317,137 @@
<TR>
<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod2">SingleStatement</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>( <A HREF="#prod3">DataverseDeclaration</A> | <A HREF="#prod4">FunctionDeclaration</A> | <A HREF="#prod5">CreateStatement</A> | <A HREF="#prod6">LoadStatement</A> | <A HREF="#prod7">DropStatement</A> | <A HREF="#prod8">WriteStatement</A> | <A HREF="#prod9">SetStatement</A> | <A HREF="#prod10">InsertStatement</A> | <A HREF="#prod11">DeleteStatement</A> | <A HREF="#prod12">UpdateStatement</A> | <A HREF="#prod13">FeedStatement</A> | <A HREF="#prod14">CompactStatement</A> | <A HREF="#prod15">Query</A> | <A HREF="#prod16">RefreshExternalDatasetStatement</A> )</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( <A HREF="#prod3">DataverseDeclaration</A> | <A HREF="#prod4">FunctionDeclaration</A> | <A HREF="#prod5">CreateStatement</A> | <A HREF="#prod6">LoadStatement</A> | <A HREF="#prod7">DropStatement</A> | <A HREF="#prod8">WriteStatement</A> | <A HREF="#prod9">SetStatement</A> | <A HREF="#prod10">InsertStatement</A> | <A HREF="#prod11">DeleteStatement</A> | <A HREF="#prod12">UpdateStatement</A> | <A HREF="#prod13">FeedStatement</A> | <A HREF="#prod14">CompactStatement</A> | <A HREF="#prod15">Query</A> | <A HREF="#prod16">RefreshExternalDatasetStatement</A> | <A HREF="#prod17">RunStatement</A> )</TD>
</TR>
<TR>
<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod3">DataverseDeclaration</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>"use" "dataverse" <A HREF="#prod17">Identifier</A></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"use" "dataverse" <A HREF="#prod18">Identifier</A></TD>
</TR>
<TR>
<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod5">CreateStatement</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>"create" ( <A HREF="#prod18">TypeSpecification</A> | <A HREF="#prod19">NodegroupSpecification</A> | <A HREF="#prod20">DatasetSpecification</A> | <A HREF="#prod21">IndexSpecification</A> | <A HREF="#prod22">DataverseSpecification</A> | <A HREF="#prod23">FunctionSpecification</A> | <A HREF="#prod24">FeedSpecification</A> )</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"create" ( <A HREF="#prod19">TypeSpecification</A> | <A HREF="#prod20">NodegroupSpecification</A> | <A HREF="#prod21">DatasetSpecification</A> | <A HREF="#prod22">IndexSpecification</A> | <A HREF="#prod23">DataverseSpecification</A> | <A HREF="#prod24">FunctionSpecification</A> | <A HREF="#prod25">FeedSpecification</A> )</TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod18">TypeSpecification</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod19">TypeSpecification</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>"type" <A HREF="#prod25">TypeName</A> <A HREF="#prod26">IfNotExists</A> "as" <A HREF="#prod27">TypeExpr</A></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"type" <A HREF="#prod26">TypeName</A> <A HREF="#prod27">IfNotExists</A> "as" <A HREF="#prod28">TypeExpr</A></TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod19">NodegroupSpecification</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod20">NodegroupSpecification</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>"nodegroup" <A HREF="#prod17">Identifier</A> <A HREF="#prod26">IfNotExists</A> "on" <A HREF="#prod17">Identifier</A> ( <COMMA> <A HREF="#prod17">Identifier</A> )*</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"nodegroup" <A HREF="#prod18">Identifier</A> <A HREF="#prod27">IfNotExists</A> "on" <A HREF="#prod18">Identifier</A> ( <COMMA> <A HREF="#prod18">Identifier</A> )*</TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod20">DatasetSpecification</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod21">DatasetSpecification</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>( "external" <DATASET> <A HREF="#prod28">QualifiedName</A> <LEFTPAREN> <A HREF="#prod17">Identifier</A> <RIGHTPAREN> <A HREF="#prod26">IfNotExists</A> "using" <A HREF="#prod29">AdapterName</A> <A HREF="#prod30">Configuration</A> ( "on" <A HREF="#prod17">Identifier</A> )? ( "hints" <A HREF="#prod31">Properties</A> )? ( "using" "compaction" "policy" <A HREF="#prod32">CompactionPolicy</A> ( <A HREF="#prod30">Configuration</A> )? )? | ( "internal" )? <DATASET> <A HREF="#prod28">QualifiedName</A> <LEFTPAREN> <A HREF="#prod17">Identifier</A> <RIGHTPAREN> <A HREF="#prod26">IfNotExists</A> <A HREF="#prod33">PrimaryKey</A> ( "autogenerated" )? ( "on" <A HREF="#prod17">Identifier</A> )? ( "hints" <A HREF="#prod31">Properties</A> )? ( "using" "compaction" "policy" <A HREF="#prod32">CompactionPolicy</A> ( <A HREF="#prod30">Configuration</A> )? )? ( "with filter on" <A HREF="#prod34">FilterField</A> )? )</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( "external" <DATASET> <A HREF="#prod29">QualifiedName</A> <LEFTPAREN> <A HREF="#prod18">Identifier</A> <RIGHTPAREN> <A HREF="#prod27">IfNotExists</A> "using" <A HREF="#prod30">AdapterName</A> <A HREF="#prod31">Configuration</A> ( "on" <A HREF="#prod18">Identifier</A> )? ( "hints" <A HREF="#prod32">Properties</A> )? ( "using" "compaction" "policy" <A HREF="#prod33">CompactionPolicy</A> ( <A HREF="#prod31">Configuration</A> )? )? | ( "internal" | "temporary" )? <DATASET> <A HREF="#prod29">QualifiedName</A> <LEFTPAREN> <A HREF="#prod18">Identifier</A> <RIGHTPAREN> <A HREF="#prod27">IfNotExists</A> <A HREF="#prod34">PrimaryKey</A> ( "autogenerated" )? ( "on" <A HREF="#prod18">Identifier</A> )? ( "hints" <A HREF="#prod32">Properties</A> )? ( "using" "compaction" "policy" <A HREF="#prod33">CompactionPolicy</A> ( <A HREF="#prod31">Configuration</A> )? )? ( "with filter on" <A HREF="#prod35">FilterField</A> )? )</TD>
</TR>
<TR>
<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod16">RefreshExternalDatasetStatement</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>"refresh external" <DATASET> <A HREF="#prod28">QualifiedName</A></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"refresh external" <DATASET> <A HREF="#prod29">QualifiedName</A></TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod21">IndexSpecification</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod17">RunStatement</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>"index" <A HREF="#prod17">Identifier</A> <A HREF="#prod26">IfNotExists</A> "on" <A HREF="#prod28">QualifiedName</A> <LEFTPAREN> ( <A HREF="#prod17">Identifier</A> ) ( <COMMA> <A HREF="#prod17">Identifier</A> )* <RIGHTPAREN> ( "type" <A HREF="#prod35">IndexType</A> )?</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"run" <A HREF="#prod18">Identifier</A> <LEFTPAREN> ( <A HREF="#prod18">Identifier</A> ( <COMMA> )? )* <RIGHTPAREN> <FROM> <DATASET> <A HREF="#prod29">QualifiedName</A> "to" <DATASET> <A HREF="#prod29">QualifiedName</A></TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod32">CompactionPolicy</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod22">IndexSpecification</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod17">Identifier</A></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"index" <A HREF="#prod18">Identifier</A> <A HREF="#prod27">IfNotExists</A> "on" <A HREF="#prod29">QualifiedName</A> <LEFTPAREN> ( <A HREF="#prod18">Identifier</A> ) ( <COMMA> <A HREF="#prod18">Identifier</A> )* <RIGHTPAREN> ( "type" <A HREF="#prod36">IndexType</A> )?</TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod34">FilterField</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod33">CompactionPolicy</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod17">Identifier</A></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod18">Identifier</A></TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod35">IndexType</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod35">FilterField</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod18">Identifier</A></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod36">IndexType</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
<TD ALIGN=LEFT VALIGN=BASELINE>( "btree" | "rtree" | "keyword" | "ngram" <LEFTPAREN> <INTEGER_LITERAL> <RIGHTPAREN> )</TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod22">DataverseSpecification</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod23">DataverseSpecification</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>"dataverse" <A HREF="#prod17">Identifier</A> <A HREF="#prod26">IfNotExists</A> ( "with format" <A HREF="#prod36">StringLiteral</A> )?</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"dataverse" <A HREF="#prod18">Identifier</A> <A HREF="#prod27">IfNotExists</A> ( "with format" <A HREF="#prod37">StringLiteral</A> )?</TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod23">FunctionSpecification</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod24">FunctionSpecification</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>"function" <A HREF="#prod37">FunctionName</A> <A HREF="#prod26">IfNotExists</A> <A HREF="#prod38">ParameterList</A> <LEFTBRACE> <A HREF="#prod39">Expression</A> <RIGHTBRACE></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"function" <A HREF="#prod38">FunctionName</A> <A HREF="#prod27">IfNotExists</A> <A HREF="#prod39">ParameterList</A> <LEFTBRACE> <A HREF="#prod40">Expression</A> <RIGHTBRACE></TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod24">FeedSpecification</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod25">FeedSpecification</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>( "feed" <A HREF="#prod28">QualifiedName</A> <A HREF="#prod26">IfNotExists</A> "using" <A HREF="#prod29">AdapterName</A> <A HREF="#prod30">Configuration</A> ( <A HREF="#prod40">ApplyFunction</A> )? )</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( "feed" <A HREF="#prod29">QualifiedName</A> <A HREF="#prod27">IfNotExists</A> "using" <A HREF="#prod30">AdapterName</A> <A HREF="#prod31">Configuration</A> ( <A HREF="#prod41">ApplyFunction</A> )? )</TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod38">ParameterList</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod39">ParameterList</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
<TD ALIGN=LEFT VALIGN=BASELINE><LEFTPAREN> ( <VARIABLE> ( <COMMA> <VARIABLE> )* )? <RIGHTPAREN></TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod26">IfNotExists</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod27">IfNotExists</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
<TD ALIGN=LEFT VALIGN=BASELINE>( "if not exists" )?</TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod40">ApplyFunction</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod41">ApplyFunction</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>"apply" "function" <A HREF="#prod37">FunctionName</A></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"apply" "function" <A HREF="#prod38">FunctionName</A></TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod41">GetPolicy</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod42">GetPolicy</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>"using" "policy" <A HREF="#prod17">Identifier</A></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"using" "policy" <A HREF="#prod18">Identifier</A></TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod42">FunctionSignature</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod43">FunctionSignature</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod37">FunctionName</A> "@" <INTEGER_LITERAL></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod38">FunctionName</A> "@" <INTEGER_LITERAL></TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod33">PrimaryKey</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod34">PrimaryKey</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>"primary" "key" <A HREF="#prod17">Identifier</A> ( <COMMA> <A HREF="#prod17">Identifier</A> )*</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"primary" "key" <A HREF="#prod18">Identifier</A> ( <COMMA> <A HREF="#prod18">Identifier</A> )*</TD>
</TR>
<TR>
<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod7">DropStatement</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>"drop" ( <DATASET> <A HREF="#prod28">QualifiedName</A> <A HREF="#prod43">IfExists</A> | "index" <A HREF="#prod44">DoubleQualifiedName</A> <A HREF="#prod43">IfExists</A> | "nodegroup" <A HREF="#prod17">Identifier</A> <A HREF="#prod43">IfExists</A> | "type" <A HREF="#prod25">TypeName</A> <A HREF="#prod43">IfExists</A> | "dataverse" <A HREF="#prod17">Identifier</A> <A HREF="#prod43">IfExists</A> | "function" <A HREF="#prod42">FunctionSignature</A> <A HREF="#prod43">IfExists</A> | "feed" <A HREF="#prod28">QualifiedName</A> <A HREF="#prod43">IfExists</A> )</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"drop" ( <DATASET> <A HREF="#prod29">QualifiedName</A> <A HREF="#prod44">IfExists</A> | "index" <A HREF="#prod45">DoubleQualifiedName</A> <A HREF="#prod44">IfExists</A> | "nodegroup" <A HREF="#prod18">Identifier</A> <A HREF="#prod44">IfExists</A> | "type" <A HREF="#prod26">TypeName</A> <A HREF="#prod44">IfExists</A> | "dataverse" <A HREF="#prod18">Identifier</A> <A HREF="#prod44">IfExists</A> | "function" <A HREF="#prod43">FunctionSignature</A> <A HREF="#prod44">IfExists</A> | "feed" <A HREF="#prod29">QualifiedName</A> <A HREF="#prod44">IfExists</A> )</TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod43">IfExists</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod44">IfExists</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
<TD ALIGN=LEFT VALIGN=BASELINE>( <IF> "exists" )?</TD>
</TR>
<TR>
<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod10">InsertStatement</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>"insert" "into" <DATASET> <A HREF="#prod28">QualifiedName</A> <A HREF="#prod15">Query</A></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"insert" "into" <DATASET> <A HREF="#prod29">QualifiedName</A> <A HREF="#prod15">Query</A></TD>
</TR>
<TR>
<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod11">DeleteStatement</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>"delete" <A HREF="#prod45">Variable</A> <FROM> <DATASET> <A HREF="#prod28">QualifiedName</A> ( <WHERE> <A HREF="#prod39">Expression</A> )?</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"delete" <A HREF="#prod46">Variable</A> <FROM> <DATASET> <A HREF="#prod29">QualifiedName</A> ( <WHERE> <A HREF="#prod40">Expression</A> )?</TD>
</TR>
<TR>
<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod12">UpdateStatement</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>"update" <A HREF="#prod45">Variable</A> <IN> <A HREF="#prod39">Expression</A> <WHERE> <A HREF="#prod39">Expression</A> <LEFTPAREN> ( <A HREF="#prod46">UpdateClause</A> ( <COMMA> <A HREF="#prod46">UpdateClause</A> )* ) <RIGHTPAREN></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"update" <A HREF="#prod46">Variable</A> <IN> <A HREF="#prod40">Expression</A> <WHERE> <A HREF="#prod40">Expression</A> <LEFTPAREN> ( <A HREF="#prod47">UpdateClause</A> ( <COMMA> <A HREF="#prod47">UpdateClause</A> )* ) <RIGHTPAREN></TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod46">UpdateClause</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod47">UpdateClause</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>"set" <A HREF="#prod39">Expression</A> <ASSIGN> <A HREF="#prod39">Expression</A></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"set" <A HREF="#prod40">Expression</A> <ASSIGN> <A HREF="#prod40">Expression</A></TD>
</TR>
<TR>
<TD ALIGN=RIGHT VALIGN=BASELINE></TD>
@@ -461,302 +467,302 @@
<TR>
<TD ALIGN=RIGHT VALIGN=BASELINE></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>|</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><IF> <LEFTPAREN> <A HREF="#prod39">Expression</A> <RIGHTPAREN> <THEN> <A HREF="#prod46">UpdateClause</A> ( <ELSE> <A HREF="#prod46">UpdateClause</A> )?</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><IF> <LEFTPAREN> <A HREF="#prod40">Expression</A> <RIGHTPAREN> <THEN> <A HREF="#prod47">UpdateClause</A> ( <ELSE> <A HREF="#prod47">UpdateClause</A> )?</TD>
</TR>
<TR>
<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod9">SetStatement</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>"set" <A HREF="#prod17">Identifier</A> <A HREF="#prod36">StringLiteral</A></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"set" <A HREF="#prod18">Identifier</A> <A HREF="#prod37">StringLiteral</A></TD>
</TR>
<TR>
<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod8">WriteStatement</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>"write" "output" "to" <A HREF="#prod17">Identifier</A> <COLON> <A HREF="#prod36">StringLiteral</A> ( "using" <A HREF="#prod36">StringLiteral</A> )?</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"write" "output" "to" <A HREF="#prod18">Identifier</A> <COLON> <A HREF="#prod37">StringLiteral</A> ( "using" <A HREF="#prod37">StringLiteral</A> )?</TD>
</TR>
<TR>
<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod6">LoadStatement</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>"load" <DATASET> <A HREF="#prod28">QualifiedName</A> "using" <A HREF="#prod29">AdapterName</A> <A HREF="#prod30">Configuration</A> ( "pre-sorted" )?</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"load" <DATASET> <A HREF="#prod29">QualifiedName</A> "using" <A HREF="#prod30">AdapterName</A> <A HREF="#prod31">Configuration</A> ( "pre-sorted" )?</TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod29">AdapterName</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod30">AdapterName</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod17">Identifier</A></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod18">Identifier</A></TD>
</TR>
<TR>
<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod14">CompactStatement</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>"compact" <DATASET> <A HREF="#prod28">QualifiedName</A></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"compact" <DATASET> <A HREF="#prod29">QualifiedName</A></TD>
</TR>
<TR>
<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod13">FeedStatement</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>( "connect" "feed" <A HREF="#prod28">QualifiedName</A> "to" <DATASET> <A HREF="#prod28">QualifiedName</A> ( <A HREF="#prod41">GetPolicy</A> )? | "disconnect" "feed" <A HREF="#prod28">QualifiedName</A> <FROM> <DATASET> <A HREF="#prod28">QualifiedName</A> )</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( "connect" "feed" <A HREF="#prod29">QualifiedName</A> "to" <DATASET> <A HREF="#prod29">QualifiedName</A> ( <A HREF="#prod42">GetPolicy</A> )? | "disconnect" "feed" <A HREF="#prod29">QualifiedName</A> <FROM> <DATASET> <A HREF="#prod29">QualifiedName</A> )</TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod30">Configuration</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod31">Configuration</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><LEFTPAREN> ( <A HREF="#prod47">KeyValuePair</A> ( <COMMA> <A HREF="#prod47">KeyValuePair</A> )* )? <RIGHTPAREN></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><LEFTPAREN> ( <A HREF="#prod48">KeyValuePair</A> ( <COMMA> <A HREF="#prod48">KeyValuePair</A> )* )? <RIGHTPAREN></TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod47">KeyValuePair</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod48">KeyValuePair</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><LEFTPAREN> <A HREF="#prod36">StringLiteral</A> <EQ> <A HREF="#prod36">StringLiteral</A> <RIGHTPAREN></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><LEFTPAREN> <A HREF="#prod37">StringLiteral</A> <EQ> <A HREF="#prod37">StringLiteral</A> <RIGHTPAREN></TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod31">Properties</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod32">Properties</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>( <LEFTPAREN> <A HREF="#prod48">Property</A> ( <COMMA> <A HREF="#prod48">Property</A> )* <RIGHTPAREN> )?</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( <LEFTPAREN> <A HREF="#prod49">Property</A> ( <COMMA> <A HREF="#prod49">Property</A> )* <RIGHTPAREN> )?</TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod48">Property</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod49">Property</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod17">Identifier</A> <EQ> ( <A HREF="#prod36">StringLiteral</A> | <INTEGER_LITERAL> )</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod18">Identifier</A> <EQ> ( <A HREF="#prod37">StringLiteral</A> | <INTEGER_LITERAL> )</TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod27">TypeExpr</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod28">TypeExpr</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>( <A HREF="#prod49">RecordTypeDef</A> | <A HREF="#prod50">TypeReference</A> | <A HREF="#prod51">OrderedListTypeDef</A> | <A HREF="#prod52">UnorderedListTypeDef</A> )</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( <A HREF="#prod50">RecordTypeDef</A> | <A HREF="#prod51">TypeReference</A> | <A HREF="#prod52">OrderedListTypeDef</A> | <A HREF="#prod53">UnorderedListTypeDef</A> )</TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod49">RecordTypeDef</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod50">RecordTypeDef</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>( "closed" | "open" )? <LEFTBRACE> ( <A HREF="#prod53">RecordField</A> ( <COMMA> <A HREF="#prod53">RecordField</A> )* )? <RIGHTBRACE></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( "closed" | "open" )? <LEFTBRACE> ( <A HREF="#prod54">RecordField</A> ( <COMMA> <A HREF="#prod54">RecordField</A> )* )? <RIGHTBRACE></TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod53">RecordField</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod54">RecordField</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod17">Identifier</A> <COLON> <A HREF="#prod27">TypeExpr</A> ( <QUES> )?</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod18">Identifier</A> <COLON> <A HREF="#prod28">TypeExpr</A> ( <QUES> )?</TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod50">TypeReference</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod51">TypeReference</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod17">Identifier</A></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod18">Identifier</A></TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod51">OrderedListTypeDef</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod52">OrderedListTypeDef</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><LEFTBRACKET> ( <A HREF="#prod27">TypeExpr</A> ) <RIGHTBRACKET></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><LEFTBRACKET> ( <A HREF="#prod28">TypeExpr</A> ) <RIGHTBRACKET></TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod52">UnorderedListTypeDef</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod53">UnorderedListTypeDef</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><LEFTDBLBRACE> ( <A HREF="#prod27">TypeExpr</A> ) <RIGHTDBLBRACE></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><LEFTDBLBRACE> ( <A HREF="#prod28">TypeExpr</A> ) <RIGHTDBLBRACE></TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod37">FunctionName</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod38">FunctionName</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod17">Identifier</A> ( <DOT> <A HREF="#prod17">Identifier</A> ( "#" <A HREF="#prod17">Identifier</A> )? | "#" <A HREF="#prod17">Identifier</A> )?</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod18">Identifier</A> ( <DOT> <A HREF="#prod18">Identifier</A> ( "#" <A HREF="#prod18">Identifier</A> )? | "#" <A HREF="#prod18">Identifier</A> )?</TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod25">TypeName</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod26">TypeName</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod28">QualifiedName</A></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod29">QualifiedName</A></TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod17">Identifier</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod18">Identifier</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
<TD ALIGN=LEFT VALIGN=BASELINE><IDENTIFIER></TD>
</TR>
<TR>
<TD ALIGN=RIGHT VALIGN=BASELINE></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>|</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod36">StringLiteral</A></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod37">StringLiteral</A></TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod36">StringLiteral</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod37">StringLiteral</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
<TD ALIGN=LEFT VALIGN=BASELINE><STRING_LITERAL></TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod28">QualifiedName</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod29">QualifiedName</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod17">Identifier</A> ( <DOT> <A HREF="#prod17">Identifier</A> )?</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod18">Identifier</A> ( <DOT> <A HREF="#prod18">Identifier</A> )?</TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod44">DoubleQualifiedName</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod45">DoubleQualifiedName</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod17">Identifier</A> <DOT> <A HREF="#prod17">Identifier</A> ( <DOT> <A HREF="#prod17">Identifier</A> )?</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod18">Identifier</A> <DOT> <A HREF="#prod18">Identifier</A> ( <DOT> <A HREF="#prod18">Identifier</A> )?</TD>
</TR>
<TR>
<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod4">FunctionDeclaration</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>"declare" "function" <A HREF="#prod17">Identifier</A> <A HREF="#prod38">ParameterList</A> <LEFTBRACE> <A HREF="#prod39">Expression</A> <RIGHTBRACE></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"declare" "function" <A HREF="#prod18">Identifier</A> <A HREF="#prod39">ParameterList</A> <LEFTBRACE> <A HREF="#prod40">Expression</A> <RIGHTBRACE></TD>
</TR>
<TR>
<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod15">Query</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod39">Expression</A></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod40">Expression</A></TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod39">Expression</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod40">Expression</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>( <A HREF="#prod54">OperatorExpr</A> | <A HREF="#prod55">IfThenElse</A> | <A HREF="#prod56">FLWOGR</A> | <A HREF="#prod57">QuantifiedExpression</A> )</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( <A HREF="#prod55">OperatorExpr</A> | <A HREF="#prod56">IfThenElse</A> | <A HREF="#prod57">FLWOGR</A> | <A HREF="#prod58">QuantifiedExpression</A> )</TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod54">OperatorExpr</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod55">OperatorExpr</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod58">AndExpr</A> ( <OR> <A HREF="#prod58">AndExpr</A> )*</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod59">AndExpr</A> ( <OR> <A HREF="#prod59">AndExpr</A> )*</TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod58">AndExpr</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod59">AndExpr</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod59">RelExpr</A> ( <AND> <A HREF="#prod59">RelExpr</A> )*</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod60">RelExpr</A> ( <AND> <A HREF="#prod60">RelExpr</A> )*</TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod59">RelExpr</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod60">RelExpr</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod60">AddExpr</A> ( ( <LT> | <GT> | <LE> | <GE> | <EQ> | <NE> | <SIMILAR> ) <A HREF="#prod60">AddExpr</A> )?</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod61">AddExpr</A> ( ( <LT> | <GT> | <LE> | <GE> | <EQ> | <NE> | <SIMILAR> ) <A HREF="#prod61">AddExpr</A> )?</TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod60">AddExpr</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod61">AddExpr</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod61">MultExpr</A> ( ( <PLUS> | <MINUS> ) <A HREF="#prod61">MultExpr</A> )*</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod62">MultExpr</A> ( ( <PLUS> | <MINUS> ) <A HREF="#prod62">MultExpr</A> )*</TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod61">MultExpr</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod62">MultExpr</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod62">UnionExpr</A> ( ( <MUL> | <DIV> | <MOD> | <CARET> | <IDIV> ) <A HREF="#prod62">UnionExpr</A> )*</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod63">UnionExpr</A> ( ( <MUL> | <DIV> | <MOD> | <CARET> | <IDIV> ) <A HREF="#prod63">UnionExpr</A> )*</TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod62">UnionExpr</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod63">UnionExpr</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod63">UnaryExpr</A> ( <UNION> ( <A HREF="#prod63">UnaryExpr</A> ) )*</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod64">UnaryExpr</A> ( <UNION> ( <A HREF="#prod64">UnaryExpr</A> ) )*</TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod63">UnaryExpr</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod64">UnaryExpr</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>( ( <PLUS> | <MINUS> ) )? <A HREF="#prod64">ValueExpr</A></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( ( <PLUS> | <MINUS> ) )? <A HREF="#prod65">ValueExpr</A></TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod64">ValueExpr</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod65">ValueExpr</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod65">PrimaryExpr</A> ( <A HREF="#prod66">Field</A> | <A HREF="#prod67">Index</A> )*</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod66">PrimaryExpr</A> ( <A HREF="#prod67">Field</A> | <A HREF="#prod68">Index</A> )*</TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod66">Field</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod67">Field</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><DOT> <A HREF="#prod17">Identifier</A></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><DOT> <A HREF="#prod18">Identifier</A></TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod67">Index</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod68">Index</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><LEFTBRACKET> ( <A HREF="#prod39">Expression</A> | <QUES> ) <RIGHTBRACKET></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><LEFTBRACKET> ( <A HREF="#prod40">Expression</A> | <QUES> ) <RIGHTBRACKET></TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod65">PrimaryExpr</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod66">PrimaryExpr</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>( <A HREF="#prod68">FunctionCallExpr</A> | <A HREF="#prod69">Literal</A> | <A HREF="#prod70">DatasetAccessExpression</A> | <A HREF="#prod71">VariableRef</A> | <A HREF="#prod72">ListConstructor</A> | <A HREF="#prod73">RecordConstructor</A> | <A HREF="#prod74">ParenthesizedExpression</A> )</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( <A HREF="#prod69">FunctionCallExpr</A> | <A HREF="#prod70">Literal</A> | <A HREF="#prod71">DatasetAccessExpression</A> | <A HREF="#prod72">VariableRef</A> | <A HREF="#prod73">ListConstructor</A> | <A HREF="#prod74">RecordConstructor</A> | <A HREF="#prod75">ParenthesizedExpression</A> )</TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod69">Literal</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod70">Literal</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>( <A HREF="#prod36">StringLiteral</A> | <INTEGER_LITERAL> | <FLOAT_LITERAL> | <DOUBLE_LITERAL> | <NULL> | <TRUE> | <FALSE> )</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( <A HREF="#prod37">StringLiteral</A> | <INTEGER_LITERAL> | <FLOAT_LITERAL> | <DOUBLE_LITERAL> | <NULL> | <TRUE> | <FALSE> )</TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod71">VariableRef</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod72">VariableRef</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
<TD ALIGN=LEFT VALIGN=BASELINE><VARIABLE></TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod45">Variable</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod46">Variable</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
<TD ALIGN=LEFT VALIGN=BASELINE><VARIABLE></TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod72">ListConstructor</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod73">ListConstructor</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>( <A HREF="#prod75">OrderedListConstructor</A> | <A HREF="#prod76">UnorderedListConstructor</A> )</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( <A HREF="#prod76">OrderedListConstructor</A> | <A HREF="#prod77">UnorderedListConstructor</A> )</TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod75">OrderedListConstructor</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod76">OrderedListConstructor</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><LEFTBRACKET> ( <A HREF="#prod39">Expression</A> ( <COMMA> <A HREF="#prod39">Expression</A> )* )? <RIGHTBRACKET></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><LEFTBRACKET> ( <A HREF="#prod40">Expression</A> ( <COMMA> <A HREF="#prod40">Expression</A> )* )? <RIGHTBRACKET></TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod76">UnorderedListConstructor</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod77">UnorderedListConstructor</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><LEFTDBLBRACE> ( <A HREF="#prod39">Expression</A> ( <COMMA> <A HREF="#prod39">Expression</A> )* )? <RIGHTDBLBRACE></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><LEFTDBLBRACE> ( <A HREF="#prod40">Expression</A> ( <COMMA> <A HREF="#prod40">Expression</A> )* )? <RIGHTDBLBRACE></TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod73">RecordConstructor</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod74">RecordConstructor</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><LEFTBRACE> ( <A HREF="#prod77">FieldBinding</A> ( <COMMA> <A HREF="#prod77">FieldBinding</A> )* )? <RIGHTBRACE></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><LEFTBRACE> ( <A HREF="#prod78">FieldBinding</A> ( <COMMA> <A HREF="#prod78">FieldBinding</A> )* )? <RIGHTBRACE></TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod77">FieldBinding</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod78">FieldBinding</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod39">Expression</A> <COLON> <A HREF="#prod39">Expression</A></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod40">Expression</A> <COLON> <A HREF="#prod40">Expression</A></TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod68">FunctionCallExpr</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod69">FunctionCallExpr</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod37">FunctionName</A> <LEFTPAREN> ( <A HREF="#prod39">Expression</A> ( <COMMA> <A HREF="#prod39">Expression</A> )* )? <RIGHTPAREN></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod38">FunctionName</A> <LEFTPAREN> ( <A HREF="#prod40">Expression</A> ( <COMMA> <A HREF="#prod40">Expression</A> )* )? <RIGHTPAREN></TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod70">DatasetAccessExpression</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod71">DatasetAccessExpression</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><DATASET> ( ( <A HREF="#prod17">Identifier</A> ( <DOT> <A HREF="#prod17">Identifier</A> )? ) | ( <LEFTPAREN> <A HREF="#prod39">Expression</A> <RIGHTPAREN> ) )</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><DATASET> ( ( <A HREF="#prod18">Identifier</A> ( <DOT> <A HREF="#prod18">Identifier</A> )? ) | ( <LEFTPAREN> <A HREF="#prod40">Expression</A> <RIGHTPAREN> ) )</TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod74">ParenthesizedExpression</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod75">ParenthesizedExpression</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><LEFTPAREN> <A HREF="#prod39">Expression</A> <RIGHTPAREN></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><LEFTPAREN> <A HREF="#prod40">Expression</A> <RIGHTPAREN></TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod55">IfThenElse</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod56">IfThenElse</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><IF> <LEFTPAREN> <A HREF="#prod39">Expression</A> <RIGHTPAREN> <THEN> <A HREF="#prod39">Expression</A> <ELSE> <A HREF="#prod39">Expression</A></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><IF> <LEFTPAREN> <A HREF="#prod40">Expression</A> <RIGHTPAREN> <THEN> <A HREF="#prod40">Expression</A> <ELSE> <A HREF="#prod40">Expression</A></TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod56">FLWOGR</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod57">FLWOGR</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>( <A HREF="#prod78">ForClause</A> | <A HREF="#prod79">LetClause</A> ) ( <A HREF="#prod80">Clause</A> )* ( <RETURN> | <SELECT> ) <A HREF="#prod39">Expression</A></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( <A HREF="#prod79">ForClause</A> | <A HREF="#prod80">LetClause</A> ) ( <A HREF="#prod81">Clause</A> )* ( <RETURN> | <SELECT> ) <A HREF="#prod40">Expression</A></TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod80">Clause</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod81">Clause</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>( <A HREF="#prod78">ForClause</A> | <A HREF="#prod79">LetClause</A> | <A HREF="#prod81">WhereClause</A> | <A HREF="#prod82">OrderbyClause</A> | <A HREF="#prod83">GroupClause</A> | <A HREF="#prod84">LimitClause</A> | <A HREF="#prod85">DistinctClause</A> )</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( <A HREF="#prod79">ForClause</A> | <A HREF="#prod80">LetClause</A> | <A HREF="#prod82">WhereClause</A> | <A HREF="#prod83">OrderbyClause</A> | <A HREF="#prod84">GroupClause</A> | <A HREF="#prod85">LimitClause</A> | <A HREF="#prod86">DistinctClause</A> )</TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod78">ForClause</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod79">ForClause</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>( <FOR> | <FROM> ) <A HREF="#prod45">Variable</A> ( <AT> <A HREF="#prod45">Variable</A> )? <IN> ( <A HREF="#prod39">Expression</A> )</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( <FOR> | <FROM> ) <A HREF="#prod46">Variable</A> ( <AT> <A HREF="#prod46">Variable</A> )? <IN> ( <A HREF="#prod40">Expression</A> )</TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod79">LetClause</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod80">LetClause</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>( <LET> | <WITH> ) <A HREF="#prod45">Variable</A> <ASSIGN> <A HREF="#prod39">Expression</A></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( <LET> | <WITH> ) <A HREF="#prod46">Variable</A> <ASSIGN> <A HREF="#prod40">Expression</A></TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod81">WhereClause</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod82">WhereClause</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><WHERE> <A HREF="#prod39">Expression</A></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><WHERE> <A HREF="#prod40">Expression</A></TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod82">OrderbyClause</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod83">OrderbyClause</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>( <ORDER> <BY> <A HREF="#prod39">Expression</A> ( ( <ASC> ) | ( <DESC> ) )? ( <COMMA> <A HREF="#prod39">Expression</A> ( ( <ASC> ) | ( <DESC> ) )? )* )</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( <ORDER> <BY> <A HREF="#prod40">Expression</A> ( ( <ASC> ) | ( <DESC> ) )? ( <COMMA> <A HREF="#prod40">Expression</A> ( ( <ASC> ) | ( <DESC> ) )? )* )</TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod83">GroupClause</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod84">GroupClause</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><GROUP> <BY> ( <A HREF="#prod45">Variable</A> <ASSIGN> )? <A HREF="#prod39">Expression</A> ( <COMMA> ( <A HREF="#prod45">Variable</A> <ASSIGN> )? <A HREF="#prod39">Expression</A> )* ( <DECOR> <A HREF="#prod45">Variable</A> <ASSIGN> <A HREF="#prod39">Expression</A> ( <COMMA> <DECOR> <A HREF="#prod45">Variable</A> <ASSIGN> <A HREF="#prod39">Expression</A> )* )? <WITH> <A HREF="#prod71">VariableRef</A> ( <COMMA> <A HREF="#prod71">VariableRef</A> )*</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><GROUP> <BY> ( <A HREF="#prod46">Variable</A> <ASSIGN> )? <A HREF="#prod40">Expression</A> ( <COMMA> ( <A HREF="#prod46">Variable</A> <ASSIGN> )? <A HREF="#prod40">Expression</A> )* ( <DECOR> <A HREF="#prod46">Variable</A> <ASSIGN> <A HREF="#prod40">Expression</A> ( <COMMA> <DECOR> <A HREF="#prod46">Variable</A> <ASSIGN> <A HREF="#prod40">Expression</A> )* )? ( <WITH> | <KEEPING> ) <A HREF="#prod72">VariableRef</A> ( <COMMA> <A HREF="#prod72">VariableRef</A> )*</TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod84">LimitClause</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod85">LimitClause</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><LIMIT> <A HREF="#prod39">Expression</A> ( <OFFSET> <A HREF="#prod39">Expression</A> )?</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><LIMIT> <A HREF="#prod40">Expression</A> ( <OFFSET> <A HREF="#prod40">Expression</A> )?</TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod85">DistinctClause</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod86">DistinctClause</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE><DISTINCT> <BY> <A HREF="#prod39">Expression</A> ( <COMMA> <A HREF="#prod39">Expression</A> )*</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><DISTINCT> <BY> <A HREF="#prod40">Expression</A> ( <COMMA> <A HREF="#prod40">Expression</A> )*</TD>
</TR>
<TR>
-<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod57">QuantifiedExpression</A></TD>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod58">QuantifiedExpression</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>( ( <SOME> ) | ( <EVERY> ) ) <A HREF="#prod45">Variable</A> <IN> <A HREF="#prod39">Expression</A> ( <COMMA> <A HREF="#prod45">Variable</A> <IN> <A HREF="#prod39">Expression</A> )* <SATISFIES> <A HREF="#prod39">Expression</A></TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( ( <SOME> ) | ( <EVERY> ) ) <A HREF="#prod46">Variable</A> <IN> <A HREF="#prod40">Expression</A> ( <COMMA> <A HREF="#prod46">Variable</A> <IN> <A HREF="#prod40">Expression</A> )* <SATISFIES> <A HREF="#prod40">Expression</A></TD>
</TR>
</TABLE>
</BODY>
diff --git a/asterix-aql/src/main/javacc/AQL.jj b/asterix-aql/src/main/javacc/AQL.jj
index 3e64da4..8150871 100644
--- a/asterix-aql/src/main/javacc/AQL.jj
+++ b/asterix-aql/src/main/javacc/AQL.jj
@@ -407,6 +407,7 @@
DatasetDecl dsetDecl = null;
boolean autogenerated = false;
String compactionPolicy = null;
+ boolean temp = false;
List<String> filterField = null;
}
{
@@ -434,7 +435,11 @@
ifNotExists);
}
- | ("internal")? <DATASET> nameComponents = QualifiedName()
+ | ("internal" | "temporary" {
+ temp = token.image.toLowerCase().equals("temporary");
+ }
+ )?
+ <DATASET> nameComponents = QualifiedName()
<LEFTPAREN> typeName = Identifier() <RIGHTPAREN>
ifNotExists = IfNotExists()
primaryKeyFields = PrimaryKey()
@@ -451,7 +456,8 @@
autogenerated,
compactionPolicy,
compactionPolicyProperties,
- filterField);
+ filterField,
+ temp);
dsetDecl = new DatasetDecl(nameComponents.first,
nameComponents.second,
new Identifier(typeName),
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/DatasetConfig.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/DatasetConfig.java
index ec11c5d..debab84 100644
--- a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/DatasetConfig.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/DatasetConfig.java
@@ -39,15 +39,15 @@
}
public enum ExternalDatasetTransactionState {
- COMMIT, // The committed state <- nothing is required->
- BEGIN, // The state after starting the refresh transaction <- will either abort moving to committed state or move to ready to commit->
+ COMMIT, // The committed state <- nothing is required->
+ BEGIN, // The state after starting the refresh transaction <- will either abort moving to committed state or move to ready to commit->
READY_TO_COMMIT // The transaction is ready to commit <- can only move forward to committed state->
};
public enum ExternalFilePendingOp {
- PENDING_NO_OP, // the stored file is part of a committed transaction nothing is required
- PENDING_ADD_OP, // the stored file is part of an ongoing transaction (will be added if transaction succeed)
- PENDING_DROP_OP, // the stored file is part of an ongoing transaction (will be dropped if transaction succeed)
- PENDING_APPEND_OP // the stored file is part of an ongoing transaction (will be updated if transaction succeed)
+ PENDING_NO_OP, // the stored file is part of a committed transaction nothing is required
+ PENDING_ADD_OP, // the stored file is part of an ongoing transaction (will be added if transaction succeed)
+ PENDING_DROP_OP, // the stored file is part of an ongoing transaction (will be dropped if transaction succeed)
+ PENDING_APPEND_OP // the stored file is part of an ongoing transaction (will be updated if transaction succeed)
};
}
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/context/DatasetLifecycleManager.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/context/DatasetLifecycleManager.java
index b7064d3..9a91289 100644
--- a/asterix-common/src/main/java/edu/uci/ics/asterix/common/context/DatasetLifecycleManager.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/context/DatasetLifecycleManager.java
@@ -446,7 +446,8 @@
AbstractLSMIOOperationCallback ioCallback = (AbstractLSMIOOperationCallback) iInfo.index
.getIOOperationCallback();
if (!(((AbstractLSMIndex) iInfo.index).isCurrentMutableComponentEmpty()
- || ioCallback.hasPendingFlush() || opTracker.isFlushLogCreated() || opTracker.isFlushOnExit())) {
+ || ioCallback.hasPendingFlush() || opTracker.isFlushLogCreated() || opTracker
+ .isFlushOnExit())) {
long firstLSN = ioCallback.getFirstLSN();
if (firstLSN < targetLSN) {
opTracker.setFlushOnExit(true);
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/context/PrimaryIndexOperationTracker.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/context/PrimaryIndexOperationTracker.java
index ee12248..7a56600 100644
--- a/asterix-common/src/main/java/edu/uci/ics/asterix/common/context/PrimaryIndexOperationTracker.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/context/PrimaryIndexOperationTracker.java
@@ -70,12 +70,15 @@
}
@Override
- public synchronized void completeOperation(ILSMIndex index, LSMOperationType opType, ISearchOperationCallback searchCallback,
- IModificationOperationCallback modificationCallback) throws HyracksDataException {
+ public synchronized void completeOperation(ILSMIndex index, LSMOperationType opType,
+ ISearchOperationCallback searchCallback, IModificationOperationCallback modificationCallback)
+ throws HyracksDataException {
if (opType == LSMOperationType.MODIFICATION || opType == LSMOperationType.FORCE_MODIFICATION) {
decrementNumActiveOperations(modificationCallback);
if (numActiveOperations.get() == 0) {
flushIfRequested();
+ } else if (numActiveOperations.get() < 0) {
+ throw new HyracksDataException("The number of active operations cannot be negative!");
}
} else if (opType == LSMOperationType.FLUSH || opType == LSMOperationType.MERGE) {
dsInfo.undeclareActiveIOOperation();
@@ -91,7 +94,8 @@
if (!flushOnExit) {
for (ILSMIndex lsmIndex : indexes) {
- if (((ILSMIndexInternal) lsmIndex).hasFlushRequestForCurrentMutableComponent()) {
+ ILSMIndexInternal lsmIndexInternal = (ILSMIndexInternal) lsmIndex;
+ if (lsmIndexInternal.hasFlushRequestForCurrentMutableComponent()) {
needsFlush = true;
break;
}
@@ -135,7 +139,6 @@
//schedule flush after update
accessor.scheduleFlush(lsmIndex.getIOOperationCallback());
-
}
flushLogCreated = false;
diff --git a/asterix-common/src/test/java/edu/uci/ics/asterix/test/aql/TestsUtils.java b/asterix-common/src/test/java/edu/uci/ics/asterix/test/aql/TestsUtils.java
index 70e2f6f..1283dd6 100644
--- a/asterix-common/src/test/java/edu/uci/ics/asterix/test/aql/TestsUtils.java
+++ b/asterix-common/src/test/java/edu/uci/ics/asterix/test/aql/TestsUtils.java
@@ -390,6 +390,7 @@
for (TestFileContext ctx : testFileCtxs) {
testFile = ctx.getFile();
statement = TestsUtils.readTestFile(testFile);
+ boolean failed = false;
try {
switch (ctx.getType()) {
case "ddl":
@@ -468,7 +469,14 @@
TestsUtils.executeUpdate(statement);
} catch (Exception e) {
//An exception is expected.
+ failed = true;
+ e.printStackTrace();
}
+ if (!failed) {
+ throw new Exception("Test \"" + testFile + "\" FAILED!\n An exception"
+ + "is expected.");
+ }
+ System.err.println("...but that was expected.");
break;
case "script":
try {
@@ -489,10 +497,16 @@
case "errddl": // a ddlquery that expects error
try {
TestsUtils.executeDDL(statement);
-
} catch (Exception e) {
// expected error happens
+ failed = true;
+ e.printStackTrace();
}
+ if (!failed) {
+ throw new Exception("Test \"" + testFile + "\" FAILED!\n An exception"
+ + "is expected.");
+ }
+ System.err.println("...but that was expected.");
break;
default:
throw new IllegalArgumentException("No statements of type " + ctx.getType());
@@ -505,9 +519,9 @@
System.err.println("...Unexpected!");
throw new Exception("Test \"" + testFile + "\" FAILED!", e);
} else {
- System.err.println("...but that was expected.");
LOGGER.info("[TEST]: " + testCaseCtx.getTestCase().getFilePath() + "/" + cUnit.getName()
+ " failed as expected: " + e.getMessage());
+ System.err.println("...but that was expected.");
}
}
}
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/indexing/dataflow/HDFSLookupAdapterFactory.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/indexing/dataflow/HDFSLookupAdapterFactory.java
index a6dda87..3c7d239 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/indexing/dataflow/HDFSLookupAdapterFactory.java
+++ b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/indexing/dataflow/HDFSLookupAdapterFactory.java
@@ -50,6 +50,8 @@
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallbackFactory;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
import edu.uci.ics.hyracks.storage.am.lsm.btree.dataflow.ExternalBTreeDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
@@ -140,13 +142,14 @@
throw new AlgebricksException(" Unabel to create merge policy factory for external dataset", e);
}
+ boolean temp = dataset.getDatasetDetails().isTemp();
// Create the file index data flow helper
ExternalBTreeDataflowHelperFactory indexDataflowHelperFactory = new ExternalBTreeDataflowHelperFactory(
compactionInfo.first, compactionInfo.second, new SecondaryIndexOperationTrackerProvider(
dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
LSMBTreeIOOperationCallbackFactory.INSTANCE, metadataProvider.getStorageProperties()
.getBloomFilterFalsePositiveRate(), ExternalDatasetsRegistry.INSTANCE.getAndLockDatasetVersion(
- dataset, metadataProvider));
+ dataset, metadataProvider), !temp);
// Create the out record descriptor, appContext and fileSplitProvider for the files index
RecordDescriptor outRecDesc = JobGenHelper.mkRecordDescriptor(typeEnv, opSchema, context);
@@ -160,12 +163,14 @@
throw new AlgebricksException(e);
}
+ ISearchOperationCallbackFactory searchOpCallbackFactory = temp ? NoOpOperationCallbackFactory.INSTANCE
+ : new SecondaryIndexSearchOperationCallbackFactory();
// Create the operator
ExternalLoopkupOperatorDiscriptor op = new ExternalLoopkupOperatorDiscriptor(jobSpec, adapterFactory,
outRecDesc, indexDataflowHelperFactory, retainInput, appContext.getIndexLifecycleManagerProvider(),
appContext.getStorageManagerInterface(), spPc.first, dataset.getDatasetId(), metadataProvider
- .getStorageProperties().getBloomFilterFalsePositiveRate(),
- new SecondaryIndexSearchOperationCallbackFactory(),retainNull,context.getNullWriterFactory());
+ .getStorageProperties().getBloomFilterFalsePositiveRate(), searchOpCallbackFactory, retainNull,
+ context.getNullWriterFactory());
// Return value
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(op, spPc.second);
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_index_only/primary_index_only.1.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_index_only/primary_index_only.1.script.aql
new file mode 100644
index 0000000..323b1cf
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_index_only/primary_index_only.1.script.aql
@@ -0,0 +1 @@
+create_and_start.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_index_only/primary_index_only.2.ddl.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_index_only/primary_index_only.2.ddl.aql
new file mode 100644
index 0000000..b7f6b19
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_index_only/primary_index_only.2.ddl.aql
@@ -0,0 +1,41 @@
+/*
+ * Test case Name : primary_index_only.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : September 25 2013
+ */
+
+drop dataverse recovery if exists;
+create dataverse recovery;
+use dataverse recovery;
+
+/* For raw Fragile data */
+create type FragileTypeRaw as closed {
+ row_id: int32,
+ sid: int32,
+ date: string,
+ day: int32,
+ time: string,
+ bpm: int32,
+ RR: float
+};
+
+/* For cleaned Fragile data */
+create type FragileType as closed {
+ row_id: int32,
+ sid: int32,
+ date: date,
+ day: int32,
+ time: time,
+ bpm: int32,
+ RR: float
+};
+
+/* Create dataset for loading raw Fragile data */
+create temporary dataset Fragile_raw (FragileTypeRaw)
+primary key row_id;
+
+/* Create dataset for cleaned Fragile data */
+create temporary dataset Fragile (FragileType)
+primary key row_id;
+
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_index_only/primary_index_only.3.update.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_index_only/primary_index_only.3.update.aql
new file mode 100644
index 0000000..6d0e431
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_index_only/primary_index_only.3.update.aql
@@ -0,0 +1,11 @@
+/*
+ * Test case Name : primary_index_only.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : September 25 2013
+ */
+
+use dataverse recovery;
+
+load dataset Fragile_raw using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="127.0.0.1://../../../../../../asterix-app/data/csv/fragile_01.csv"),("format"="delimited-text"),("delimiter"=",")) pre-sorted;
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_index_only/primary_index_only.4.txneu.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_index_only/primary_index_only.4.txneu.aql
new file mode 100644
index 0000000..3001e9b
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_index_only/primary_index_only.4.txneu.aql
@@ -0,0 +1,22 @@
+/*
+ * Test case Name : primary_index_only.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : September 25 2013
+ */
+
+use dataverse recovery;
+
+/* Load Fragile data from raw dataset into cleaned dataset */
+insert into dataset Fragile (
+ for $t in dataset Fragile_raw
+ return {
+ "row_id": $t.row_id % 28000,
+ "sid": $t.sid,
+ "date": date($t.date),
+ "day": $t.day,
+ "time": parse-time($t.time, "h:m:s"),
+ "bpm": $t.bpm,
+ "RR": $t.RR
+ }
+);
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_index_only/primary_index_only.5.txnqbc.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_index_only/primary_index_only.5.txnqbc.aql
new file mode 100644
index 0000000..e77b008
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_index_only/primary_index_only.5.txnqbc.aql
@@ -0,0 +1,10 @@
+/*
+ * Test case Name : primary_index_only.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : September 25 2013
+ */
+
+use dataverse recovery;
+
+count (for $x in dataset Fragile return $x);
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_index_only/primary_index_only.6.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_index_only/primary_index_only.6.script.aql
new file mode 100644
index 0000000..31d37ae
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_index_only/primary_index_only.6.script.aql
@@ -0,0 +1 @@
+kill_cc_and_nc.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_index_only/primary_index_only.7.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_index_only/primary_index_only.7.script.aql
new file mode 100644
index 0000000..37ef6c0
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_index_only/primary_index_only.7.script.aql
@@ -0,0 +1 @@
+stop_and_start.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_index_only/primary_index_only.8.txnqar.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_index_only/primary_index_only.8.txnqar.aql
new file mode 100644
index 0000000..e77b008
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_index_only/primary_index_only.8.txnqar.aql
@@ -0,0 +1,10 @@
+/*
+ * Test case Name : primary_index_only.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : September 25 2013
+ */
+
+use dataverse recovery;
+
+count (for $x in dataset Fragile return $x);
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_index_only/primary_index_only.9.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_index_only/primary_index_only.9.script.aql
new file mode 100644
index 0000000..f75dfc9
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_index_only/primary_index_only.9.script.aql
@@ -0,0 +1 @@
+stop_and_delete.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_default_secondary_index/primary_plus_default_secondary_index.1.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_default_secondary_index/primary_plus_default_secondary_index.1.script.aql
new file mode 100644
index 0000000..323b1cf
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_default_secondary_index/primary_plus_default_secondary_index.1.script.aql
@@ -0,0 +1 @@
+create_and_start.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_default_secondary_index/primary_plus_default_secondary_index.2.ddl.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_default_secondary_index/primary_plus_default_secondary_index.2.ddl.aql
new file mode 100644
index 0000000..65685ae
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_default_secondary_index/primary_plus_default_secondary_index.2.ddl.aql
@@ -0,0 +1,53 @@
+/*
+ * Test case Name : primary_plus_default_secondary_index.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : Oct 15 2013
+ */
+
+drop dataverse recovery if exists;
+create dataverse recovery;
+use dataverse recovery;
+
+/* For raw Fragile data */
+create type FragileTypeRaw as closed {
+ row_id: int32,
+ sid: int32,
+ date: string,
+ day: int32,
+ time: string,
+ bpm: int32,
+ RR: float,
+ /* new string field and location field*/
+ text: string,
+ location: point,
+ text2: string
+};
+
+/* For cleaned Fragile data */
+create type FragileType as closed {
+ row_id: int32,
+ sid: int32,
+ date: date,
+ day: int32,
+ time: time,
+ bpm: int32,
+ RR: float,
+
+ /* new string field and location field*/
+ text: string,
+ location: point,
+ text2: string
+};
+
+/* Create dataset for loading raw Fragile data */
+create temporary dataset Fragile_raw (FragileTypeRaw)
+primary key row_id;
+
+/* Create dataset for cleaned Fragile data */
+create temporary dataset Fragile (FragileType)
+primary key row_id;
+
+/* Create default secondary index on dataset clean Fragile */
+create index cfSidIdx on Fragile(sid);
+
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_default_secondary_index/primary_plus_default_secondary_index.3.update.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_default_secondary_index/primary_plus_default_secondary_index.3.update.aql
new file mode 100644
index 0000000..5d7d6f2
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_default_secondary_index/primary_plus_default_secondary_index.3.update.aql
@@ -0,0 +1,11 @@
+/*
+ * Test case Name : primary_plus_default_secondary_index.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : Oct 15 2013
+ */
+
+use dataverse recovery;
+
+load dataset Fragile_raw using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="127.0.0.1://../../../../../../asterix-app/data/csv/fragile_02.adm"),("format"="adm")) pre-sorted;
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_default_secondary_index/primary_plus_default_secondary_index.4.txneu.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_default_secondary_index/primary_plus_default_secondary_index.4.txneu.aql
new file mode 100644
index 0000000..12fd699
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_default_secondary_index/primary_plus_default_secondary_index.4.txneu.aql
@@ -0,0 +1,25 @@
+/*
+ * Test case Name : primary_plus_default_secondary_index.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : Oct 15 2013
+ */
+
+use dataverse recovery;
+
+/* Load Fragile data from raw dataset into cleaned dataset */
+insert into dataset Fragile (
+ for $t in dataset Fragile_raw
+ return {
+ "row_id": $t.row_id % 28000,
+ "sid": $t.sid,
+ "date": date($t.date),
+ "day": $t.day,
+ "time": parse-time($t.time, "h:m:s"),
+ "bpm": $t.bpm,
+ "RR": $t.RR,
+ "text": $t.text,
+ "location": $t.location,
+ "text2": $t.text2
+ }
+);
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_default_secondary_index/primary_plus_default_secondary_index.5.txnqbc.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_default_secondary_index/primary_plus_default_secondary_index.5.txnqbc.aql
new file mode 100644
index 0000000..4d41446
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_default_secondary_index/primary_plus_default_secondary_index.5.txnqbc.aql
@@ -0,0 +1,10 @@
+/*
+ * Test case Name : primary_plus_default_secondary_index.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : Otc 15 2013
+ */
+
+use dataverse recovery;
+
+count (for $x in dataset Fragile where $x.sid=1 return $x);
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_default_secondary_index/primary_plus_default_secondary_index.6.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_default_secondary_index/primary_plus_default_secondary_index.6.script.aql
new file mode 100644
index 0000000..31d37ae
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_default_secondary_index/primary_plus_default_secondary_index.6.script.aql
@@ -0,0 +1 @@
+kill_cc_and_nc.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_default_secondary_index/primary_plus_default_secondary_index.7.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_default_secondary_index/primary_plus_default_secondary_index.7.script.aql
new file mode 100644
index 0000000..37ef6c0
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_default_secondary_index/primary_plus_default_secondary_index.7.script.aql
@@ -0,0 +1 @@
+stop_and_start.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_default_secondary_index/primary_plus_default_secondary_index.8.txnqar.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_default_secondary_index/primary_plus_default_secondary_index.8.txnqar.aql
new file mode 100644
index 0000000..dd2cde5
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_default_secondary_index/primary_plus_default_secondary_index.8.txnqar.aql
@@ -0,0 +1,10 @@
+/*
+ * Test case Name : primary_plus_default_secondary_index.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : Oct 15 2013
+ */
+
+use dataverse recovery;
+
+count (for $x in dataset Fragile where $x.sid=1 return $x);
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_default_secondary_index/primary_plus_default_secondary_index.9.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_default_secondary_index/primary_plus_default_secondary_index.9.script.aql
new file mode 100644
index 0000000..f75dfc9
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_default_secondary_index/primary_plus_default_secondary_index.9.script.aql
@@ -0,0 +1 @@
+stop_and_delete.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_keyword_secondary_index/primary_plus_keyword_secondary_index.1.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_keyword_secondary_index/primary_plus_keyword_secondary_index.1.script.aql
new file mode 100644
index 0000000..323b1cf
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_keyword_secondary_index/primary_plus_keyword_secondary_index.1.script.aql
@@ -0,0 +1 @@
+create_and_start.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_keyword_secondary_index/primary_plus_keyword_secondary_index.2.ddl.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_keyword_secondary_index/primary_plus_keyword_secondary_index.2.ddl.aql
new file mode 100644
index 0000000..80fb1ff
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_keyword_secondary_index/primary_plus_keyword_secondary_index.2.ddl.aql
@@ -0,0 +1,54 @@
+/*
+ * Test case Name : primary_plus_keyword_secondary_index.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : Oct 15 2013
+ */
+
+drop dataverse recovery if exists;
+create dataverse recovery;
+use dataverse recovery;
+
+/* For raw Fragile data */
+create type FragileTypeRaw as closed {
+ row_id: int32,
+ sid: int32,
+ date: string,
+ day: int32,
+ time: string,
+ bpm: int32,
+ RR: float,
+ /* new string field and location field*/
+ text: string,
+ location: point,
+ text2: string
+
+};
+
+/* For cleaned Fragile data */
+create type FragileType as closed {
+ row_id: int32,
+ sid: int32,
+ date: date,
+ day: int32,
+ time: time,
+ bpm: int32,
+ RR: float,
+
+ /* new string field and location field*/
+ text: string,
+ location: point,
+ text2: string
+};
+
+/* Create dataset for loading raw Fragile data */
+create temporary dataset Fragile_raw (FragileTypeRaw)
+primary key row_id;
+
+/* Create dataset for cleaned Fragile data */
+create temporary dataset Fragile (FragileType)
+primary key row_id;
+
+/* Create keyword secondary index on dataset clean Fragile */
+create index cfText on Fragile(text) type keyword;
+
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_keyword_secondary_index/primary_plus_keyword_secondary_index.3.update.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_keyword_secondary_index/primary_plus_keyword_secondary_index.3.update.aql
new file mode 100644
index 0000000..9d9757f
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_keyword_secondary_index/primary_plus_keyword_secondary_index.3.update.aql
@@ -0,0 +1,12 @@
+/*
+ * Test case Name : primary_plus_keyword_secondary_index.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : Oct 15 2013
+ */
+
+use dataverse recovery;
+
+
+load dataset Fragile_raw using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="127.0.0.1://../../../../../../asterix-app/data/csv/fragile_02.adm"),("format"="adm")) pre-sorted;
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_keyword_secondary_index/primary_plus_keyword_secondary_index.4.txneu.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_keyword_secondary_index/primary_plus_keyword_secondary_index.4.txneu.aql
new file mode 100644
index 0000000..3c38973
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_keyword_secondary_index/primary_plus_keyword_secondary_index.4.txneu.aql
@@ -0,0 +1,25 @@
+/*
+ * Test case Name : primary_plus_keyword_secondary_index.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : Oct 15 2013
+ */
+
+use dataverse recovery;
+
+/* Load Fragile data from raw dataset into cleaned dataset */
+insert into dataset Fragile (
+ for $t in dataset Fragile_raw
+ return {
+ "row_id": $t.row_id % 28000,
+ "sid": $t.sid,
+ "date": date($t.date),
+ "day": $t.day,
+ "time": parse-time($t.time, "h:m:s"),
+ "bpm": $t.bpm,
+ "RR": $t.RR,
+ "text": $t.text,
+ "location": $t.location,
+ "text2": $t.text2
+ }
+);
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_keyword_secondary_index/primary_plus_keyword_secondary_index.5.txnqbc.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_keyword_secondary_index/primary_plus_keyword_secondary_index.5.txnqbc.aql
new file mode 100644
index 0000000..f069443
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_keyword_secondary_index/primary_plus_keyword_secondary_index.5.txnqbc.aql
@@ -0,0 +1,5 @@
+use dataverse recovery;
+
+for $x in dataset Fragile
+where word-tokens($x.text) ~= word-tokens(" 1 20130417 1")
+return $x.text;
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_keyword_secondary_index/primary_plus_keyword_secondary_index.6.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_keyword_secondary_index/primary_plus_keyword_secondary_index.6.script.aql
new file mode 100644
index 0000000..31d37ae
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_keyword_secondary_index/primary_plus_keyword_secondary_index.6.script.aql
@@ -0,0 +1 @@
+kill_cc_and_nc.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_keyword_secondary_index/primary_plus_keyword_secondary_index.7.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_keyword_secondary_index/primary_plus_keyword_secondary_index.7.script.aql
new file mode 100644
index 0000000..37ef6c0
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_keyword_secondary_index/primary_plus_keyword_secondary_index.7.script.aql
@@ -0,0 +1 @@
+stop_and_start.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_keyword_secondary_index/primary_plus_keyword_secondary_index.8.txnqar.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_keyword_secondary_index/primary_plus_keyword_secondary_index.8.txnqar.aql
new file mode 100644
index 0000000..f069443
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_keyword_secondary_index/primary_plus_keyword_secondary_index.8.txnqar.aql
@@ -0,0 +1,5 @@
+use dataverse recovery;
+
+for $x in dataset Fragile
+where word-tokens($x.text) ~= word-tokens(" 1 20130417 1")
+return $x.text;
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_keyword_secondary_index/primary_plus_keyword_secondary_index.9.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_keyword_secondary_index/primary_plus_keyword_secondary_index.9.script.aql
new file mode 100644
index 0000000..f75dfc9
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_keyword_secondary_index/primary_plus_keyword_secondary_index.9.script.aql
@@ -0,0 +1 @@
+stop_and_delete.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_multiple_secondary_indices/primary_plus_multiple_secondary_indices.1.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_multiple_secondary_indices/primary_plus_multiple_secondary_indices.1.script.aql
new file mode 100644
index 0000000..323b1cf
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_multiple_secondary_indices/primary_plus_multiple_secondary_indices.1.script.aql
@@ -0,0 +1 @@
+create_and_start.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_multiple_secondary_indices/primary_plus_multiple_secondary_indices.2.ddl.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_multiple_secondary_indices/primary_plus_multiple_secondary_indices.2.ddl.aql
new file mode 100644
index 0000000..7f3e1db
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_multiple_secondary_indices/primary_plus_multiple_secondary_indices.2.ddl.aql
@@ -0,0 +1,60 @@
+/*
+ * Test case Name : primary_plus_keyword_secondary_index.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : Oct 15 2013
+ */
+
+drop dataverse recovery if exists;
+create dataverse recovery;
+use dataverse recovery;
+
+/* For raw Fragile data */
+create type FragileTypeRaw as closed {
+ row_id: int32,
+ sid: int32,
+ date: string,
+ day: int32,
+ time: string,
+ bpm: int32,
+ RR: float,
+ /* new string field and location field*/
+ text: string,
+ location: point,
+ text2: string
+
+};
+
+/* For cleaned Fragile data */
+create type FragileType as closed {
+ row_id: int32,
+ sid: int32,
+ date: date,
+ day: int32,
+ time: time,
+ bpm: int32,
+ RR: float,
+
+ /* new string field and location field*/
+ text: string,
+ location: point,
+ text2: string
+};
+
+/* Create dataset for loading raw Fragile data */
+create temporary dataset Fragile_raw (FragileTypeRaw)
+primary key row_id;
+
+/* Create dataset for cleaned Fragile data */
+create temporary dataset Fragile (FragileType)
+primary key row_id;
+
+
+/* Create rtree secondary index on dataset clean Fragile */
+create index cfLocation on Fragile(location) type rtree;
+
+create index cfText on Fragile(text) type keyword;
+
+create index cfSidIdx on Fragile(sid);
+
+create index cfText2Ix on Fragile(text2) type ngram(3);
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_multiple_secondary_indices/primary_plus_multiple_secondary_indices.3.update.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_multiple_secondary_indices/primary_plus_multiple_secondary_indices.3.update.aql
new file mode 100644
index 0000000..9d9757f
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_multiple_secondary_indices/primary_plus_multiple_secondary_indices.3.update.aql
@@ -0,0 +1,12 @@
+/*
+ * Test case Name : primary_plus_keyword_secondary_index.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : Oct 15 2013
+ */
+
+use dataverse recovery;
+
+
+load dataset Fragile_raw using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="127.0.0.1://../../../../../../asterix-app/data/csv/fragile_02.adm"),("format"="adm")) pre-sorted;
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_multiple_secondary_indices/primary_plus_multiple_secondary_indices.4.txneu.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_multiple_secondary_indices/primary_plus_multiple_secondary_indices.4.txneu.aql
new file mode 100644
index 0000000..3c38973
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_multiple_secondary_indices/primary_plus_multiple_secondary_indices.4.txneu.aql
@@ -0,0 +1,25 @@
+/*
+ * Test case Name : primary_plus_keyword_secondary_index.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : Oct 15 2013
+ */
+
+use dataverse recovery;
+
+/* Load Fragile data from raw dataset into cleaned dataset */
+insert into dataset Fragile (
+ for $t in dataset Fragile_raw
+ return {
+ "row_id": $t.row_id % 28000,
+ "sid": $t.sid,
+ "date": date($t.date),
+ "day": $t.day,
+ "time": parse-time($t.time, "h:m:s"),
+ "bpm": $t.bpm,
+ "RR": $t.RR,
+ "text": $t.text,
+ "location": $t.location,
+ "text2": $t.text2
+ }
+);
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_multiple_secondary_indices/primary_plus_multiple_secondary_indices.5.txnqbc.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_multiple_secondary_indices/primary_plus_multiple_secondary_indices.5.txnqbc.aql
new file mode 100644
index 0000000..97935fa
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_multiple_secondary_indices/primary_plus_multiple_secondary_indices.5.txnqbc.aql
@@ -0,0 +1,12 @@
+/*
+ * Test case Name : primary_plus_keyword_secondary_index.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : Otc 15 2013
+ */
+
+
+use dataverse recovery;
+
+count (for $x in dataset Fragile
+where contains($x.text2, "location") return $x);
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_multiple_secondary_indices/primary_plus_multiple_secondary_indices.6.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_multiple_secondary_indices/primary_plus_multiple_secondary_indices.6.script.aql
new file mode 100644
index 0000000..31d37ae
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_multiple_secondary_indices/primary_plus_multiple_secondary_indices.6.script.aql
@@ -0,0 +1 @@
+kill_cc_and_nc.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_multiple_secondary_indices/primary_plus_multiple_secondary_indices.7.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_multiple_secondary_indices/primary_plus_multiple_secondary_indices.7.script.aql
new file mode 100644
index 0000000..37ef6c0
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_multiple_secondary_indices/primary_plus_multiple_secondary_indices.7.script.aql
@@ -0,0 +1 @@
+stop_and_start.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_multiple_secondary_indices/primary_plus_multiple_secondary_indices.8.txnqar.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_multiple_secondary_indices/primary_plus_multiple_secondary_indices.8.txnqar.aql
new file mode 100644
index 0000000..5e18be6
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_multiple_secondary_indices/primary_plus_multiple_secondary_indices.8.txnqar.aql
@@ -0,0 +1,5 @@
+
+use dataverse recovery;
+
+count (for $x in dataset Fragile
+where contains($x.text2, "location") return $x);
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_multiple_secondary_indices/primary_plus_multiple_secondary_indices.9.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_multiple_secondary_indices/primary_plus_multiple_secondary_indices.9.script.aql
new file mode 100644
index 0000000..f75dfc9
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_multiple_secondary_indices/primary_plus_multiple_secondary_indices.9.script.aql
@@ -0,0 +1 @@
+stop_and_delete.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_ngram_index/primary_plus_ngram_index.1.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_ngram_index/primary_plus_ngram_index.1.script.aql
new file mode 100644
index 0000000..323b1cf
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_ngram_index/primary_plus_ngram_index.1.script.aql
@@ -0,0 +1 @@
+create_and_start.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_ngram_index/primary_plus_ngram_index.2.ddl.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_ngram_index/primary_plus_ngram_index.2.ddl.aql
new file mode 100644
index 0000000..7182466
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_ngram_index/primary_plus_ngram_index.2.ddl.aql
@@ -0,0 +1,53 @@
+/*
+ * Test case Name : primary_plus_default_secondary_index.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : Oct 15 2013
+ */
+
+drop dataverse recovery if exists;
+create dataverse recovery;
+use dataverse recovery;
+
+/* For raw Fragile data */
+create type FragileTypeRaw as closed {
+ row_id: int32,
+ sid: int32,
+ date: string,
+ day: int32,
+ time: string,
+ bpm: int32,
+ RR: float,
+ /* new string field and location field*/
+ text: string,
+ location: point,
+ text2: string
+};
+
+/* For cleaned Fragile data */
+create type FragileType as closed {
+ row_id: int32,
+ sid: int32,
+ date: date,
+ day: int32,
+ time: time,
+ bpm: int32,
+ RR: float,
+
+ /* new string field and location field*/
+ text: string,
+ location: point,
+ text2: string
+};
+
+/* Create dataset for loading raw Fragile data */
+create temporary dataset Fragile_raw (FragileTypeRaw)
+primary key row_id;
+
+/* Create dataset for cleaned Fragile data */
+create temporary dataset Fragile (FragileType)
+primary key row_id;
+
+/* Create default secondary index on dataset clean Fragile */
+create index cfText2Ix on Fragile(text2) type ngram(3);
+
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_ngram_index/primary_plus_ngram_index.3.update.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_ngram_index/primary_plus_ngram_index.3.update.aql
new file mode 100644
index 0000000..5d7d6f2
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_ngram_index/primary_plus_ngram_index.3.update.aql
@@ -0,0 +1,11 @@
+/*
+ * Test case Name : primary_plus_default_secondary_index.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : Oct 15 2013
+ */
+
+use dataverse recovery;
+
+load dataset Fragile_raw using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="127.0.0.1://../../../../../../asterix-app/data/csv/fragile_02.adm"),("format"="adm")) pre-sorted;
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_ngram_index/primary_plus_ngram_index.4.txneu.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_ngram_index/primary_plus_ngram_index.4.txneu.aql
new file mode 100644
index 0000000..12fd699
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_ngram_index/primary_plus_ngram_index.4.txneu.aql
@@ -0,0 +1,25 @@
+/*
+ * Test case Name : primary_plus_default_secondary_index.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : Oct 15 2013
+ */
+
+use dataverse recovery;
+
+/* Load Fragile data from raw dataset into cleaned dataset */
+insert into dataset Fragile (
+ for $t in dataset Fragile_raw
+ return {
+ "row_id": $t.row_id % 28000,
+ "sid": $t.sid,
+ "date": date($t.date),
+ "day": $t.day,
+ "time": parse-time($t.time, "h:m:s"),
+ "bpm": $t.bpm,
+ "RR": $t.RR,
+ "text": $t.text,
+ "location": $t.location,
+ "text2": $t.text2
+ }
+);
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_ngram_index/primary_plus_ngram_index.5.txnqbc.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_ngram_index/primary_plus_ngram_index.5.txnqbc.aql
new file mode 100644
index 0000000..434e8b0
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_ngram_index/primary_plus_ngram_index.5.txnqbc.aql
@@ -0,0 +1,11 @@
+/*
+ * Test case Name : primary_plus_default_secondary_index.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : Otc 15 2013
+ */
+
+use dataverse recovery;
+
+count (for $x in dataset Fragile
+where contains($x.text2, "location") return $x);
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_ngram_index/primary_plus_ngram_index.6.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_ngram_index/primary_plus_ngram_index.6.script.aql
new file mode 100644
index 0000000..31d37ae
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_ngram_index/primary_plus_ngram_index.6.script.aql
@@ -0,0 +1 @@
+kill_cc_and_nc.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_ngram_index/primary_plus_ngram_index.7.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_ngram_index/primary_plus_ngram_index.7.script.aql
new file mode 100644
index 0000000..37ef6c0
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_ngram_index/primary_plus_ngram_index.7.script.aql
@@ -0,0 +1 @@
+stop_and_start.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_ngram_index/primary_plus_ngram_index.8.txnqar.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_ngram_index/primary_plus_ngram_index.8.txnqar.aql
new file mode 100644
index 0000000..21a5d28
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_ngram_index/primary_plus_ngram_index.8.txnqar.aql
@@ -0,0 +1,11 @@
+/*
+ * Test case Name : primary_plus_default_secondary_index.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : Oct 15 2013
+ */
+
+use dataverse recovery;
+
+count (for $x in dataset Fragile
+where contains($x.text2, "location") return $x);
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_ngram_index/primary_plus_ngram_index.9.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_ngram_index/primary_plus_ngram_index.9.script.aql
new file mode 100644
index 0000000..f75dfc9
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_ngram_index/primary_plus_ngram_index.9.script.aql
@@ -0,0 +1 @@
+stop_and_delete.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_rtree_index/primary_plus_rtree_index.1.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_rtree_index/primary_plus_rtree_index.1.script.aql
new file mode 100644
index 0000000..323b1cf
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_rtree_index/primary_plus_rtree_index.1.script.aql
@@ -0,0 +1 @@
+create_and_start.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_rtree_index/primary_plus_rtree_index.2.ddl.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_rtree_index/primary_plus_rtree_index.2.ddl.aql
new file mode 100644
index 0000000..21c17bc
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_rtree_index/primary_plus_rtree_index.2.ddl.aql
@@ -0,0 +1,54 @@
+/*
+ * Test case Name : primary_plus_keyword_secondary_index.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : Oct 15 2013
+ */
+
+drop dataverse recovery if exists;
+create dataverse recovery;
+use dataverse recovery;
+
+/* For raw Fragile data */
+create type FragileTypeRaw as closed {
+ row_id: int32,
+ sid: int32,
+ date: string,
+ day: int32,
+ time: string,
+ bpm: int32,
+ RR: float,
+ /* new string field and location field*/
+ text: string,
+ location: point,
+ text2: string
+
+};
+
+/* For cleaned Fragile data */
+create type FragileType as closed {
+ row_id: int32,
+ sid: int32,
+ date: date,
+ day: int32,
+ time: time,
+ bpm: int32,
+ RR: float,
+
+ /* new string field and location field*/
+ text: string,
+ location: point,
+ text2: string
+};
+
+/* Create dataset for loading raw Fragile data */
+create temporary dataset Fragile_raw (FragileTypeRaw)
+primary key row_id;
+
+/* Create dataset for cleaned Fragile data */
+create temporary dataset Fragile (FragileType)
+primary key row_id;
+
+
+/* Create rtree secondary index on dataset clean Fragile */
+create index cfLocation on Fragile(location) type rtree;
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_rtree_index/primary_plus_rtree_index.3.update.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_rtree_index/primary_plus_rtree_index.3.update.aql
new file mode 100644
index 0000000..9d9757f
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_rtree_index/primary_plus_rtree_index.3.update.aql
@@ -0,0 +1,12 @@
+/*
+ * Test case Name : primary_plus_keyword_secondary_index.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : Oct 15 2013
+ */
+
+use dataverse recovery;
+
+
+load dataset Fragile_raw using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="127.0.0.1://../../../../../../asterix-app/data/csv/fragile_02.adm"),("format"="adm")) pre-sorted;
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_rtree_index/primary_plus_rtree_index.4.txneu.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_rtree_index/primary_plus_rtree_index.4.txneu.aql
new file mode 100644
index 0000000..3c38973
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_rtree_index/primary_plus_rtree_index.4.txneu.aql
@@ -0,0 +1,25 @@
+/*
+ * Test case Name : primary_plus_keyword_secondary_index.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : Oct 15 2013
+ */
+
+use dataverse recovery;
+
+/* Load Fragile data from raw dataset into cleaned dataset */
+insert into dataset Fragile (
+ for $t in dataset Fragile_raw
+ return {
+ "row_id": $t.row_id % 28000,
+ "sid": $t.sid,
+ "date": date($t.date),
+ "day": $t.day,
+ "time": parse-time($t.time, "h:m:s"),
+ "bpm": $t.bpm,
+ "RR": $t.RR,
+ "text": $t.text,
+ "location": $t.location,
+ "text2": $t.text2
+ }
+);
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_rtree_index/primary_plus_rtree_index.5.txnqbc.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_rtree_index/primary_plus_rtree_index.5.txnqbc.aql
new file mode 100644
index 0000000..9189b28
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_rtree_index/primary_plus_rtree_index.5.txnqbc.aql
@@ -0,0 +1,5 @@
+use dataverse recovery;
+
+count (for $x in dataset Fragile where
+spatial-intersect($x.location, create-polygon([0.0,0.0, 2.0,2.0, 0.0,2.0, 2.0,0.0]))
+return $x);
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_rtree_index/primary_plus_rtree_index.6.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_rtree_index/primary_plus_rtree_index.6.script.aql
new file mode 100644
index 0000000..31d37ae
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_rtree_index/primary_plus_rtree_index.6.script.aql
@@ -0,0 +1 @@
+kill_cc_and_nc.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_rtree_index/primary_plus_rtree_index.7.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_rtree_index/primary_plus_rtree_index.7.script.aql
new file mode 100644
index 0000000..37ef6c0
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_rtree_index/primary_plus_rtree_index.7.script.aql
@@ -0,0 +1 @@
+stop_and_start.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_rtree_index/primary_plus_rtree_index.8.txnqar.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_rtree_index/primary_plus_rtree_index.8.txnqar.aql
new file mode 100644
index 0000000..9189b28
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_rtree_index/primary_plus_rtree_index.8.txnqar.aql
@@ -0,0 +1,5 @@
+use dataverse recovery;
+
+count (for $x in dataset Fragile where
+spatial-intersect($x.location, create-polygon([0.0,0.0, 2.0,2.0, 0.0,2.0, 2.0,0.0]))
+return $x);
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_rtree_index/primary_plus_rtree_index.9.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_rtree_index/primary_plus_rtree_index.9.script.aql
new file mode 100644
index 0000000..f75dfc9
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/temp_primary_plus_rtree_index/primary_plus_rtree_index.9.script.aql
@@ -0,0 +1 @@
+stop_and_delete.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_dataset_recovery/dataset_recovery.1.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_dataset_recovery/dataset_recovery.1.script.aql
new file mode 100644
index 0000000..323b1cf
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_dataset_recovery/dataset_recovery.1.script.aql
@@ -0,0 +1 @@
+create_and_start.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_dataset_recovery/dataset_recovery.2.ddl.aql b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_dataset_recovery/dataset_recovery.2.ddl.aql
new file mode 100644
index 0000000..ab9b51e
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_dataset_recovery/dataset_recovery.2.ddl.aql
@@ -0,0 +1,19 @@
+/*
+ * Test case Name : dataverse_recovery
+ * Description :
+ * Expected Result :
+ * Date :
+ */
+
+drop dataverse SampleDV if exists;
+create dataverse SampleDV;
+
+use dataverse SampleDV;
+
+create type SampleType as open {
+ id: int32,
+ text: string
+}
+
+create temporary dataset SampleDS(SampleType)
+primary key id;
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_dataset_recovery/dataset_recovery.3.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_dataset_recovery/dataset_recovery.3.script.aql
new file mode 100644
index 0000000..31d37ae
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_dataset_recovery/dataset_recovery.3.script.aql
@@ -0,0 +1 @@
+kill_cc_and_nc.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_dataset_recovery/dataset_recovery.4.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_dataset_recovery/dataset_recovery.4.script.aql
new file mode 100644
index 0000000..3ba1dc0
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_dataset_recovery/dataset_recovery.4.script.aql
@@ -0,0 +1 @@
+stop_and_start.sh
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_dataset_recovery/dataset_recovery.5.errddl.aql b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_dataset_recovery/dataset_recovery.5.errddl.aql
new file mode 100644
index 0000000..da254c2
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_dataset_recovery/dataset_recovery.5.errddl.aql
@@ -0,0 +1,2 @@
+use dataverse SampleDV;
+drop dataset SampleDS;
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_dataset_recovery/dataset_recovery.6.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_dataset_recovery/dataset_recovery.6.script.aql
new file mode 100644
index 0000000..10e1a51
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_dataset_recovery/dataset_recovery.6.script.aql
@@ -0,0 +1 @@
+stop_and_delete.sh
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_delete_after_recovery/delete_after_recovery.1.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_delete_after_recovery/delete_after_recovery.1.script.aql
new file mode 100644
index 0000000..323b1cf
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_delete_after_recovery/delete_after_recovery.1.script.aql
@@ -0,0 +1 @@
+create_and_start.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_delete_after_recovery/delete_after_recovery.2.ddl.aql b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_delete_after_recovery/delete_after_recovery.2.ddl.aql
new file mode 100644
index 0000000..b7f6b19
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_delete_after_recovery/delete_after_recovery.2.ddl.aql
@@ -0,0 +1,41 @@
+/*
+ * Test case Name : primary_index_only.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : September 25 2013
+ */
+
+drop dataverse recovery if exists;
+create dataverse recovery;
+use dataverse recovery;
+
+/* For raw Fragile data */
+create type FragileTypeRaw as closed {
+ row_id: int32,
+ sid: int32,
+ date: string,
+ day: int32,
+ time: string,
+ bpm: int32,
+ RR: float
+};
+
+/* For cleaned Fragile data */
+create type FragileType as closed {
+ row_id: int32,
+ sid: int32,
+ date: date,
+ day: int32,
+ time: time,
+ bpm: int32,
+ RR: float
+};
+
+/* Create dataset for loading raw Fragile data */
+create temporary dataset Fragile_raw (FragileTypeRaw)
+primary key row_id;
+
+/* Create dataset for cleaned Fragile data */
+create temporary dataset Fragile (FragileType)
+primary key row_id;
+
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_delete_after_recovery/delete_after_recovery.3.update.aql b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_delete_after_recovery/delete_after_recovery.3.update.aql
new file mode 100644
index 0000000..6d0e431
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_delete_after_recovery/delete_after_recovery.3.update.aql
@@ -0,0 +1,11 @@
+/*
+ * Test case Name : primary_index_only.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : September 25 2013
+ */
+
+use dataverse recovery;
+
+load dataset Fragile_raw using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="127.0.0.1://../../../../../../asterix-app/data/csv/fragile_01.csv"),("format"="delimited-text"),("delimiter"=",")) pre-sorted;
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_delete_after_recovery/delete_after_recovery.4.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_delete_after_recovery/delete_after_recovery.4.script.aql
new file mode 100644
index 0000000..31d37ae
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_delete_after_recovery/delete_after_recovery.4.script.aql
@@ -0,0 +1 @@
+kill_cc_and_nc.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_delete_after_recovery/delete_after_recovery.5.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_delete_after_recovery/delete_after_recovery.5.script.aql
new file mode 100644
index 0000000..3ba1dc0
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_delete_after_recovery/delete_after_recovery.5.script.aql
@@ -0,0 +1 @@
+stop_and_start.sh
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_delete_after_recovery/delete_after_recovery.6.update.aql b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_delete_after_recovery/delete_after_recovery.6.update.aql
new file mode 100644
index 0000000..a5e94d2
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_delete_after_recovery/delete_after_recovery.6.update.aql
@@ -0,0 +1,5 @@
+use dataverse recovery;
+
+delete $r from dataset Fragile_raw
+ where $r.row_id % 2 = 1
+;
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_delete_after_recovery/delete_after_recovery.7.query.aql b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_delete_after_recovery/delete_after_recovery.7.query.aql
new file mode 100644
index 0000000..a63aee2
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_delete_after_recovery/delete_after_recovery.7.query.aql
@@ -0,0 +1,11 @@
+/*
+ * Test case Name : primary_index_only.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : September 25 2013
+ */
+
+use dataverse recovery;
+
+count (for $x in dataset Fragile_raw return $x);
+
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_delete_after_recovery/delete_after_recovery.8.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_delete_after_recovery/delete_after_recovery.8.script.aql
new file mode 100644
index 0000000..f75dfc9
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_delete_after_recovery/delete_after_recovery.8.script.aql
@@ -0,0 +1 @@
+stop_and_delete.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_insert_after_recovery/insert_after_recovery.1.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_insert_after_recovery/insert_after_recovery.1.script.aql
new file mode 100644
index 0000000..323b1cf
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_insert_after_recovery/insert_after_recovery.1.script.aql
@@ -0,0 +1 @@
+create_and_start.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_insert_after_recovery/insert_after_recovery.2.ddl.aql b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_insert_after_recovery/insert_after_recovery.2.ddl.aql
new file mode 100644
index 0000000..b7f6b19
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_insert_after_recovery/insert_after_recovery.2.ddl.aql
@@ -0,0 +1,41 @@
+/*
+ * Test case Name : primary_index_only.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : September 25 2013
+ */
+
+drop dataverse recovery if exists;
+create dataverse recovery;
+use dataverse recovery;
+
+/* For raw Fragile data */
+create type FragileTypeRaw as closed {
+ row_id: int32,
+ sid: int32,
+ date: string,
+ day: int32,
+ time: string,
+ bpm: int32,
+ RR: float
+};
+
+/* For cleaned Fragile data */
+create type FragileType as closed {
+ row_id: int32,
+ sid: int32,
+ date: date,
+ day: int32,
+ time: time,
+ bpm: int32,
+ RR: float
+};
+
+/* Create dataset for loading raw Fragile data */
+create temporary dataset Fragile_raw (FragileTypeRaw)
+primary key row_id;
+
+/* Create dataset for cleaned Fragile data */
+create temporary dataset Fragile (FragileType)
+primary key row_id;
+
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_insert_after_recovery/insert_after_recovery.3.update.aql b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_insert_after_recovery/insert_after_recovery.3.update.aql
new file mode 100644
index 0000000..6d0e431
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_insert_after_recovery/insert_after_recovery.3.update.aql
@@ -0,0 +1,11 @@
+/*
+ * Test case Name : primary_index_only.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : September 25 2013
+ */
+
+use dataverse recovery;
+
+load dataset Fragile_raw using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="127.0.0.1://../../../../../../asterix-app/data/csv/fragile_01.csv"),("format"="delimited-text"),("delimiter"=",")) pre-sorted;
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_insert_after_recovery/insert_after_recovery.4.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_insert_after_recovery/insert_after_recovery.4.script.aql
new file mode 100644
index 0000000..31d37ae
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_insert_after_recovery/insert_after_recovery.4.script.aql
@@ -0,0 +1 @@
+kill_cc_and_nc.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_insert_after_recovery/insert_after_recovery.5.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_insert_after_recovery/insert_after_recovery.5.script.aql
new file mode 100644
index 0000000..3ba1dc0
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_insert_after_recovery/insert_after_recovery.5.script.aql
@@ -0,0 +1 @@
+stop_and_start.sh
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_insert_after_recovery/insert_after_recovery.6.update.aql b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_insert_after_recovery/insert_after_recovery.6.update.aql
new file mode 100644
index 0000000..26d998d
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_insert_after_recovery/insert_after_recovery.6.update.aql
@@ -0,0 +1,15 @@
+use dataverse recovery;
+
+/* Load Fragile data from raw dataset into cleaned dataset */
+insert into dataset Fragile (
+ for $t in dataset Fragile_raw
+ return {
+ "row_id": $t.row_id,
+ "sid": $t.sid,
+ "date": date($t.date),
+ "day": $t.day,
+ "time": parse-time($t.time, "h:m:s"),
+ "bpm": $t.bpm,
+ "RR": $t.RR
+ }
+);
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_insert_after_recovery/insert_after_recovery.7.query.aql b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_insert_after_recovery/insert_after_recovery.7.query.aql
new file mode 100644
index 0000000..f8ed3d0
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_insert_after_recovery/insert_after_recovery.7.query.aql
@@ -0,0 +1,11 @@
+/*
+ * Test case Name : primary_index_only.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : September 25 2013
+ */
+
+use dataverse recovery;
+
+count (for $x in dataset Fragile return $x);
+
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_insert_after_recovery/insert_after_recovery.8.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_insert_after_recovery/insert_after_recovery.8.script.aql
new file mode 100644
index 0000000..f75dfc9
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_insert_after_recovery/insert_after_recovery.8.script.aql
@@ -0,0 +1 @@
+stop_and_delete.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_load_after_recovery/load_after_recovery.1.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_load_after_recovery/load_after_recovery.1.script.aql
new file mode 100644
index 0000000..323b1cf
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_load_after_recovery/load_after_recovery.1.script.aql
@@ -0,0 +1 @@
+create_and_start.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_load_after_recovery/load_after_recovery.2.ddl.aql b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_load_after_recovery/load_after_recovery.2.ddl.aql
new file mode 100644
index 0000000..b7f6b19
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_load_after_recovery/load_after_recovery.2.ddl.aql
@@ -0,0 +1,41 @@
+/*
+ * Test case Name : primary_index_only.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : September 25 2013
+ */
+
+drop dataverse recovery if exists;
+create dataverse recovery;
+use dataverse recovery;
+
+/* For raw Fragile data */
+create type FragileTypeRaw as closed {
+ row_id: int32,
+ sid: int32,
+ date: string,
+ day: int32,
+ time: string,
+ bpm: int32,
+ RR: float
+};
+
+/* For cleaned Fragile data */
+create type FragileType as closed {
+ row_id: int32,
+ sid: int32,
+ date: date,
+ day: int32,
+ time: time,
+ bpm: int32,
+ RR: float
+};
+
+/* Create dataset for loading raw Fragile data */
+create temporary dataset Fragile_raw (FragileTypeRaw)
+primary key row_id;
+
+/* Create dataset for cleaned Fragile data */
+create temporary dataset Fragile (FragileType)
+primary key row_id;
+
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_load_after_recovery/load_after_recovery.3.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_load_after_recovery/load_after_recovery.3.script.aql
new file mode 100644
index 0000000..31d37ae
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_load_after_recovery/load_after_recovery.3.script.aql
@@ -0,0 +1 @@
+kill_cc_and_nc.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_load_after_recovery/load_after_recovery.4.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_load_after_recovery/load_after_recovery.4.script.aql
new file mode 100644
index 0000000..37ef6c0
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_load_after_recovery/load_after_recovery.4.script.aql
@@ -0,0 +1 @@
+stop_and_start.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_load_after_recovery/load_after_recovery.5.update.aql b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_load_after_recovery/load_after_recovery.5.update.aql
new file mode 100644
index 0000000..6d0e431
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_load_after_recovery/load_after_recovery.5.update.aql
@@ -0,0 +1,11 @@
+/*
+ * Test case Name : primary_index_only.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : September 25 2013
+ */
+
+use dataverse recovery;
+
+load dataset Fragile_raw using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="127.0.0.1://../../../../../../asterix-app/data/csv/fragile_01.csv"),("format"="delimited-text"),("delimiter"=",")) pre-sorted;
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_load_after_recovery/load_after_recovery.6.query.aql b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_load_after_recovery/load_after_recovery.6.query.aql
new file mode 100644
index 0000000..fcd9e05
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_load_after_recovery/load_after_recovery.6.query.aql
@@ -0,0 +1,10 @@
+/*
+ * Test case Name : primary_index_only.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : September 25 2013
+ */
+
+use dataverse recovery;
+
+count (for $x in dataset Fragile_raw return $x);
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_load_after_recovery/load_after_recovery.7.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_load_after_recovery/load_after_recovery.7.script.aql
new file mode 100644
index 0000000..f75dfc9
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_load_after_recovery/load_after_recovery.7.script.aql
@@ -0,0 +1 @@
+stop_and_delete.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_secondary_index_recovery/secondary_index_recovery.1.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_secondary_index_recovery/secondary_index_recovery.1.script.aql
new file mode 100644
index 0000000..323b1cf
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_secondary_index_recovery/secondary_index_recovery.1.script.aql
@@ -0,0 +1 @@
+create_and_start.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_secondary_index_recovery/secondary_index_recovery.2.ddl.aql b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_secondary_index_recovery/secondary_index_recovery.2.ddl.aql
new file mode 100644
index 0000000..28f3cf8
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_secondary_index_recovery/secondary_index_recovery.2.ddl.aql
@@ -0,0 +1,21 @@
+/*
+ * Test case Name : dataverse_recovery
+ * Description :
+ * Expected Result :
+ * Date :
+ */
+
+drop dataverse SampleDV if exists;
+create dataverse SampleDV;
+
+use dataverse SampleDV;
+
+create type SampleType as open {
+ id: int32,
+ text: string
+}
+
+create temporary dataset SampleDS(SampleType)
+primary key id;
+
+create index SampleDSix on SampleDS(text);
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_secondary_index_recovery/secondary_index_recovery.3.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_secondary_index_recovery/secondary_index_recovery.3.script.aql
new file mode 100644
index 0000000..31d37ae
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_secondary_index_recovery/secondary_index_recovery.3.script.aql
@@ -0,0 +1 @@
+kill_cc_and_nc.sh
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_secondary_index_recovery/secondary_index_recovery.4.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_secondary_index_recovery/secondary_index_recovery.4.script.aql
new file mode 100644
index 0000000..3ba1dc0
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_secondary_index_recovery/secondary_index_recovery.4.script.aql
@@ -0,0 +1 @@
+stop_and_start.sh
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_secondary_index_recovery/secondary_index_recovery.5.errddl.aql b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_secondary_index_recovery/secondary_index_recovery.5.errddl.aql
new file mode 100644
index 0000000..d765b16
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_secondary_index_recovery/secondary_index_recovery.5.errddl.aql
@@ -0,0 +1,2 @@
+use dataverse SampleDV;
+drop index SampleDS.SampleDSix;
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_secondary_index_recovery/secondary_index_recovery.6.script.aql b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_secondary_index_recovery/secondary_index_recovery.6.script.aql
new file mode 100644
index 0000000..10e1a51
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/queries/recovery_ddl/temp_secondary_index_recovery/secondary_index_recovery.6.script.aql
@@ -0,0 +1 @@
+stop_and_delete.sh
diff --git a/asterix-installer/src/test/resources/transactionts/results/recovery_ddl/temp_delete_after_recovery/delete_after_recovery.1.adm b/asterix-installer/src/test/resources/transactionts/results/recovery_ddl/temp_delete_after_recovery/delete_after_recovery.1.adm
new file mode 100644
index 0000000..5c1b51d
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/results/recovery_ddl/temp_delete_after_recovery/delete_after_recovery.1.adm
@@ -0,0 +1,2 @@
+[ 129088
+ ]
diff --git a/asterix-installer/src/test/resources/transactionts/results/recovery_ddl/temp_insert_after_recovery/insert_after_recovery.1.adm b/asterix-installer/src/test/resources/transactionts/results/recovery_ddl/temp_insert_after_recovery/insert_after_recovery.1.adm
new file mode 100644
index 0000000..de0b6c2
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/results/recovery_ddl/temp_insert_after_recovery/insert_after_recovery.1.adm
@@ -0,0 +1,2 @@
+[ 258176
+ ]
diff --git a/asterix-installer/src/test/resources/transactionts/results/recovery_ddl/temp_load_after_recovery/load_after_recovery.1.adm b/asterix-installer/src/test/resources/transactionts/results/recovery_ddl/temp_load_after_recovery/load_after_recovery.1.adm
new file mode 100644
index 0000000..de0b6c2
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/results/recovery_ddl/temp_load_after_recovery/load_after_recovery.1.adm
@@ -0,0 +1,2 @@
+[ 258176
+ ]
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_index_only/create_and_start.sh b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_index_only/create_and_start.sh
new file mode 100755
index 0000000..945f01d
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_index_only/create_and_start.sh
@@ -0,0 +1 @@
+$MANAGIX_HOME/bin/managix create -n nc1 -c $MANAGIX_HOME/clusters/local/local.xml;
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_index_only/kill_cc_and_nc.sh b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_index_only/kill_cc_and_nc.sh
new file mode 100755
index 0000000..096d7df
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_index_only/kill_cc_and_nc.sh
@@ -0,0 +1 @@
+jps | awk '{if ($2 == "NCDriver" || $2 == "CCDriver") print $1;}' | xargs -n 1 kill -9
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_index_only/stop_and_delete.sh b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_index_only/stop_and_delete.sh
new file mode 100755
index 0000000..d7deea3
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_index_only/stop_and_delete.sh
@@ -0,0 +1,3 @@
+$MANAGIX_HOME/bin/managix stop -n nc1;
+$MANAGIX_HOME/bin/managix delete -n nc1;
+
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_index_only/stop_and_start.sh b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_index_only/stop_and_start.sh
new file mode 100755
index 0000000..7855938
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_index_only/stop_and_start.sh
@@ -0,0 +1,3 @@
+$MANAGIX_HOME/bin/managix stop -n nc1;
+$MANAGIX_HOME/bin/managix start -n nc1;
+
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_default_secondary_index/create_and_start.sh b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_default_secondary_index/create_and_start.sh
new file mode 100755
index 0000000..945f01d
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_default_secondary_index/create_and_start.sh
@@ -0,0 +1 @@
+$MANAGIX_HOME/bin/managix create -n nc1 -c $MANAGIX_HOME/clusters/local/local.xml;
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_default_secondary_index/kill_cc_and_nc.sh b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_default_secondary_index/kill_cc_and_nc.sh
new file mode 100755
index 0000000..096d7df
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_default_secondary_index/kill_cc_and_nc.sh
@@ -0,0 +1 @@
+jps | awk '{if ($2 == "NCDriver" || $2 == "CCDriver") print $1;}' | xargs -n 1 kill -9
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_default_secondary_index/stop_and_delete.sh b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_default_secondary_index/stop_and_delete.sh
new file mode 100755
index 0000000..d7deea3
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_default_secondary_index/stop_and_delete.sh
@@ -0,0 +1,3 @@
+$MANAGIX_HOME/bin/managix stop -n nc1;
+$MANAGIX_HOME/bin/managix delete -n nc1;
+
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_default_secondary_index/stop_and_start.sh b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_default_secondary_index/stop_and_start.sh
new file mode 100755
index 0000000..7855938
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_default_secondary_index/stop_and_start.sh
@@ -0,0 +1,3 @@
+$MANAGIX_HOME/bin/managix stop -n nc1;
+$MANAGIX_HOME/bin/managix start -n nc1;
+
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_keyword_secondary_index/create_and_start.sh b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_keyword_secondary_index/create_and_start.sh
new file mode 100755
index 0000000..945f01d
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_keyword_secondary_index/create_and_start.sh
@@ -0,0 +1 @@
+$MANAGIX_HOME/bin/managix create -n nc1 -c $MANAGIX_HOME/clusters/local/local.xml;
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_keyword_secondary_index/kill_cc_and_nc.sh b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_keyword_secondary_index/kill_cc_and_nc.sh
new file mode 100755
index 0000000..096d7df
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_keyword_secondary_index/kill_cc_and_nc.sh
@@ -0,0 +1 @@
+jps | awk '{if ($2 == "NCDriver" || $2 == "CCDriver") print $1;}' | xargs -n 1 kill -9
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_keyword_secondary_index/stop_and_delete.sh b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_keyword_secondary_index/stop_and_delete.sh
new file mode 100755
index 0000000..d7deea3
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_keyword_secondary_index/stop_and_delete.sh
@@ -0,0 +1,3 @@
+$MANAGIX_HOME/bin/managix stop -n nc1;
+$MANAGIX_HOME/bin/managix delete -n nc1;
+
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_keyword_secondary_index/stop_and_start.sh b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_keyword_secondary_index/stop_and_start.sh
new file mode 100755
index 0000000..7855938
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_keyword_secondary_index/stop_and_start.sh
@@ -0,0 +1,3 @@
+$MANAGIX_HOME/bin/managix stop -n nc1;
+$MANAGIX_HOME/bin/managix start -n nc1;
+
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_multiple_secondary_indices/create_and_start.sh b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_multiple_secondary_indices/create_and_start.sh
new file mode 100755
index 0000000..945f01d
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_multiple_secondary_indices/create_and_start.sh
@@ -0,0 +1 @@
+$MANAGIX_HOME/bin/managix create -n nc1 -c $MANAGIX_HOME/clusters/local/local.xml;
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_multiple_secondary_indices/kill_cc_and_nc.sh b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_multiple_secondary_indices/kill_cc_and_nc.sh
new file mode 100755
index 0000000..096d7df
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_multiple_secondary_indices/kill_cc_and_nc.sh
@@ -0,0 +1 @@
+jps | awk '{if ($2 == "NCDriver" || $2 == "CCDriver") print $1;}' | xargs -n 1 kill -9
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_multiple_secondary_indices/stop_and_delete.sh b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_multiple_secondary_indices/stop_and_delete.sh
new file mode 100755
index 0000000..d7deea3
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_multiple_secondary_indices/stop_and_delete.sh
@@ -0,0 +1,3 @@
+$MANAGIX_HOME/bin/managix stop -n nc1;
+$MANAGIX_HOME/bin/managix delete -n nc1;
+
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_multiple_secondary_indices/stop_and_start.sh b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_multiple_secondary_indices/stop_and_start.sh
new file mode 100755
index 0000000..7855938
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_multiple_secondary_indices/stop_and_start.sh
@@ -0,0 +1,3 @@
+$MANAGIX_HOME/bin/managix stop -n nc1;
+$MANAGIX_HOME/bin/managix start -n nc1;
+
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_ngram_index/create_and_start.sh b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_ngram_index/create_and_start.sh
new file mode 100755
index 0000000..945f01d
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_ngram_index/create_and_start.sh
@@ -0,0 +1 @@
+$MANAGIX_HOME/bin/managix create -n nc1 -c $MANAGIX_HOME/clusters/local/local.xml;
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_ngram_index/kill_cc_and_nc.sh b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_ngram_index/kill_cc_and_nc.sh
new file mode 100755
index 0000000..096d7df
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_ngram_index/kill_cc_and_nc.sh
@@ -0,0 +1 @@
+jps | awk '{if ($2 == "NCDriver" || $2 == "CCDriver") print $1;}' | xargs -n 1 kill -9
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_ngram_index/stop_and_delete.sh b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_ngram_index/stop_and_delete.sh
new file mode 100755
index 0000000..d7deea3
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_ngram_index/stop_and_delete.sh
@@ -0,0 +1,3 @@
+$MANAGIX_HOME/bin/managix stop -n nc1;
+$MANAGIX_HOME/bin/managix delete -n nc1;
+
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_ngram_index/stop_and_start.sh b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_ngram_index/stop_and_start.sh
new file mode 100755
index 0000000..7855938
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_ngram_index/stop_and_start.sh
@@ -0,0 +1,3 @@
+$MANAGIX_HOME/bin/managix stop -n nc1;
+$MANAGIX_HOME/bin/managix start -n nc1;
+
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_rtree_index/create_and_start.sh b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_rtree_index/create_and_start.sh
new file mode 100755
index 0000000..945f01d
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_rtree_index/create_and_start.sh
@@ -0,0 +1 @@
+$MANAGIX_HOME/bin/managix create -n nc1 -c $MANAGIX_HOME/clusters/local/local.xml;
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_rtree_index/kill_cc_and_nc.sh b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_rtree_index/kill_cc_and_nc.sh
new file mode 100755
index 0000000..096d7df
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_rtree_index/kill_cc_and_nc.sh
@@ -0,0 +1 @@
+jps | awk '{if ($2 == "NCDriver" || $2 == "CCDriver") print $1;}' | xargs -n 1 kill -9
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_rtree_index/stop_and_delete.sh b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_rtree_index/stop_and_delete.sh
new file mode 100755
index 0000000..d7deea3
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_rtree_index/stop_and_delete.sh
@@ -0,0 +1,3 @@
+$MANAGIX_HOME/bin/managix stop -n nc1;
+$MANAGIX_HOME/bin/managix delete -n nc1;
+
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_rtree_index/stop_and_start.sh b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_rtree_index/stop_and_start.sh
new file mode 100755
index 0000000..7855938
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/temp_primary_plus_rtree_index/stop_and_start.sh
@@ -0,0 +1,3 @@
+$MANAGIX_HOME/bin/managix stop -n nc1;
+$MANAGIX_HOME/bin/managix start -n nc1;
+
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_dataset_recovery/create_and_start.sh b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_dataset_recovery/create_and_start.sh
new file mode 100755
index 0000000..945f01d
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_dataset_recovery/create_and_start.sh
@@ -0,0 +1 @@
+$MANAGIX_HOME/bin/managix create -n nc1 -c $MANAGIX_HOME/clusters/local/local.xml;
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_dataset_recovery/kill_cc_and_nc.sh b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_dataset_recovery/kill_cc_and_nc.sh
new file mode 100755
index 0000000..096d7df
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_dataset_recovery/kill_cc_and_nc.sh
@@ -0,0 +1 @@
+jps | awk '{if ($2 == "NCDriver" || $2 == "CCDriver") print $1;}' | xargs -n 1 kill -9
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_dataset_recovery/stop_and_delete.sh b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_dataset_recovery/stop_and_delete.sh
new file mode 100755
index 0000000..d7deea3
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_dataset_recovery/stop_and_delete.sh
@@ -0,0 +1,3 @@
+$MANAGIX_HOME/bin/managix stop -n nc1;
+$MANAGIX_HOME/bin/managix delete -n nc1;
+
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_dataset_recovery/stop_and_start.sh b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_dataset_recovery/stop_and_start.sh
new file mode 100755
index 0000000..7855938
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_dataset_recovery/stop_and_start.sh
@@ -0,0 +1,3 @@
+$MANAGIX_HOME/bin/managix stop -n nc1;
+$MANAGIX_HOME/bin/managix start -n nc1;
+
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_delete_after_recovery/create_and_start.sh b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_delete_after_recovery/create_and_start.sh
new file mode 100755
index 0000000..945f01d
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_delete_after_recovery/create_and_start.sh
@@ -0,0 +1 @@
+$MANAGIX_HOME/bin/managix create -n nc1 -c $MANAGIX_HOME/clusters/local/local.xml;
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_delete_after_recovery/kill_cc_and_nc.sh b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_delete_after_recovery/kill_cc_and_nc.sh
new file mode 100755
index 0000000..096d7df
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_delete_after_recovery/kill_cc_and_nc.sh
@@ -0,0 +1 @@
+jps | awk '{if ($2 == "NCDriver" || $2 == "CCDriver") print $1;}' | xargs -n 1 kill -9
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_delete_after_recovery/stop_and_delete.sh b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_delete_after_recovery/stop_and_delete.sh
new file mode 100755
index 0000000..d7deea3
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_delete_after_recovery/stop_and_delete.sh
@@ -0,0 +1,3 @@
+$MANAGIX_HOME/bin/managix stop -n nc1;
+$MANAGIX_HOME/bin/managix delete -n nc1;
+
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_delete_after_recovery/stop_and_start.sh b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_delete_after_recovery/stop_and_start.sh
new file mode 100755
index 0000000..7855938
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_delete_after_recovery/stop_and_start.sh
@@ -0,0 +1,3 @@
+$MANAGIX_HOME/bin/managix stop -n nc1;
+$MANAGIX_HOME/bin/managix start -n nc1;
+
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_insert_after_recovery/create_and_start.sh b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_insert_after_recovery/create_and_start.sh
new file mode 100755
index 0000000..945f01d
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_insert_after_recovery/create_and_start.sh
@@ -0,0 +1 @@
+$MANAGIX_HOME/bin/managix create -n nc1 -c $MANAGIX_HOME/clusters/local/local.xml;
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_insert_after_recovery/kill_cc_and_nc.sh b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_insert_after_recovery/kill_cc_and_nc.sh
new file mode 100755
index 0000000..096d7df
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_insert_after_recovery/kill_cc_and_nc.sh
@@ -0,0 +1 @@
+jps | awk '{if ($2 == "NCDriver" || $2 == "CCDriver") print $1;}' | xargs -n 1 kill -9
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_insert_after_recovery/stop_and_delete.sh b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_insert_after_recovery/stop_and_delete.sh
new file mode 100755
index 0000000..d7deea3
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_insert_after_recovery/stop_and_delete.sh
@@ -0,0 +1,3 @@
+$MANAGIX_HOME/bin/managix stop -n nc1;
+$MANAGIX_HOME/bin/managix delete -n nc1;
+
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_insert_after_recovery/stop_and_start.sh b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_insert_after_recovery/stop_and_start.sh
new file mode 100755
index 0000000..7855938
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_insert_after_recovery/stop_and_start.sh
@@ -0,0 +1,3 @@
+$MANAGIX_HOME/bin/managix stop -n nc1;
+$MANAGIX_HOME/bin/managix start -n nc1;
+
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_load_after_recovery/create_and_start.sh b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_load_after_recovery/create_and_start.sh
new file mode 100755
index 0000000..945f01d
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_load_after_recovery/create_and_start.sh
@@ -0,0 +1 @@
+$MANAGIX_HOME/bin/managix create -n nc1 -c $MANAGIX_HOME/clusters/local/local.xml;
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_load_after_recovery/kill_cc_and_nc.sh b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_load_after_recovery/kill_cc_and_nc.sh
new file mode 100755
index 0000000..096d7df
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_load_after_recovery/kill_cc_and_nc.sh
@@ -0,0 +1 @@
+jps | awk '{if ($2 == "NCDriver" || $2 == "CCDriver") print $1;}' | xargs -n 1 kill -9
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_load_after_recovery/stop_and_delete.sh b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_load_after_recovery/stop_and_delete.sh
new file mode 100755
index 0000000..d7deea3
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_load_after_recovery/stop_and_delete.sh
@@ -0,0 +1,3 @@
+$MANAGIX_HOME/bin/managix stop -n nc1;
+$MANAGIX_HOME/bin/managix delete -n nc1;
+
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_load_after_recovery/stop_and_start.sh b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_load_after_recovery/stop_and_start.sh
new file mode 100755
index 0000000..7855938
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_load_after_recovery/stop_and_start.sh
@@ -0,0 +1,3 @@
+$MANAGIX_HOME/bin/managix stop -n nc1;
+$MANAGIX_HOME/bin/managix start -n nc1;
+
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_secondary_index_recovery/create_and_start.sh b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_secondary_index_recovery/create_and_start.sh
new file mode 100755
index 0000000..945f01d
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_secondary_index_recovery/create_and_start.sh
@@ -0,0 +1 @@
+$MANAGIX_HOME/bin/managix create -n nc1 -c $MANAGIX_HOME/clusters/local/local.xml;
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_secondary_index_recovery/kill_cc_and_nc.sh b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_secondary_index_recovery/kill_cc_and_nc.sh
new file mode 100755
index 0000000..096d7df
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_secondary_index_recovery/kill_cc_and_nc.sh
@@ -0,0 +1 @@
+jps | awk '{if ($2 == "NCDriver" || $2 == "CCDriver") print $1;}' | xargs -n 1 kill -9
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_secondary_index_recovery/stop_and_delete.sh b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_secondary_index_recovery/stop_and_delete.sh
new file mode 100755
index 0000000..d7deea3
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_secondary_index_recovery/stop_and_delete.sh
@@ -0,0 +1,3 @@
+$MANAGIX_HOME/bin/managix stop -n nc1;
+$MANAGIX_HOME/bin/managix delete -n nc1;
+
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_secondary_index_recovery/stop_and_start.sh b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_secondary_index_recovery/stop_and_start.sh
new file mode 100755
index 0000000..7855938
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/recovery_ddl/temp_secondary_index_recovery/stop_and_start.sh
@@ -0,0 +1,3 @@
+$MANAGIX_HOME/bin/managix stop -n nc1;
+$MANAGIX_HOME/bin/managix start -n nc1;
+
diff --git a/asterix-installer/src/test/resources/transactionts/testsuite.xml b/asterix-installer/src/test/resources/transactionts/testsuite.xml
index f2bd2b5..a757f5e 100644
--- a/asterix-installer/src/test/resources/transactionts/testsuite.xml
+++ b/asterix-installer/src/test/resources/transactionts/testsuite.xml
@@ -13,7 +13,7 @@
! limitations under the License.
!-->
<test-suite xmlns="urn:xml.testframework.asterix.ics.uci.edu" ResultOffsetPath="results" QueryOffsetPath="queries" QueryFileExtension=".aql">
-
+
<test-group name="recover_after_abort">
<test-case FilePath="recover_after_abort">
@@ -33,13 +33,13 @@
<output-dir compare="Text">primary_plus_rtree_index</output-dir>
</compilation-unit>
</test-case>
-
+
<test-case FilePath="recover_after_abort">
<compilation-unit name="primary_plus_keyword_secondary_index">
<output-dir compare="Text">primary_plus_keyword_secondary_index</output-dir>
</compilation-unit>
</test-case>
-
+
<test-case FilePath="recover_after_abort">
<compilation-unit name="primary_plus_ngram_index">
<output-dir compare="Text">primary_plus_ngram_index</output-dir>
@@ -52,10 +52,51 @@
</compilation-unit>
</test-case>
+ <test-case FilePath="recover_after_abort">
+ <compilation-unit name="temp_primary_index_only">
+ <output-dir compare="Text">primary_index_only</output-dir>
+ <expected-error>edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException</expected-error>
+ </compilation-unit>
+ </test-case>
+
+ <test-case FilePath="recover_after_abort">
+ <compilation-unit name="temp_primary_plus_default_secondary_index">
+ <output-dir compare="Text">primary_plus_default_secondary_index</output-dir>
+ <expected-error>edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException</expected-error>
+ </compilation-unit>
+ </test-case>
+
+ <test-case FilePath="recover_after_abort">
+ <compilation-unit name="temp_primary_plus_rtree_index">
+ <output-dir compare="Text">primary_plus_rtree_index</output-dir>
+ <expected-error>edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException</expected-error>
+ </compilation-unit>
+ </test-case>
+
+ <test-case FilePath="recover_after_abort">
+ <compilation-unit name="temp_primary_plus_keyword_secondary_index">
+ <output-dir compare="Text">primary_plus_keyword_secondary_index</output-dir>
+ <expected-error>edu.uci.ics.hyracks.api.exceptions.HyracksDataException</expected-error>
+ </compilation-unit>
+ </test-case>
+
+ <test-case FilePath="recover_after_abort">
+ <compilation-unit name="temp_primary_plus_ngram_index">
+ <output-dir compare="Text">primary_plus_ngram_index</output-dir>
+ <expected-error>edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException</expected-error>
+ </compilation-unit>
+ </test-case>
+
+ <test-case FilePath="recover_after_abort">
+ <compilation-unit name="primary_plus_multiple_secondary_indices">
+ <output-dir compare="Text">primary_plus_multiple_secondary_indices</output-dir>
+ <expected-error>edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException</expected-error>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="recovery_ddl">
-
+
<test-case FilePath="recovery_ddl">
<compilation-unit name="dataverse_recovery">
<output-dir compare="Text">dataverse_recovery</output-dir>
@@ -67,25 +108,25 @@
<output-dir compare="Text">datatype_recovery</output-dir>
</compilation-unit>
</test-case>
-
+
<test-case FilePath="recovery_ddl">
<compilation-unit name="dataset_recovery">
<output-dir compare="Text">dataset_recovery</output-dir>
</compilation-unit>
</test-case>
-
+
<test-case FilePath="recovery_ddl">
<compilation-unit name="secondary_index_recovery">
<output-dir compare="Text">secondary_index_recovery</output-dir>
</compilation-unit>
</test-case>
-
+
<test-case FilePath="recovery_ddl">
<compilation-unit name="load_after_recovery">
<output-dir compare="Text">load_after_recovery</output-dir>
</compilation-unit>
</test-case>
-
+
<test-case FilePath="recovery_ddl">
<compilation-unit name="insert_after_recovery">
<output-dir compare="Text">insert_after_recovery</output-dir>
@@ -97,14 +138,48 @@
<output-dir compare="Text">delete_after_recovery</output-dir>
</compilation-unit>
</test-case>
-
+
<test-case FilePath="recovery_ddl">
<compilation-unit name="function_recovery">
<output-dir compare="Text">function_recovery</output-dir>
</compilation-unit>
</test-case>
+
+ <test-case FilePath="recovery_ddl">
+ <compilation-unit name="temp_dataset_recovery">
+ <output-dir compare="Text">dataset_recovery</output-dir>
+ <expected-error>edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException</expected-error>
+ </compilation-unit>
+ </test-case>
+
+ <test-case FilePath="recovery_ddl">
+ <compilation-unit name="temp_delete_after_recovery">
+ <output-dir compare="Text">delete_after_recovery</output-dir>
+ <expected-error>edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException</expected-error>
+ </compilation-unit>
+ </test-case>
+
+ <test-case FilePath="recovery_ddl">
+ <compilation-unit name="temp_insert_after_recovery">
+ <output-dir compare="Text">insert_after_recovery</output-dir>
+ <expected-error>edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException</expected-error>
+ </compilation-unit>
+ </test-case>
+
+ <test-case FilePath="recovery_ddl">
+ <compilation-unit name="temp_load_after_recovery">
+ <output-dir compare="Text">load_after_recovery</output-dir>
+ <expected-error>edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException</expected-error>
+ </compilation-unit>
+ </test-case>
+
+ <test-case FilePath="recovery_ddl">
+ <compilation-unit name="temp_secondary_index_recovery">
+ <output-dir compare="Text">secondary_index_recovery</output-dir>
+ <expected-error>edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException</expected-error>
+ </compilation-unit>
+ </test-case>
</test-group>
-
</test-suite>
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/GarbageCollector.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/GarbageCollector.java
new file mode 100644
index 0000000..3f1cbca
--- /dev/null
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/GarbageCollector.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.metadata;
+
+/**
+ * Periodically recycle temporary datasets.
+ *
+ * @author yingyib
+ */
+public class GarbageCollector implements Runnable {
+
+ private static long CLEANUP_PERIOD = 3600 * 24;
+
+ @Override
+ public void run() {
+ try {
+ synchronized (this) {
+ this.wait(CLEANUP_PERIOD);
+ }
+ MetadataManager.INSTANCE.cleanupTempDatasets();
+ } catch (Exception e) {
+ // Prints the stack trace to log.
+ e.printStackTrace();
+ }
+ }
+
+}
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/IDatasetDetails.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/IDatasetDetails.java
index 3ed94de..17c0302 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/IDatasetDetails.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/IDatasetDetails.java
@@ -24,8 +24,33 @@
public interface IDatasetDetails extends Serializable {
public DatasetType getDatasetType();
+
public String getNodeGroupName();
+
public void writeDatasetDetailsRecordType(DataOutput out) throws HyracksDataException;
+
public String getCompactionPolicy();
+
public Map<String, String> getCompactionPolicyProperties();
+
+ /**
+ * @return if the dataset is a temporary dataset.
+ * Here is a summary of temporary datasets:
+ * 1. Different from a persistent dataset, reads and writes over a temporary dataset do not require any lock.
+ * Writes over a temporary dataset do not generate any write-ahead update and commit log but generate
+ * flush log and job commit log.
+ * 2. A temporary dataset can only be an internal dataset, stored in partitioned LSM-Btrees.
+ * 3. All secondary indexes for persistent datasets are supported for temporary datasets.
+ * 4. A temporary dataset will be automatically garbage collected if it is not active in the past 30 days.
+ * A temporary dataset could be used for the following scenarios:
+ * 1. A data scientist wants to run some one-time data analysis queries over a dataset that s/he pre-processed
+ * and the dataset is only used by her/himself in an one-query-at-a-time manner.
+ * 2. Articulate AQL with external systems such as Pregelix/IMRU/Spark. A user can first run an AQL
+ * query to populate a temporary dataset, then kick off an external runtime to read this dataset,
+ * dump the results of the external runtime to yet-another-temporary dataset, and finally run yet-another AQL
+ * over the second temporary dataset.
+ */
+ public boolean isTemp();
+
+ public long getLastAccessTime();
}
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataCache.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataCache.java
index c36726b..6eb60f0 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataCache.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataCache.java
@@ -17,9 +17,12 @@
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
+import edu.uci.ics.asterix.common.config.DatasetConfig.IndexType;
import edu.uci.ics.asterix.common.feeds.FeedConnectionId;
import edu.uci.ics.asterix.common.functions.FunctionSignature;
import edu.uci.ics.asterix.metadata.api.IMetadataEntity;
@@ -33,6 +36,7 @@
import edu.uci.ics.asterix.metadata.entities.FeedPolicy;
import edu.uci.ics.asterix.metadata.entities.Function;
import edu.uci.ics.asterix.metadata.entities.Index;
+import edu.uci.ics.asterix.metadata.entities.InternalDatasetDetails;
import edu.uci.ics.asterix.metadata.entities.Library;
import edu.uci.ics.asterix.metadata.entities.NodeGroup;
@@ -44,6 +48,8 @@
*/
public class MetadataCache {
+ // Default life time period of a temp dataset. It is 30 days.
+ private final static long TEMP_DATASET_INACTIVE_TIME_THRESHOLD = 3600 * 24 * 30 * 1000;
// Key is dataverse name.
protected final Map<String, Dataverse> dataverses = new HashMap<String, Dataverse>();
// Key is dataverse name. Key of value map is dataset name.
@@ -147,34 +153,33 @@
public Object addDatasetIfNotExists(Dataset dataset) {
synchronized (datasets) {
- Map<String, Dataset> m = datasets.get(dataset.getDataverseName());
- if (m == null) {
- m = new HashMap<String, Dataset>();
- datasets.put(dataset.getDataverseName(), m);
+ synchronized (indexes) {
+ // Add the primary index associated with the dataset, if the dataset is an
+ // internal dataset.
+ if (dataset.getDatasetType() == DatasetType.INTERNAL) {
+ InternalDatasetDetails id = (InternalDatasetDetails) dataset.getDatasetDetails();
+ Index index = new Index(dataset.getDataverseName(), dataset.getDatasetName(),
+ dataset.getDatasetName(), IndexType.BTREE, id.getPartitioningKey(), id.getPrimaryKeyType(),
+ false, true, dataset.getPendingOp());
+ addIndexIfNotExistsInternal(index);
+ }
+
+ Map<String, Dataset> m = datasets.get(dataset.getDataverseName());
+ if (m == null) {
+ m = new HashMap<String, Dataset>();
+ datasets.put(dataset.getDataverseName(), m);
+ }
+ if (!m.containsKey(dataset.getDatasetName())) {
+ return m.put(dataset.getDatasetName(), dataset);
+ }
+ return null;
}
- if (!m.containsKey(dataset.getDatasetName())) {
- return m.put(dataset.getDatasetName(), dataset);
- }
- return null;
}
}
public Object addIndexIfNotExists(Index index) {
synchronized (indexes) {
- Map<String, Map<String, Index>> datasetMap = indexes.get(index.getDataverseName());
- if (datasetMap == null) {
- datasetMap = new HashMap<String, Map<String, Index>>();
- indexes.put(index.getDataverseName(), datasetMap);
- }
- Map<String, Index> indexMap = datasetMap.get(index.getDatasetName());
- if (indexMap == null) {
- indexMap = new HashMap<String, Index>();
- datasetMap.put(index.getDatasetName(), indexMap);
- }
- if (!indexMap.containsKey(index.getIndexName())) {
- return indexMap.put(index.getIndexName(), index);
- }
- return null;
+ return addIndexIfNotExistsInternal(index);
}
}
@@ -309,7 +314,6 @@
if (indexMap == null) {
return null;
}
-
return indexMap.remove(index.getIndexName());
}
}
@@ -396,6 +400,20 @@
}
}
+ public List<Index> getDatasetIndexes(String dataverseName, String datasetName) {
+ List<Index> retIndexes = new ArrayList<Index>();
+ synchronized (datasets) {
+ Map<String, Index> map = indexes.get(dataverseName).get(datasetName);
+ if (map == null) {
+ return retIndexes;
+ }
+ for (Map.Entry<String, Index> entry : map.entrySet()) {
+ retIndexes.add(entry.getValue());
+ }
+ return retIndexes;
+ }
+ }
+
/**
* Represents a logical operation against the metadata.
*/
@@ -560,4 +578,43 @@
return null;
}
}
+
+ private Object addIndexIfNotExistsInternal(Index index) {
+ Map<String, Map<String, Index>> datasetMap = indexes.get(index.getDataverseName());
+ if (datasetMap == null) {
+ datasetMap = new HashMap<String, Map<String, Index>>();
+ indexes.put(index.getDataverseName(), datasetMap);
+ }
+ Map<String, Index> indexMap = datasetMap.get(index.getDatasetName());
+ if (indexMap == null) {
+ indexMap = new HashMap<String, Index>();
+ datasetMap.put(index.getDatasetName(), indexMap);
+ }
+ if (!indexMap.containsKey(index.getIndexName())) {
+ return indexMap.put(index.getIndexName(), index);
+ }
+ return null;
+ }
+
+ /**
+ * Clean up temp datasets that are expired.
+ * The garbage collection will pause other dataset operations.
+ */
+ public void cleanupTempDatasets() {
+ synchronized (datasets) {
+ for (Map<String, Dataset> map : datasets.values()) {
+ Iterator<Dataset> datasetIterator = map.values().iterator();
+ while (datasetIterator.hasNext()) {
+ Dataset dataset = datasetIterator.next();
+ if (dataset.getDatasetDetails().isTemp()) {
+ long currentTime = System.currentTimeMillis();
+ long duration = currentTime - dataset.getDatasetDetails().getLastAccessTime();
+ if (duration > TEMP_DATASET_INACTIVE_TIME_THRESHOLD) {
+ datasetIterator.remove();
+ }
+ }
+ }
+ }
+ }
+ }
}
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataManager.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataManager.java
index a42eee7..d7e9bbb 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataManager.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataManager.java
@@ -16,6 +16,7 @@
package edu.uci.ics.asterix.metadata;
import java.rmi.RemoteException;
+import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -89,7 +90,7 @@
// Set in init().
public static MetadataManager INSTANCE;
private final MetadataCache cache = new MetadataCache();
- private IAsterixStateProxy proxy;
+ private final IAsterixStateProxy proxy;
private IMetadataNode metadataNode;
private final ReadWriteLock metadataLatch;
private final AsterixMetadataProperties metadataProperties;
@@ -141,6 +142,11 @@
+ metadataProperties.getMetadataNodeName());
}
}
+
+ // Starts the garbage collector thread which
+ // should always be running.
+ Thread garbageCollectorThread = new Thread(new GarbageCollector());
+ garbageCollectorThread.start();
}
@Override
@@ -236,11 +242,23 @@
@Override
public List<Dataset> getDataverseDatasets(MetadataTransactionContext ctx, String dataverseName)
throws MetadataException {
- List<Dataset> dataverseDatasets;
+ List<Dataset> dataverseDatasets = new ArrayList<Dataset>();
+ // add uncommitted temporary datasets
+ for (Dataset dataset : ctx.getDataverseDatasets(dataverseName)) {
+ if (dataset.getDatasetDetails().isTemp()) {
+ dataverseDatasets.add(dataset);
+ }
+ }
+ // add the committed temporary datasets with the cache
+ for (Dataset dataset : cache.getDataverseDatasets(dataverseName)) {
+ if (dataset.getDatasetDetails().isTemp()) {
+ dataverseDatasets.add(dataset);
+ }
+ }
try {
// Assuming that the transaction can read its own writes on the
// metadata node.
- dataverseDatasets = metadataNode.getDataverseDatasets(ctx.getJobId(), dataverseName);
+ dataverseDatasets.addAll(metadataNode.getDataverseDatasets(ctx.getJobId(), dataverseName));
} catch (RemoteException e) {
throw new MetadataException(e);
}
@@ -252,10 +270,12 @@
@Override
public void addDataset(MetadataTransactionContext ctx, Dataset dataset) throws MetadataException {
// add dataset into metadataNode
- try {
- metadataNode.addDataset(ctx.getJobId(), dataset);
- } catch (RemoteException e) {
- throw new MetadataException(e);
+ if (!dataset.getDatasetDetails().isTemp()) {
+ try {
+ metadataNode.addDataset(ctx.getJobId(), dataset);
+ } catch (RemoteException e) {
+ throw new MetadataException(e);
+ }
}
// reflect the dataset into the cache
@@ -265,11 +285,17 @@
@Override
public void dropDataset(MetadataTransactionContext ctx, String dataverseName, String datasetName)
throws MetadataException {
- try {
- metadataNode.dropDataset(ctx.getJobId(), dataverseName, datasetName);
- } catch (RemoteException e) {
- throw new MetadataException(e);
+ Dataset dataset = findDataset(ctx, dataverseName, datasetName);
+ // If a dataset is not in the cache, then it could not be a temp dataset
+ if (dataset == null || !dataset.getDatasetDetails().isTemp()) {
+ try {
+ metadataNode.dropDataset(ctx.getJobId(), dataverseName, datasetName);
+ } catch (RemoteException e) {
+ throw new MetadataException(e);
+ }
}
+
+ // Drops the dataset from cache
ctx.dropDataset(dataverseName, datasetName);
}
@@ -312,13 +338,23 @@
@Override
public List<Index> getDatasetIndexes(MetadataTransactionContext ctx, String dataverseName, String datasetName)
throws MetadataException {
- List<Index> datsetIndexes;
- try {
- datsetIndexes = metadataNode.getDatasetIndexes(ctx.getJobId(), dataverseName, datasetName);
- } catch (RemoteException e) {
- throw new MetadataException(e);
+ List<Index> datasetIndexes = new ArrayList<Index>();
+ Dataset dataset = findDataset(ctx, dataverseName, datasetName);
+ if (dataset == null) {
+ return datasetIndexes;
}
- return datsetIndexes;
+ if (dataset.getDatasetDetails().isTemp()) {
+ // for temp datsets
+ datasetIndexes = cache.getDatasetIndexes(dataverseName, datasetName);
+ } else {
+ try {
+ // for persistent datasets
+ datasetIndexes = metadataNode.getDatasetIndexes(ctx.getJobId(), dataverseName, datasetName);
+ } catch (RemoteException e) {
+ throw new MetadataException(e);
+ }
+ }
+ return datasetIndexes;
}
@Override
@@ -413,10 +449,15 @@
@Override
public void addIndex(MetadataTransactionContext ctx, Index index) throws MetadataException {
- try {
- metadataNode.addIndex(ctx.getJobId(), index);
- } catch (RemoteException e) {
- throw new MetadataException(e);
+ String dataverseName = index.getDataverseName();
+ String datasetName = index.getDatasetName();
+ Dataset dataset = findDataset(ctx, dataverseName, datasetName);
+ if (dataset == null || !dataset.getDatasetDetails().isTemp()) {
+ try {
+ metadataNode.addIndex(ctx.getJobId(), index);
+ } catch (RemoteException e) {
+ throw new MetadataException(e);
+ }
}
ctx.addIndex(index);
}
@@ -435,10 +476,15 @@
@Override
public void dropIndex(MetadataTransactionContext ctx, String dataverseName, String datasetName, String indexName)
throws MetadataException {
- try {
- metadataNode.dropIndex(ctx.getJobId(), dataverseName, datasetName, indexName);
- } catch (RemoteException e) {
- throw new MetadataException(e);
+ Dataset dataset = findDataset(ctx, dataverseName, datasetName);
+ // If a dataset is not in the cache, then it could be an unloaded persistent dataset.
+ // If the dataset is a temp dataset, then we do not need to call any MedataNode operations.
+ if (dataset == null || !dataset.getDatasetDetails().isTemp()) {
+ try {
+ metadataNode.dropIndex(ctx.getJobId(), dataverseName, datasetName, indexName);
+ } catch (RemoteException e) {
+ throw new MetadataException(e);
+ }
}
ctx.dropIndex(dataverseName, datasetName, indexName);
}
@@ -691,6 +737,7 @@
return feedActivity;
}
+ @Override
public void dropLibrary(MetadataTransactionContext ctx, String dataverseName, String libraryName)
throws MetadataException {
try {
@@ -815,6 +862,7 @@
ctx.addFeed(feed);
}
+ @Override
public List<FeedActivity> getConnectFeedActivitiesForFeed(MetadataTransactionContext ctx, String dataverse,
String feedName) throws MetadataException {
List<FeedActivity> feedActivities = null;
@@ -901,4 +949,17 @@
ctx.dropDataset(dataset);
ctx.addDataset(dataset);
}
+
+ @Override
+ public void cleanupTempDatasets() {
+ cache.cleanupTempDatasets();
+ }
+
+ private Dataset findDataset(MetadataTransactionContext ctx, String dataverseName, String datasetName) {
+ Dataset dataset = ctx.getDataset(dataverseName, datasetName);
+ if (dataset == null) {
+ dataset = cache.getDataset(dataverseName, datasetName);
+ }
+ return dataset;
+ }
}
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataTransactionContext.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataTransactionContext.java
index 3bdb2bfd..826ebbf 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataTransactionContext.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataTransactionContext.java
@@ -122,7 +122,8 @@
}
public void dropIndex(String dataverseName, String datasetName, String indexName) {
- Index index = new Index(dataverseName, datasetName, indexName, null, null, null, false, false, IMetadataEntity.PENDING_NO_OP);
+ Index index = new Index(dataverseName, datasetName, indexName, null, null, null, false, false,
+ IMetadataEntity.PENDING_NO_OP);
droppedCache.addIndexIfNotExists(index);
logAndApply(new MetadataLogicalOperation(index, false));
}
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/api/IMetadataManager.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/api/IMetadataManager.java
index 59e255e..0b3bae6 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/api/IMetadataManager.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/api/IMetadataManager.java
@@ -616,7 +616,8 @@
* added
* @throws MetadataException
*/
- public void addExternalFile(MetadataTransactionContext mdTxnCtx, ExternalFile externalFile) throws MetadataException;
+ public void addExternalFile(MetadataTransactionContext mdTxnCtx, ExternalFile externalFile)
+ throws MetadataException;
/**
* @param mdTxnCtx
@@ -637,7 +638,8 @@
* dropped
* @throws MetadataException
*/
- public void dropExternalFile(MetadataTransactionContext mdTxnCtx, ExternalFile externalFile) throws MetadataException;
+ public void dropExternalFile(MetadataTransactionContext mdTxnCtx, ExternalFile externalFile)
+ throws MetadataException;
/**
* @param mdTxnCtx
@@ -648,9 +650,10 @@
* @throws MetadataException
*/
public void dropDatasetExternalFiles(MetadataTransactionContext mdTxnCtx, Dataset dataset) throws MetadataException;
-
+
/**
* Get en external file
+ *
* @param mdTxnCtx
* @param dataverseName
* @param datasetName
@@ -658,9 +661,9 @@
* @return
* @throws MetadataException
*/
- public ExternalFile getExternalFile(MetadataTransactionContext mdTxnCtx, String dataverseName, String datasetName, Integer fileNumber)
- throws MetadataException;
-
+ public ExternalFile getExternalFile(MetadataTransactionContext mdTxnCtx, String dataverseName, String datasetName,
+ Integer fileNumber) throws MetadataException;
+
/**
* update an existing dataset in metadata.
*
@@ -672,5 +675,12 @@
* For example, if the dataset already exists.
*/
public void updateDataset(MetadataTransactionContext ctx, Dataset dataset) throws MetadataException;
-
+
+ /**
+ * Clean up temporary datasets that have not been active for a long time.
+ *
+ * @throws MetadataException
+ */
+ public void cleanupTempDatasets() throws MetadataException;
+
}
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java
index af425d5..c93c29e 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java
@@ -239,7 +239,7 @@
primaryIndexes[i].getPartitioningExpr(), primaryIndexes[i].getPartitioningExpr(),
primaryIndexes[i].getPartitioningExprType(), primaryIndexes[i].getNodeGroupName(), false,
GlobalConfig.DEFAULT_COMPACTION_POLICY_NAME, GlobalConfig.DEFAULT_COMPACTION_POLICY_PROPERTIES,
- null);
+ null, false);
MetadataManager.INSTANCE.addDataset(mdTxnCtx, new Dataset(primaryIndexes[i].getDataverseName(),
primaryIndexes[i].getIndexedDatasetName(), primaryIndexes[i].getPayloadRecordType().getTypeName(),
id, new HashMap<String, String>(), DatasetType.INTERNAL, primaryIndexes[i].getDatasetId().getId(),
@@ -390,7 +390,7 @@
ILSMOperationTracker opTracker = index.isPrimaryIndex() ? runtimeContext.getLSMBTreeOperationTracker(index
.getDatasetId().getId()) : new BaseOperationTracker((DatasetLifecycleManager) indexLifecycleManager,
index.getDatasetId().getId(), ((DatasetLifecycleManager) indexLifecycleManager).getDatasetInfo(index
- .getDatasetId().getId()));
+ .getDatasetId().getId()));
final String path = file.getFile().getPath();
if (create) {
lsmBtree = LSMBTreeUtils.createLSMTree(
@@ -406,7 +406,7 @@
GlobalConfig.DEFAULT_COMPACTION_POLICY_PROPERTIES, indexLifecycleManager), opTracker,
runtimeContext.getLSMIOScheduler(),
LSMBTreeIOOperationCallbackFactory.INSTANCE.createIOOperationCallback(), index.isPrimaryIndex(),
- null, null, null, null);
+ null, null, null, null, true);
lsmBtree.create();
resourceID = runtimeContext.getResourceIdFactory().createId();
ILocalResourceMetadata localResourceMetadata = new LSMBTreeLocalResourceMetadata(typeTraits,
@@ -435,7 +435,7 @@
runtimeContext.getMetadataMergePolicyFactory().createMergePolicy(
GlobalConfig.DEFAULT_COMPACTION_POLICY_PROPERTIES, indexLifecycleManager), opTracker,
runtimeContext.getLSMIOScheduler(), LSMBTreeIOOperationCallbackFactory.INSTANCE
- .createIOOperationCallback(), index.isPrimaryIndex(), null, null, null, null);
+ .createIOOperationCallback(), index.isPrimaryIndex(), null, null, null, null, true);
indexLifecycleManager.register(resourceID, lsmBtree);
}
}
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
index bc05e04..66a932e 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
@@ -97,6 +97,8 @@
import edu.uci.ics.asterix.runtime.formats.FormatUtils;
import edu.uci.ics.asterix.runtime.formats.NonTaggedDataFormat;
import edu.uci.ics.asterix.runtime.job.listener.JobEventListenerFactory;
+import edu.uci.ics.asterix.transaction.management.opcallbacks.TempDatasetPrimaryIndexModificationOperationCallbackFactory;
+import edu.uci.ics.asterix.transaction.management.opcallbacks.TempDatasetSecondaryIndexModificationOperationCallbackFactory;
import edu.uci.ics.asterix.transaction.management.opcallbacks.PrimaryIndexInstantSearchOperationCallbackFactory;
import edu.uci.ics.asterix.transaction.management.opcallbacks.PrimaryIndexModificationOperationCallbackFactory;
import edu.uci.ics.asterix.transaction.management.opcallbacks.PrimaryIndexOperationTrackerProvider;
@@ -152,11 +154,13 @@
import edu.uci.ics.hyracks.dataflow.std.result.ResultWriterOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallbackFactory;
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallbackFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
import edu.uci.ics.hyracks.storage.am.lsm.btree.dataflow.ExternalBTreeWithBuddyDataflowHelperFactory;
@@ -176,7 +180,7 @@
private static Logger LOGGER = Logger.getLogger(AqlMetadataProvider.class.getName());
private MetadataTransactionContext mdTxnCtx;
private boolean isWriteTransaction;
- private Map<String, String[]> stores;
+ private final Map<String, String[]> stores;
private Map<String, String> config;
private IAWriterFactory writerFactory;
private FileSplit outputFile;
@@ -187,6 +191,7 @@
private final Dataverse defaultDataverse;
private JobId jobId;
private Map<String, Integer> locks;
+ private boolean isTemporaryDatasetWriteJob = true;
private final AsterixStorageProperties storageProperties;
@@ -299,7 +304,7 @@
@Override
public AqlDataSource findDataSource(AqlSourceId id) throws AlgebricksException {
- AqlSourceId aqlId = (AqlSourceId) id;
+ AqlSourceId aqlId = id;
try {
return lookupSourceInMetadata(aqlId);
} catch (MetadataException e) {
@@ -308,9 +313,15 @@
}
public boolean isWriteTransaction() {
+ // The transaction writes persistent datasets.
return isWriteTransaction;
}
+ public boolean isTemporaryDatasetWriteJob() {
+ // The transaction only writes temporary datasets.
+ return isTemporaryDatasetWriteJob;
+ }
+
@Override
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getScannerRuntime(
IDataSource<AqlSourceId> dataSource, List<LogicalVariable> scanVariables,
@@ -557,15 +568,14 @@
case INTERNAL:
feedIngestor = new FeedIntakeOperatorDescriptor(jobSpec, new FeedConnectionId(
feedDataSource.getDatasourceDataverse(), feedDataSource.getDatasourceName(), feedDataSource
- .getFeedConnectionId().getDatasetName()), adapterFactory,
- (ARecordType) adapterOutputType, feedDesc, feedPolicy.getProperties());
+ .getFeedConnectionId().getDatasetName()), adapterFactory, adapterOutputType,
+ feedDesc, feedPolicy.getProperties());
break;
case EXTERNAL:
String libraryName = feedDataSource.getFeed().getAdapterName().split("#")[0];
feedIngestor = new FeedIntakeOperatorDescriptor(jobSpec, feedDataSource.getFeedConnectionId(),
libraryName, adapterFactory.getClass().getName(), feedDataSource.getFeed()
- .getAdapterConfiguration(), (ARecordType) adapterOutputType, feedDesc,
- feedPolicy.getProperties());
+ .getAdapterConfiguration(), adapterOutputType, feedDesc, feedPolicy.getProperties());
break;
}
if (LOGGER.isLoggable(Level.INFO)) {
@@ -611,6 +621,7 @@
boolean isSecondary = true;
int numSecondaryKeys = 0;
try {
+ boolean temp = dataset.getDatasetDetails().isTemp();
Index primaryIndex = MetadataManager.INSTANCE.getIndex(mdTxnCtx, dataset.getDataverseName(),
dataset.getDatasetName(), dataset.getDatasetName());
if (primaryIndex != null && dataset.getDatasetType() != DatasetType.EXTERNAL) {
@@ -670,14 +681,15 @@
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> spPc;
try {
spPc = splitProviderAndPartitionConstraintsForDataset(dataset.getDataverseName(),
- dataset.getDatasetName(), indexName);
+ dataset.getDatasetName(), indexName, temp);
} catch (Exception e) {
throw new AlgebricksException(e);
}
ISearchOperationCallbackFactory searchCallbackFactory = null;
if (isSecondary) {
- searchCallbackFactory = new SecondaryIndexSearchOperationCallbackFactory();
+ searchCallbackFactory = temp ? NoOpOperationCallbackFactory.INSTANCE
+ : new SecondaryIndexSearchOperationCallbackFactory();
} else {
JobId jobId = ((JobEventListenerFactory) jobSpec.getJobletEventListenerFactory()).getJobId();
int datasetId = dataset.getDatasetId();
@@ -689,11 +701,13 @@
AqlMetadataImplConfig aqlMetadataImplConfig = (AqlMetadataImplConfig) implConfig;
ITransactionSubsystemProvider txnSubsystemProvider = new TransactionSubsystemProvider();
if (aqlMetadataImplConfig != null && aqlMetadataImplConfig.isInstantLock()) {
- searchCallbackFactory = new PrimaryIndexInstantSearchOperationCallbackFactory(jobId, datasetId,
- primaryKeyFields, txnSubsystemProvider, ResourceType.LSM_BTREE);
+ searchCallbackFactory = temp ? NoOpOperationCallbackFactory.INSTANCE
+ : new PrimaryIndexInstantSearchOperationCallbackFactory(jobId, datasetId, primaryKeyFields,
+ txnSubsystemProvider, ResourceType.LSM_BTREE);
} else {
- searchCallbackFactory = new PrimaryIndexSearchOperationCallbackFactory(jobId, datasetId,
- primaryKeyFields, txnSubsystemProvider, ResourceType.LSM_BTREE);
+ searchCallbackFactory = temp ? NoOpOperationCallbackFactory.INSTANCE
+ : new PrimaryIndexSearchOperationCallbackFactory(jobId, datasetId, primaryKeyFields,
+ txnSubsystemProvider, ResourceType.LSM_BTREE);
}
}
Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(
@@ -711,7 +725,7 @@
dataset.getDatasetId()), rtcProvider,
LSMBTreeIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(), !isSecondary, filterTypeTraits,
- filterCmpFactories, btreeFields, filterFields), retainInput, retainNull,
+ filterCmpFactories, btreeFields, filterFields, !temp), retainInput, retainNull,
context.getNullWriterFactory(), searchCallbackFactory, minFilterFieldIndexes,
maxFilterFieldIndexes);
} else {
@@ -723,7 +737,7 @@
dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
LSMBTreeWithBuddyIOOperationCallbackFactory.INSTANCE, getStorageProperties()
.getBloomFilterFalsePositiveRate(), buddyBreeFields,
- ExternalDatasetsRegistry.INSTANCE.getAndLockDatasetVersion(dataset, this));
+ ExternalDatasetsRegistry.INSTANCE.getAndLockDatasetVersion(dataset, this), !temp);
btreeSearchOp = new ExternalBTreeSearchOperatorDescriptor(jobSpec, outputRecDesc, rtcProvider,
rtcProvider, spPc.first, typeTraits, comparatorFactories, bloomFilterKeyFields, lowKeyFields,
highKeyFields, lowKeyInclusive, highKeyInclusive, indexDataflowHelperFactory, retainInput,
@@ -746,6 +760,7 @@
ARecordType recType = (ARecordType) findType(dataset.getDataverseName(), dataset.getItemTypeName());
int numPrimaryKeys = DatasetUtils.getPartitioningKeys(dataset).size();
+ boolean temp = dataset.getDatasetDetails().isTemp();
Index secondaryIndex = MetadataManager.INSTANCE.getIndex(mdTxnCtx, dataset.getDataverseName(),
dataset.getDatasetName(), indexName);
if (secondaryIndex == null) {
@@ -786,7 +801,7 @@
numNestedSecondaryKeyFields + numPrimaryKeys, typeEnv, context);
IAsterixApplicationContextInfo appContext = (IAsterixApplicationContextInfo) context.getAppContext();
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> spPc = splitProviderAndPartitionConstraintsForDataset(
- dataset.getDataverseName(), dataset.getDatasetName(), indexName);
+ dataset.getDataverseName(), dataset.getDatasetName(), indexName, temp);
IBinaryComparatorFactory[] primaryComparatorFactories = DatasetUtils.computeKeysBinaryComparatorFactories(
dataset, recType, context.getBinaryComparatorFactoryProvider());
@@ -812,7 +827,8 @@
IAType nestedKeyType = NonTaggedFormatUtil.getNestedSpatialType(keyType.getTypeTag());
Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(
dataset, mdTxnCtx);
- ISearchOperationCallbackFactory searchCallbackFactory = new SecondaryIndexSearchOperationCallbackFactory();
+ ISearchOperationCallbackFactory searchCallbackFactory = temp ? NoOpOperationCallbackFactory.INSTANCE
+ : new SecondaryIndexSearchOperationCallbackFactory();
RTreeSearchOperatorDescriptor rtreeSearchOp;
if (dataset.getDatasetType() == DatasetType.INTERNAL) {
@@ -826,7 +842,7 @@
LSMRTreeIOOperationCallbackFactory.INSTANCE, proposeLinearizer(
nestedKeyType.getTypeTag(), comparatorFactories.length),
storageProperties.getBloomFilterFalsePositiveRate(), rtreeFields, btreeFields,
- filterTypeTraits, filterCmpFactories, filterFields), retainInput, retainNull,
+ filterTypeTraits, filterCmpFactories, filterFields, !temp), retainInput, retainNull,
context.getNullWriterFactory(), searchCallbackFactory, minFilterFieldIndexes,
maxFilterFieldIndexes);
@@ -840,7 +856,7 @@
proposeLinearizer(nestedKeyType.getTypeTag(), comparatorFactories.length),
getStorageProperties().getBloomFilterFalsePositiveRate(),
new int[] { numNestedSecondaryKeyFields },
- ExternalDatasetsRegistry.INSTANCE.getAndLockDatasetVersion(dataset, this));
+ ExternalDatasetsRegistry.INSTANCE.getAndLockDatasetVersion(dataset, this), !temp);
// Create the operator
rtreeSearchOp = new ExternalRTreeSearchOperatorDescriptor(jobSpec, outputRecDesc,
appContext.getStorageManagerInterface(), appContext.getIndexLifecycleManagerProvider(),
@@ -859,7 +875,7 @@
public Pair<IPushRuntimeFactory, AlgebricksPartitionConstraint> getWriteFileRuntime(IDataSink sink,
int[] printColumns, IPrinterFactory[] printerFactories, RecordDescriptor inputDesc) {
FileSplitDataSink fsds = (FileSplitDataSink) sink;
- FileSplitSinkId fssi = (FileSplitSinkId) fsds.getId();
+ FileSplitSinkId fssi = fsds.getId();
FileSplit fs = fssi.getFileSplit();
File outFile = fs.getLocalFile().getFile();
String nodeId = fs.getNodeName();
@@ -875,7 +891,7 @@
int[] printColumns, IPrinterFactory[] printerFactories, RecordDescriptor inputDesc, boolean ordered,
JobSpecification spec) throws AlgebricksException {
ResultSetDataSink rsds = (ResultSetDataSink) sink;
- ResultSetSinkId rssId = (ResultSetSinkId) rsds.getId();
+ ResultSetSinkId rssId = rsds.getId();
ResultSetId rsId = rssId.getResultSetId();
ResultWriterOperatorDescriptor resultWriter = null;
@@ -898,7 +914,7 @@
Dataset dataset = ((DatasetDataSource) ads).getDataset();
try {
- String indexName = (String) indexId;
+ String indexName = indexId;
Index secondaryIndex = MetadataManager.INSTANCE.getIndex(mdTxnCtx, dataset.getDataverseName(),
dataset.getDatasetName(), indexName);
if (secondaryIndex != null) {
@@ -983,6 +999,9 @@
}
try {
+ boolean temp = dataset.getDatasetDetails().isTemp();
+ isTemporaryDatasetWriteJob = isTemporaryDatasetWriteJob && temp;
+
Index primaryIndex = MetadataManager.INSTANCE.getIndex(mdTxnCtx, dataset.getDataverseName(),
dataset.getDatasetName(), dataset.getDatasetName());
String indexName = primaryIndex.getIndexName();
@@ -995,7 +1014,7 @@
itemType, context.getBinaryComparatorFactoryProvider());
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = splitProviderAndPartitionConstraintsForDataset(
- dataSource.getId().getDataverseName(), datasetName, indexName);
+ dataSource.getId().getDataverseName(), datasetName, indexName, temp);
IAsterixApplicationContextInfo appContext = (IAsterixApplicationContextInfo) context.getAppContext();
long numElementsHint = getCardinalityPerPartitionHint(dataset);
@@ -1022,7 +1041,7 @@
dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
LSMBTreeIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(), true, filterTypeTraits,
- filterCmpFactories, btreeFields, filterFields));
+ filterCmpFactories, btreeFields, filterFields, !temp));
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(btreeBulkLoad,
splitsAndConstraint.second);
} catch (MetadataException me) {
@@ -1042,6 +1061,8 @@
throw new AlgebricksException("Unknown dataset " + datasetName + " in dataverse "
+ dataSource.getId().getDataverseName());
}
+ boolean temp = dataset.getDatasetDetails().isTemp();
+ isTemporaryDatasetWriteJob = isTemporaryDatasetWriteJob && temp;
int numKeys = keys.size();
int numFilterFields = DatasetUtils.getFilterField(dataset) == null ? 0 : 1;
@@ -1076,7 +1097,7 @@
IBinaryComparatorFactory[] comparatorFactories = DatasetUtils.computeKeysBinaryComparatorFactories(dataset,
itemType, context.getBinaryComparatorFactoryProvider());
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = splitProviderAndPartitionConstraintsForDataset(
- dataSource.getId().getDataverseName(), datasetName, indexName);
+ dataSource.getId().getDataverseName(), datasetName, indexName, temp);
// prepare callback
JobId jobId = ((JobEventListenerFactory) spec.getJobletEventListenerFactory()).getJobId();
@@ -1093,8 +1114,10 @@
int[] btreeFields = DatasetUtils.createBTreeFieldsWhenThereisAFilter(dataset);
TransactionSubsystemProvider txnSubsystemProvider = new TransactionSubsystemProvider();
- PrimaryIndexModificationOperationCallbackFactory modificationCallbackFactory = new PrimaryIndexModificationOperationCallbackFactory(
- jobId, datasetId, primaryKeyFields, txnSubsystemProvider, indexOp, ResourceType.LSM_BTREE);
+ IModificationOperationCallbackFactory modificationCallbackFactory = temp ? new TempDatasetPrimaryIndexModificationOperationCallbackFactory(
+ jobId, datasetId, primaryKeyFields, txnSubsystemProvider, indexOp, ResourceType.LSM_BTREE)
+ : new PrimaryIndexModificationOperationCallbackFactory(jobId, datasetId, primaryKeyFields,
+ txnSubsystemProvider, indexOp, ResourceType.LSM_BTREE);
Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(
dataset, mdTxnCtx);
@@ -1102,7 +1125,7 @@
datasetId), compactionInfo.first, compactionInfo.second, new PrimaryIndexOperationTrackerProvider(
dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
LSMBTreeIOOperationCallbackFactory.INSTANCE, storageProperties.getBloomFilterFalsePositiveRate(),
- true, filterTypeTraits, filterCmpFactories, btreeFields, filterFields);
+ true, filterTypeTraits, filterCmpFactories, btreeFields, filterFields, !temp);
IOperatorDescriptor op;
if (bulkload) {
long numElementsHint = getCardinalityPerPartitionHint(dataset);
@@ -1390,7 +1413,7 @@
secondaryKeyType.getTypeTag(), indexType, secondaryIndex.getGramLength());
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = splitProviderAndPartitionConstraintsForDataset(
- dataverseName, datasetName, indexName);
+ dataverseName, datasetName, indexName, dataset.getDatasetDetails().isTemp());
// Generate Output Record format
ISerializerDeserializer[] tokenKeyPairFields = new ISerializerDeserializer[numTokenKeyPairFields];
@@ -1476,6 +1499,8 @@
if (dataset == null) {
throw new AlgebricksException("Unknown dataset " + datasetName + " in dataverse " + dataverseName);
}
+ boolean temp = dataset.getDatasetDetails().isTemp();
+ isTemporaryDatasetWriteJob = isTemporaryDatasetWriteJob && temp;
int numKeys = primaryKeys.size() + secondaryKeys.size();
int numFilterFields = DatasetUtils.getFilterField(dataset) == null ? 0 : 1;
@@ -1557,15 +1582,16 @@
IAsterixApplicationContextInfo appContext = (IAsterixApplicationContextInfo) context.getAppContext();
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = splitProviderAndPartitionConstraintsForDataset(
- dataverseName, datasetName, indexName);
+ dataverseName, datasetName, indexName, temp);
// prepare callback
JobId jobId = ((JobEventListenerFactory) spec.getJobletEventListenerFactory()).getJobId();
int datasetId = dataset.getDatasetId();
TransactionSubsystemProvider txnSubsystemProvider = new TransactionSubsystemProvider();
- SecondaryIndexModificationOperationCallbackFactory modificationCallbackFactory = new SecondaryIndexModificationOperationCallbackFactory(
+ IModificationOperationCallbackFactory modificationCallbackFactory = temp ? new TempDatasetSecondaryIndexModificationOperationCallbackFactory(
jobId, datasetId, modificationCallbackPrimaryKeyFields, txnSubsystemProvider, indexOp,
- ResourceType.LSM_BTREE);
+ ResourceType.LSM_BTREE) : new SecondaryIndexModificationOperationCallbackFactory(jobId, datasetId,
+ modificationCallbackPrimaryKeyFields, txnSubsystemProvider, indexOp, ResourceType.LSM_BTREE);
Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(
dataset, mdTxnCtx);
@@ -1574,7 +1600,7 @@
new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMBTreeIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(), false, filterTypeTraits, filterCmpFactories,
- btreeFields, filterFields);
+ btreeFields, filterFields, !temp);
IOperatorDescriptor op;
if (bulkload) {
long numElementsHint = getCardinalityPerPartitionHint(dataset);
@@ -1592,7 +1618,7 @@
dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
LSMBTreeIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(), false, filterTypeTraits,
- filterCmpFactories, btreeFields, filterFields), filterFactory,
+ filterCmpFactories, btreeFields, filterFields, !temp), filterFactory,
modificationCallbackFactory, false, indexName);
}
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(op, splitsAndConstraint.second);
@@ -1635,6 +1661,8 @@
if (dataset == null) {
throw new AlgebricksException("Unknown dataset " + datasetName + " in dataverse " + dataverseName);
}
+ boolean temp = dataset.getDatasetDetails().isTemp();
+ isTemporaryDatasetWriteJob = isTemporaryDatasetWriteJob && temp;
// For tokenization, sorting and loading.
// One token (+ optional partitioning field) + primary keys: [token,
@@ -1754,14 +1782,16 @@
IAsterixApplicationContextInfo appContext = (IAsterixApplicationContextInfo) context.getAppContext();
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = splitProviderAndPartitionConstraintsForDataset(
- dataverseName, datasetName, indexName);
+ dataverseName, datasetName, indexName, temp);
// prepare callback
JobId jobId = ((JobEventListenerFactory) spec.getJobletEventListenerFactory()).getJobId();
int datasetId = dataset.getDatasetId();
TransactionSubsystemProvider txnSubsystemProvider = new TransactionSubsystemProvider();
- SecondaryIndexModificationOperationCallbackFactory modificationCallbackFactory = new SecondaryIndexModificationOperationCallbackFactory(
+ IModificationOperationCallbackFactory modificationCallbackFactory = temp ? new TempDatasetSecondaryIndexModificationOperationCallbackFactory(
jobId, datasetId, modificationCallbackPrimaryKeyFields, txnSubsystemProvider, indexOp,
+ ResourceType.LSM_INVERTED_INDEX) : new SecondaryIndexModificationOperationCallbackFactory(jobId,
+ datasetId, modificationCallbackPrimaryKeyFields, txnSubsystemProvider, indexOp,
ResourceType.LSM_INVERTED_INDEX);
Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(
@@ -1775,7 +1805,7 @@
LSMInvertedIndexIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(), invertedIndexFields, filterTypeTraits,
filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps,
- invertedIndexFieldsForNonBulkLoadOps);
+ invertedIndexFieldsForNonBulkLoadOps, !temp);
} else {
indexDataFlowFactory = new PartitionedLSMInvertedIndexDataflowHelperFactory(
new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()), compactionInfo.first,
@@ -1784,7 +1814,7 @@
LSMInvertedIndexIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(), invertedIndexFields, filterTypeTraits,
filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps,
- invertedIndexFieldsForNonBulkLoadOps);
+ invertedIndexFieldsForNonBulkLoadOps, !temp);
}
IOperatorDescriptor op;
if (bulkload) {
@@ -1816,6 +1846,10 @@
boolean bulkload) throws AlgebricksException {
try {
Dataset dataset = MetadataManager.INSTANCE.getDataset(mdTxnCtx, dataverseName, datasetName);
+
+ boolean temp = dataset.getDatasetDetails().isTemp();
+ isTemporaryDatasetWriteJob = isTemporaryDatasetWriteJob && temp;
+
String itemTypeName = dataset.getItemTypeName();
IAType itemType = MetadataManager.INSTANCE.getDatatype(mdTxnCtx, dataverseName, itemTypeName).getDatatype();
if (itemType.getTypeTag() != ATypeTag.RECORD) {
@@ -1878,7 +1912,7 @@
dataset, recType, context.getBinaryComparatorFactoryProvider());
IAsterixApplicationContextInfo appContext = (IAsterixApplicationContextInfo) context.getAppContext();
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = splitProviderAndPartitionConstraintsForDataset(
- dataverseName, datasetName, indexName);
+ dataverseName, datasetName, indexName, temp);
int[] btreeFields = new int[primaryComparatorFactories.length];
for (int k = 0; k < btreeFields.length; k++) {
btreeFields[k] = k + numSecondaryKeys;
@@ -1902,9 +1936,10 @@
JobId jobId = ((JobEventListenerFactory) spec.getJobletEventListenerFactory()).getJobId();
int datasetId = dataset.getDatasetId();
TransactionSubsystemProvider txnSubsystemProvider = new TransactionSubsystemProvider();
- SecondaryIndexModificationOperationCallbackFactory modificationCallbackFactory = new SecondaryIndexModificationOperationCallbackFactory(
+ IModificationOperationCallbackFactory modificationCallbackFactory = temp ? new TempDatasetSecondaryIndexModificationOperationCallbackFactory(
jobId, datasetId, modificationCallbackPrimaryKeyFields, txnSubsystemProvider, indexOp,
- ResourceType.LSM_RTREE);
+ ResourceType.LSM_RTREE) : new SecondaryIndexModificationOperationCallbackFactory(jobId, datasetId,
+ modificationCallbackPrimaryKeyFields, txnSubsystemProvider, indexOp, ResourceType.LSM_RTREE);
Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(
dataset, mdTxnCtx);
@@ -1915,7 +1950,7 @@
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMRTreeIOOperationCallbackFactory.INSTANCE,
proposeLinearizer(nestedKeyType.getTypeTag(), comparatorFactories.length),
storageProperties.getBloomFilterFalsePositiveRate(), rtreeFields, btreeFields, filterTypeTraits,
- filterCmpFactories, filterFields);
+ filterCmpFactories, filterFields, !temp);
IOperatorDescriptor op;
if (bulkload) {
long numElementsHint = getCardinalityPerPartitionHint(dataset);
@@ -1935,7 +1970,7 @@
LSMRTreeIOOperationCallbackFactory.INSTANCE, proposeLinearizer(
nestedKeyType.getTypeTag(), comparatorFactories.length), storageProperties
.getBloomFilterFalsePositiveRate(), rtreeFields, btreeFields, filterTypeTraits,
- filterCmpFactories, filterFields), filterFactory,
+ filterCmpFactories, filterFields, !temp), filterFactory,
modificationCallbackFactory, false, indexName);
}
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(op, splitsAndConstraint.second);
@@ -1992,8 +2027,8 @@
}
public Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitProviderAndPartitionConstraintsForDataset(
- String dataverseName, String datasetName, String targetIdxName) throws AlgebricksException {
- FileSplit[] splits = splitsForDataset(mdTxnCtx, dataverseName, datasetName, targetIdxName);
+ String dataverseName, String datasetName, String targetIdxName, boolean temp) throws AlgebricksException {
+ FileSplit[] splits = splitsForDataset(mdTxnCtx, dataverseName, datasetName, targetIdxName, temp);
return splitProviderAndPartitionConstraints(splits);
}
@@ -2038,7 +2073,7 @@
}
private FileSplit[] splitsForDataset(MetadataTransactionContext mdTxnCtx, String dataverseName, String datasetName,
- String targetIdxName) throws AlgebricksException {
+ String targetIdxName, boolean temp) throws AlgebricksException {
try {
File relPathFile = new File(getRelativePath(dataverseName, datasetName + "_idx_" + targetIdxName));
Dataset dataset = MetadataManager.INSTANCE.getDataset(mdTxnCtx, dataverseName, datasetName);
@@ -2065,19 +2100,14 @@
String[] ioDevices = AsterixClusterProperties.INSTANCE.getIODevices(nd);
for (int j = 0; j < nodeStores.length; j++) {
for (int k = 0; k < numIODevices; k++) {
- File f = new File(ioDevices[k] + File.separator + nodeStores[j] + File.separator
- + relPathFile);
+ File f = new File(ioDevices[k] + File.separator + nodeStores[j]
+ + (temp ? (File.separator + "temp") : "") + File.separator + relPathFile);
splitArray.add(new FileSplit(nd, new FileReference(f), k));
}
}
}
}
- FileSplit[] splits = new FileSplit[splitArray.size()];
- int i = 0;
- for (FileSplit fs : splitArray) {
- splits[i++] = fs;
- }
- return splits;
+ return splitArray.toArray(new FileSplit[0]);
} catch (MetadataException me) {
throw new AlgebricksException(me);
}
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/ExternalDatasetDetails.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/ExternalDatasetDetails.java
index eb83159..5f7fb74 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/ExternalDatasetDetails.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/ExternalDatasetDetails.java
@@ -46,6 +46,7 @@
private final String adapter;
private final Map<String, String> properties;
private final String nodeGroupName;
+ private final long addToCacheTime;
private Date lastRefreshTime;
private ExternalDatasetTransactionState state;
protected String compactionPolicy;
@@ -57,6 +58,7 @@
this.properties = properties;
this.adapter = adapter;
this.nodeGroupName = nodeGroupName;
+ this.addToCacheTime = System.currentTimeMillis();
this.lastRefreshTime = lastRefreshTime;
this.state = state;
this.compactionPolicy = compactionPolicy;
@@ -196,6 +198,16 @@
return nodeGroupName;
}
+ @Override
+ public boolean isTemp() {
+ return false;
+ }
+
+ @Override
+ public long getLastAccessTime() {
+ return addToCacheTime;
+ }
+
public Date getTimestamp() {
return lastRefreshTime;
}
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/InternalDatasetDetails.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/InternalDatasetDetails.java
index 507a9e3..4ce8593 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/InternalDatasetDetails.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/InternalDatasetDetails.java
@@ -60,13 +60,15 @@
protected final boolean autogenerated;
protected final String compactionPolicy;
protected final Map<String, String> compactionPolicyProperties;
+ protected final boolean temp;
+ protected long lastAccessTime;
protected final List<String> filterField;
public static final String FILTER_FIELD_NAME = "FilterField";
public InternalDatasetDetails(FileStructure fileStructure, PartitioningStrategy partitioningStrategy,
List<List<String>> partitioningKey, List<List<String>> primaryKey, List<IAType> primaryKeyType,
String groupName, boolean autogenerated, String compactionPolicy,
- Map<String, String> compactionPolicyProperties, List<String> filterField) {
+ Map<String, String> compactionPolicyProperties, List<String> filterField, boolean temp) {
this.fileStructure = fileStructure;
this.partitioningStrategy = partitioningStrategy;
this.partitioningKeys = partitioningKey;
@@ -77,6 +79,8 @@
this.compactionPolicy = compactionPolicy;
this.compactionPolicyProperties = compactionPolicyProperties;
this.filterField = filterField;
+ this.temp = temp;
+ this.lastAccessTime = System.currentTimeMillis();
}
@Override
@@ -124,10 +128,21 @@
@Override
public DatasetType getDatasetType() {
+ lastAccessTime = System.currentTimeMillis();
return DatasetType.INTERNAL;
}
@Override
+ public long getLastAccessTime() {
+ return lastAccessTime;
+ }
+
+ @Override
+ public boolean isTemp() {
+ return temp;
+ }
+
+ @Override
public void writeDatasetDetailsRecordType(DataOutput out) throws HyracksDataException {
IARecordBuilder internalRecordBuilder = new RecordBuilder();
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
index 2f7185a..ed6fca4 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
@@ -75,9 +75,9 @@
public static final int DATASET_PAYLOAD_TUPLE_FIELD_INDEX = 2;
@SuppressWarnings("unchecked")
- private ISerializerDeserializer<ARecord> recordSerDes = AqlSerializerDeserializerProvider.INSTANCE
+ private final ISerializerDeserializer<ARecord> recordSerDes = AqlSerializerDeserializerProvider.INSTANCE
.getSerializerDeserializer(MetadataRecordTypes.DATASET_RECORDTYPE);
- private AMutableInt32 aInt32;
+ private final AMutableInt32 aInt32;
protected ISerializerDeserializer<AInt32> aInt32Serde;
@SuppressWarnings("unchecked")
@@ -94,7 +94,7 @@
int recordLength = frameTuple.getFieldLength(DATASET_PAYLOAD_TUPLE_FIELD_INDEX);
ByteArrayInputStream stream = new ByteArrayInputStream(serRecord, recordStartOffset, recordLength);
DataInput in = new DataInputStream(stream);
- ARecord datasetRecord = (ARecord) recordSerDes.deserialize(in);
+ ARecord datasetRecord = recordSerDes.deserialize(in);
return createDatasetFromARecord(datasetRecord);
}
@@ -177,10 +177,11 @@
}
}
+ // Temporary dataset only lives in the compiler therefore the temp field is false.
+ // DatasetTupleTranslator always read from the metadata node, so the temp flag should be always false.
datasetDetails = new InternalDatasetDetails(fileStructure, partitioningStrategy, partitioningKey,
partitioningKey, partitioningKeyType, groupName, autogenerated, compactionPolicy,
- compactionPolicyProperties, filterField);
-
+ compactionPolicyProperties, filterField, false);
break;
}
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/external/ExternalLoopkupOperatorDiscriptor.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/external/ExternalLoopkupOperatorDiscriptor.java
index 99c747b..e7df93f 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/external/ExternalLoopkupOperatorDiscriptor.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/external/ExternalLoopkupOperatorDiscriptor.java
@@ -16,7 +16,6 @@
import java.nio.ByteBuffer;
-import edu.uci.ics.asterix.transaction.management.opcallbacks.SecondaryIndexSearchOperationCallbackFactory;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
import edu.uci.ics.hyracks.api.dataflow.value.INullWriterFactory;
@@ -39,8 +38,8 @@
public class ExternalLoopkupOperatorDiscriptor extends AbstractTreeIndexOperatorDescriptor {
private static final long serialVersionUID = 1L;
- private IControlledAdapterFactory adapterFactory;
- private INullWriterFactory iNullWriterFactory;
+ private final IControlledAdapterFactory adapterFactory;
+ private final INullWriterFactory iNullWriterFactory;
public ExternalLoopkupOperatorDiscriptor(IOperatorDescriptorRegistry spec,
IControlledAdapterFactory adapterFactory, RecordDescriptor outRecDesc,
@@ -53,7 +52,7 @@
FilesIndexDescription.EXTERNAL_FILE_INDEX_TYPE_TRAITS,
FilesIndexDescription.FILES_INDEX_COMP_FACTORIES, FilesIndexDescription.BLOOM_FILTER_FIELDS,
externalFilesIndexDataFlowHelperFactory, null, propagateInput, retainNull, iNullWriterFactory, null,
- new SecondaryIndexSearchOperationCallbackFactory(), null);
+ searchOpCallbackFactory, null);
this.adapterFactory = adapterFactory;
this.iNullWriterFactory = iNullWriterFactory;
}
@@ -69,7 +68,7 @@
this);
return new AbstractUnaryInputUnaryOutputOperatorNodePushable() {
// The adapter that uses the file index along with the coming tuples to access files in HDFS
- private IControlledAdapter adapter = adapterFactory.createAdapter(ctx, fileIndexAccessor,
+ private final IControlledAdapter adapter = adapterFactory.createAdapter(ctx, fileIndexAccessor,
recordDescProvider.getInputRecordDescriptor(getActivityId(), 0));
@Override
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/opcallbacks/TempDatasetIndexModificationOperationCallback.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/opcallbacks/TempDatasetIndexModificationOperationCallback.java
new file mode 100644
index 0000000..e539bb8
--- /dev/null
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/opcallbacks/TempDatasetIndexModificationOperationCallback.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.asterix.transaction.management.opcallbacks;
+
+import edu.uci.ics.asterix.common.transactions.ILockManager;
+import edu.uci.ics.asterix.common.transactions.ITransactionContext;
+import edu.uci.ics.asterix.common.transactions.ITransactionSubsystem;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
+
+/**
+ * This class is the operation callback for temporary datasets.
+ * A temporary dataset does not require any lock and does not generate any write-ahead update and commit log
+ * but generates flush log and job commit log.
+ * The "before" and "found" method in this callback is empty so that no locking is requested for accessing a temporary
+ * dataset and no write-ahead log is written for update operations.
+ */
+public class TempDatasetIndexModificationOperationCallback extends AbstractIndexModificationOperationCallback implements
+ IModificationOperationCallback {
+
+ public TempDatasetIndexModificationOperationCallback(int datasetId, int[] primaryKeyFields,
+ ITransactionContext txnCtx, ILockManager lockManager, ITransactionSubsystem txnSubsystem, long resourceId,
+ byte resourceType, IndexOperation indexOp) {
+ super(datasetId, primaryKeyFields, txnCtx, lockManager, txnSubsystem, resourceId, resourceType, indexOp);
+ }
+
+ @Override
+ public void before(ITupleReference tuple) throws HyracksDataException {
+
+ }
+
+ @Override
+ public void found(ITupleReference before, ITupleReference after) throws HyracksDataException {
+
+ }
+}
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/opcallbacks/TempDatasetPrimaryIndexModificationOperationCallbackFactory.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/opcallbacks/TempDatasetPrimaryIndexModificationOperationCallbackFactory.java
new file mode 100644
index 0000000..535672e
--- /dev/null
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/opcallbacks/TempDatasetPrimaryIndexModificationOperationCallbackFactory.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.asterix.transaction.management.opcallbacks;
+
+import edu.uci.ics.asterix.common.context.ITransactionSubsystemProvider;
+import edu.uci.ics.asterix.common.exceptions.ACIDException;
+import edu.uci.ics.asterix.common.transactions.AbstractOperationCallback;
+import edu.uci.ics.asterix.common.transactions.AbstractOperationCallbackFactory;
+import edu.uci.ics.asterix.common.transactions.ITransactionContext;
+import edu.uci.ics.asterix.common.transactions.ITransactionSubsystem;
+import edu.uci.ics.asterix.common.transactions.JobId;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexLifecycleManager;
+import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallbackFactory;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndex;
+
+public class TempDatasetPrimaryIndexModificationOperationCallbackFactory extends AbstractOperationCallbackFactory implements
+ IModificationOperationCallbackFactory {
+
+ private static final long serialVersionUID = 1L;
+ private final IndexOperation indexOp;
+
+ public TempDatasetPrimaryIndexModificationOperationCallbackFactory(JobId jobId, int datasetId, int[] primaryKeyFields,
+ ITransactionSubsystemProvider txnSubsystemProvider, IndexOperation indexOp, byte resourceType) {
+ super(jobId, datasetId, primaryKeyFields, txnSubsystemProvider, resourceType);
+ this.indexOp = indexOp;
+ }
+
+ @Override
+ public IModificationOperationCallback createModificationOperationCallback(long resourceId, Object resource,
+ IHyracksTaskContext ctx) throws HyracksDataException {
+ ITransactionSubsystem txnSubsystem = txnSubsystemProvider.getTransactionSubsystem(ctx);
+ IIndexLifecycleManager indexLifeCycleManager = txnSubsystem.getAsterixAppRuntimeContextProvider()
+ .getIndexLifecycleManager();
+ ILSMIndex index = (ILSMIndex) indexLifeCycleManager.getIndex(resourceId);
+ if (index == null) {
+ throw new HyracksDataException("Index(id:" + resourceId + ") is not registered.");
+ }
+
+ try {
+ ITransactionContext txnCtx = txnSubsystem.getTransactionManager().getTransactionContext(jobId, false);
+ IModificationOperationCallback modCallback = new TempDatasetIndexModificationOperationCallback(datasetId,
+ primaryKeyFields, txnCtx, txnSubsystem.getLockManager(), txnSubsystem, resourceId, resourceType,
+ indexOp);
+ txnCtx.registerIndexAndCallback(resourceId, index, (AbstractOperationCallback) modCallback, true);
+ return modCallback;
+ } catch (ACIDException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+}
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/opcallbacks/TempDatasetSecondaryIndexModificationOperationCallbackFactory.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/opcallbacks/TempDatasetSecondaryIndexModificationOperationCallbackFactory.java
new file mode 100644
index 0000000..c47bd38
--- /dev/null
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/opcallbacks/TempDatasetSecondaryIndexModificationOperationCallbackFactory.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.asterix.transaction.management.opcallbacks;
+
+import edu.uci.ics.asterix.common.context.ITransactionSubsystemProvider;
+import edu.uci.ics.asterix.common.exceptions.ACIDException;
+import edu.uci.ics.asterix.common.transactions.AbstractOperationCallback;
+import edu.uci.ics.asterix.common.transactions.AbstractOperationCallbackFactory;
+import edu.uci.ics.asterix.common.transactions.ITransactionContext;
+import edu.uci.ics.asterix.common.transactions.ITransactionSubsystem;
+import edu.uci.ics.asterix.common.transactions.JobId;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexLifecycleManager;
+import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallbackFactory;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndex;
+
+public class TempDatasetSecondaryIndexModificationOperationCallbackFactory extends AbstractOperationCallbackFactory implements
+ IModificationOperationCallbackFactory {
+
+ private static final long serialVersionUID = 1L;
+ private final IndexOperation indexOp;
+
+ public TempDatasetSecondaryIndexModificationOperationCallbackFactory(JobId jobId, int datasetId, int[] primaryKeyFields,
+ ITransactionSubsystemProvider txnSubsystemProvider, IndexOperation indexOp, byte resourceType) {
+ super(jobId, datasetId, primaryKeyFields, txnSubsystemProvider, resourceType);
+ this.indexOp = indexOp;
+ }
+
+ @Override
+ public IModificationOperationCallback createModificationOperationCallback(long resourceId, Object resource,
+ IHyracksTaskContext ctx) throws HyracksDataException {
+ ITransactionSubsystem txnSubsystem = txnSubsystemProvider.getTransactionSubsystem(ctx);
+ IIndexLifecycleManager indexLifeCycleManager = txnSubsystem.getAsterixAppRuntimeContextProvider()
+ .getIndexLifecycleManager();
+ ILSMIndex index = (ILSMIndex) indexLifeCycleManager.getIndex(resourceId);
+ if (index == null) {
+ throw new HyracksDataException("Index(id:" + resourceId + ") is not registered.");
+ }
+
+ try {
+ ITransactionContext txnCtx = txnSubsystem.getTransactionManager().getTransactionContext(jobId, false);
+ IModificationOperationCallback modCallback = new TempDatasetIndexModificationOperationCallback(datasetId,
+ primaryKeyFields, txnCtx, txnSubsystem.getLockManager(), txnSubsystem, resourceId, resourceType,
+ indexOp);
+ txnCtx.registerIndexAndCallback(resourceId, index, (AbstractOperationCallback) modCallback, false);
+ return modCallback;
+ } catch (ACIDException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+}
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java
index 7d699b3..35140e6 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java
@@ -44,13 +44,20 @@
public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider, String filePath,
int partition) {
FileReference file = new FileReference(new File(filePath));
- LSMBTree lsmBTree = LSMBTreeUtils.createExternalBTree(file, runtimeContextProvider.getBufferCache(),
- runtimeContextProvider.getFileMapManager(), typeTraits, cmpFactories, bloomFilterKeyFields,
- runtimeContextProvider.getBloomFilterFalsePositiveRate(), mergePolicyFactory.createMergePolicy(
- mergePolicyProperties, runtimeContextProvider.getIndexLifecycleManager()),
+ LSMBTree lsmBTree = LSMBTreeUtils.createExternalBTree(
+ file,
+ runtimeContextProvider.getBufferCache(),
+ runtimeContextProvider.getFileMapManager(),
+ typeTraits,
+ cmpFactories,
+ bloomFilterKeyFields,
+ runtimeContextProvider.getBloomFilterFalsePositiveRate(),
+ mergePolicyFactory.createMergePolicy(mergePolicyProperties,
+ runtimeContextProvider.getIndexLifecycleManager()),
new BaseOperationTracker((DatasetLifecycleManager) runtimeContextProvider.getIndexLifecycleManager(),
- datasetID, ((DatasetLifecycleManager) runtimeContextProvider.getIndexLifecycleManager()).getDatasetInfo(datasetID)), runtimeContextProvider.getLSMIOScheduler(),
- LSMBTreeIOOperationCallbackFactory.INSTANCE.createIOOperationCallback(), -1);
+ datasetID, ((DatasetLifecycleManager) runtimeContextProvider.getIndexLifecycleManager())
+ .getDatasetInfo(datasetID)), runtimeContextProvider.getLSMIOScheduler(),
+ LSMBTreeIOOperationCallbackFactory.INSTANCE.createIOOperationCallback(), -1, true);
return lsmBTree;
}
}
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java
index ad4ff0c..c9e7f33 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java
@@ -57,12 +57,19 @@
public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider, String filePath,
int partition) throws HyracksDataException {
FileReference file = new FileReference(new File(filePath));
- return LSMBTreeUtils.createExternalBTreeWithBuddy(file, runtimeContextProvider.getBufferCache(),
- runtimeContextProvider.getFileMapManager(), typeTraits, btreeCmpFactories, runtimeContextProvider
- .getBloomFilterFalsePositiveRate(), mergePolicyFactory.createMergePolicy(mergePolicyProperties,
- runtimeContextProvider.getIndexLifecycleManager()), new BaseOperationTracker(
- (DatasetLifecycleManager) runtimeContextProvider.getIndexLifecycleManager(), datasetID, ((DatasetLifecycleManager) runtimeContextProvider.getIndexLifecycleManager()).getDatasetInfo(datasetID)),
- runtimeContextProvider.getLSMIOScheduler(), LSMBTreeWithBuddyIOOperationCallbackFactory.INSTANCE
- .createIOOperationCallback(), buddyBtreeFields, -1);
+ return LSMBTreeUtils.createExternalBTreeWithBuddy(
+ file,
+ runtimeContextProvider.getBufferCache(),
+ runtimeContextProvider.getFileMapManager(),
+ typeTraits,
+ btreeCmpFactories,
+ runtimeContextProvider.getBloomFilterFalsePositiveRate(),
+ mergePolicyFactory.createMergePolicy(mergePolicyProperties,
+ runtimeContextProvider.getIndexLifecycleManager()),
+ new BaseOperationTracker((DatasetLifecycleManager) runtimeContextProvider.getIndexLifecycleManager(),
+ datasetID, ((DatasetLifecycleManager) runtimeContextProvider.getIndexLifecycleManager())
+ .getDatasetInfo(datasetID)), runtimeContextProvider.getLSMIOScheduler(),
+ LSMBTreeWithBuddyIOOperationCallbackFactory.INSTANCE.createIOOperationCallback(), buddyBtreeFields, -1,
+ true);
}
}
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java
index 7c4a437..5c67274 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java
@@ -54,14 +54,23 @@
int partition) throws HyracksDataException {
FileReference file = new FileReference(new File(filePath));
try {
- return LSMRTreeUtils.createExternalRTree(file, runtimeContextProvider.getBufferCache(),
- runtimeContextProvider.getFileMapManager(), typeTraits, rtreeCmpFactories, btreeCmpFactories,
- valueProviderFactories, rtreePolicyType, runtimeContextProvider.getBloomFilterFalsePositiveRate(),
+ return LSMRTreeUtils.createExternalRTree(
+ file,
+ runtimeContextProvider.getBufferCache(),
+ runtimeContextProvider.getFileMapManager(),
+ typeTraits,
+ rtreeCmpFactories,
+ btreeCmpFactories,
+ valueProviderFactories,
+ rtreePolicyType,
+ runtimeContextProvider.getBloomFilterFalsePositiveRate(),
mergePolicyFactory.createMergePolicy(mergePolicyProperties,
- runtimeContextProvider.getIndexLifecycleManager()), new BaseOperationTracker(
- (DatasetLifecycleManager) runtimeContextProvider.getIndexLifecycleManager(), datasetID, ((DatasetLifecycleManager) runtimeContextProvider.getIndexLifecycleManager()).getDatasetInfo(datasetID)),
- runtimeContextProvider.getLSMIOScheduler(), LSMRTreeIOOperationCallbackFactory.INSTANCE
- .createIOOperationCallback(), linearizeCmpFactory, btreeFields, -1);
+ runtimeContextProvider.getIndexLifecycleManager()),
+ new BaseOperationTracker((DatasetLifecycleManager) runtimeContextProvider
+ .getIndexLifecycleManager(), datasetID, ((DatasetLifecycleManager) runtimeContextProvider
+ .getIndexLifecycleManager()).getDatasetInfo(datasetID)), runtimeContextProvider
+ .getLSMIOScheduler(), LSMRTreeIOOperationCallbackFactory.INSTANCE
+ .createIOOperationCallback(), linearizeCmpFactory, btreeFields, -1, true);
} catch (TreeIndexException e) {
throw new HyracksDataException(e);
}
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java
index 76dd03a..08938be 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java
@@ -62,15 +62,23 @@
int partition) {
FileReference file = new FileReference(new File(filePath));
List<IVirtualBufferCache> virtualBufferCaches = runtimeContextProvider.getVirtualBufferCaches(datasetID);
- LSMBTree lsmBTree = LSMBTreeUtils.createLSMTree(virtualBufferCaches, file, runtimeContextProvider
- .getBufferCache(), runtimeContextProvider.getFileMapManager(), typeTraits, cmpFactories,
- bloomFilterKeyFields, runtimeContextProvider.getBloomFilterFalsePositiveRate(), mergePolicyFactory
- .createMergePolicy(mergePolicyProperties, runtimeContextProvider.getIndexLifecycleManager()),
+ LSMBTree lsmBTree = LSMBTreeUtils.createLSMTree(
+ virtualBufferCaches,
+ file,
+ runtimeContextProvider.getBufferCache(),
+ runtimeContextProvider.getFileMapManager(),
+ typeTraits,
+ cmpFactories,
+ bloomFilterKeyFields,
+ runtimeContextProvider.getBloomFilterFalsePositiveRate(),
+ mergePolicyFactory.createMergePolicy(mergePolicyProperties,
+ runtimeContextProvider.getIndexLifecycleManager()),
isPrimary ? runtimeContextProvider.getLSMBTreeOperationTracker(datasetID) : new BaseOperationTracker(
- (DatasetLifecycleManager) runtimeContextProvider.getIndexLifecycleManager(), datasetID, ((DatasetLifecycleManager) runtimeContextProvider.getIndexLifecycleManager()).getDatasetInfo(datasetID)),
- runtimeContextProvider.getLSMIOScheduler(), LSMBTreeIOOperationCallbackFactory.INSTANCE
- .createIOOperationCallback(), isPrimary, filterTypeTraits, filterCmpFactories, btreeFields,
- filterFields);
+ (DatasetLifecycleManager) runtimeContextProvider.getIndexLifecycleManager(), datasetID,
+ ((DatasetLifecycleManager) runtimeContextProvider.getIndexLifecycleManager())
+ .getDatasetInfo(datasetID)), runtimeContextProvider.getLSMIOScheduler(),
+ LSMBTreeIOOperationCallbackFactory.INSTANCE.createIOOperationCallback(), isPrimary, filterTypeTraits,
+ filterCmpFactories, btreeFields, filterFields, true);
return lsmBTree;
}
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java
index 7d43626..63cf858 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java
@@ -88,10 +88,12 @@
mergePolicyFactory.createMergePolicy(mergePolicyProperties,
runtimeContextProvider.getIndexLifecycleManager()),
new BaseOperationTracker((DatasetLifecycleManager) runtimeContextProvider
- .getIndexLifecycleManager(), datasetID, ((DatasetLifecycleManager) runtimeContextProvider.getIndexLifecycleManager()).getDatasetInfo(datasetID)), runtimeContextProvider.getLSMIOScheduler(),
+ .getIndexLifecycleManager(), datasetID,
+ ((DatasetLifecycleManager) runtimeContextProvider.getIndexLifecycleManager())
+ .getDatasetInfo(datasetID)), runtimeContextProvider.getLSMIOScheduler(),
LSMInvertedIndexIOOperationCallbackFactory.INSTANCE.createIOOperationCallback(),
invertedIndexFields, filterTypeTraits, filterCmpFactories, filterFields,
- filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps);
+ filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, true);
} else {
return InvertedIndexUtils.createLSMInvertedIndex(
virtualBufferCaches,
@@ -107,10 +109,12 @@
mergePolicyFactory.createMergePolicy(mergePolicyProperties,
runtimeContextProvider.getIndexLifecycleManager()),
new BaseOperationTracker((DatasetLifecycleManager) runtimeContextProvider
- .getIndexLifecycleManager(), datasetID, ((DatasetLifecycleManager) runtimeContextProvider.getIndexLifecycleManager()).getDatasetInfo(datasetID)), runtimeContextProvider.getLSMIOScheduler(),
+ .getIndexLifecycleManager(), datasetID,
+ ((DatasetLifecycleManager) runtimeContextProvider.getIndexLifecycleManager())
+ .getDatasetInfo(datasetID)), runtimeContextProvider.getLSMIOScheduler(),
LSMInvertedIndexIOOperationCallbackFactory.INSTANCE.createIOOperationCallback(),
invertedIndexFields, filterTypeTraits, filterCmpFactories, filterFields,
- filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps);
+ filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, true);
}
} catch (IndexException e) {
throw new HyracksDataException(e);
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
index 6b0597d..8d0255d 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
@@ -75,15 +75,25 @@
FileReference file = new FileReference(new File(filePath));
List<IVirtualBufferCache> virtualBufferCaches = runtimeContextProvider.getVirtualBufferCaches(datasetID);
try {
- return LSMRTreeUtils.createLSMTree(virtualBufferCaches, file, runtimeContextProvider.getBufferCache(),
- runtimeContextProvider.getFileMapManager(), typeTraits, rtreeCmpFactories, btreeCmpFactories,
- valueProviderFactories, rtreePolicyType, runtimeContextProvider.getBloomFilterFalsePositiveRate(),
+ return LSMRTreeUtils.createLSMTree(
+ virtualBufferCaches,
+ file,
+ runtimeContextProvider.getBufferCache(),
+ runtimeContextProvider.getFileMapManager(),
+ typeTraits,
+ rtreeCmpFactories,
+ btreeCmpFactories,
+ valueProviderFactories,
+ rtreePolicyType,
+ runtimeContextProvider.getBloomFilterFalsePositiveRate(),
mergePolicyFactory.createMergePolicy(mergePolicyProperties,
- runtimeContextProvider.getIndexLifecycleManager()), new BaseOperationTracker(
- (DatasetLifecycleManager) runtimeContextProvider.getIndexLifecycleManager(), datasetID, ((DatasetLifecycleManager) runtimeContextProvider.getIndexLifecycleManager()).getDatasetInfo(datasetID)),
- runtimeContextProvider.getLSMIOScheduler(), LSMRTreeIOOperationCallbackFactory.INSTANCE
+ runtimeContextProvider.getIndexLifecycleManager()),
+ new BaseOperationTracker((DatasetLifecycleManager) runtimeContextProvider
+ .getIndexLifecycleManager(), datasetID, ((DatasetLifecycleManager) runtimeContextProvider
+ .getIndexLifecycleManager()).getDatasetInfo(datasetID)), runtimeContextProvider
+ .getLSMIOScheduler(), LSMRTreeIOOperationCallbackFactory.INSTANCE
.createIOOperationCallback(), linearizeCmpFactory, rtreeFields, btreeFields,
- filterTypeTraits, filterCmpFactories, filterFields);
+ filterTypeTraits, filterCmpFactories, filterFields, true);
} catch (TreeIndexException e) {
throw new HyracksDataException(e);
}
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/locking/LockManager.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/locking/LockManager.java
index 3720f08..8f68926 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/locking/LockManager.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/locking/LockManager.java
@@ -61,29 +61,29 @@
private static final int ESCALATED = 1;
private static final int DONOT_ESCALATE = 2;
- private TransactionSubsystem txnSubsystem;
+ private final TransactionSubsystem txnSubsystem;
//all threads accessing to LockManager's tables such as jobHT and datasetResourceHT
//are serialized through LockTableLatch. All threads waiting the latch will be fairly served
//in FIFO manner when the latch is available.
private final ReadWriteLock lockTableLatch;
private final ReadWriteLock waiterLatch;
- private HashMap<JobId, JobInfo> jobHT;
- private HashMap<DatasetId, DatasetLockInfo> datasetResourceHT;
+ private final HashMap<JobId, JobInfo> jobHT;
+ private final HashMap<DatasetId, DatasetLockInfo> datasetResourceHT;
- private EntityLockInfoManager entityLockInfoManager;
- private EntityInfoManager entityInfoManager;
- private LockWaiterManager lockWaiterManager;
+ private final EntityLockInfoManager entityLockInfoManager;
+ private final EntityInfoManager entityInfoManager;
+ private final LockWaiterManager lockWaiterManager;
- private DeadlockDetector deadlockDetector;
- private TimeOutDetector toutDetector;
- private DatasetId tempDatasetIdObj; //temporary object to avoid object creation
- private JobId tempJobIdObj;
+ private final DeadlockDetector deadlockDetector;
+ private final TimeOutDetector toutDetector;
+ private final DatasetId tempDatasetIdObj; //temporary object to avoid object creation
+ private final JobId tempJobIdObj;
private int tryLockDatasetGranuleRevertOperation;
private LockRequestTracker lockRequestTracker; //for debugging
- private ConsecutiveWakeupContext consecutiveWakeupContext;
+ private final ConsecutiveWakeupContext consecutiveWakeupContext;
public LockManager(TransactionSubsystem txnSubsystem) throws ACIDException {
this.txnSubsystem = txnSubsystem;
@@ -638,7 +638,8 @@
}
@Override
- public void unlock(DatasetId datasetId, int entityHashValue, byte lockMode, ITransactionContext txnContext) throws ACIDException {
+ public void unlock(DatasetId datasetId, int entityHashValue, byte lockMode, ITransactionContext txnContext)
+ throws ACIDException {
internalUnlock(datasetId, entityHashValue, txnContext, false);
}
@@ -2010,7 +2011,7 @@
Iterator<Entry<JobId, JobInfo>> iter = jobHT.entrySet().iterator();
while (iter.hasNext()) {
- Map.Entry<JobId, JobInfo> pair = (Map.Entry<JobId, JobInfo>) iter.next();
+ Map.Entry<JobId, JobInfo> pair = iter.next();
jobInfo = pair.getValue();
waiterObjId = jobInfo.getFirstWaitingResource();
while (waiterObjId != -1) {
@@ -2215,7 +2216,7 @@
tempJobIdObj.setId(logRecord.getJobId());
txnCtx = txnSubsystem.getTransactionManager().getTransactionContext(tempJobIdObj, false);
txnCtx.notifyOptracker(true);
- ((LogPage) logPage).notifyJobTerminator();
+ logPage.notifyJobTerminator();
}
logRecord = logPageReader.next();
}
@@ -2281,4 +2282,4 @@
}
-}
+}
\ No newline at end of file
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/LogPage.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/LogPage.java
index f3e8f77..10fe1e2 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/LogPage.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/LogPage.java
@@ -55,8 +55,8 @@
private FileChannel fileChannel;
private boolean stop;
- private DatasetId reusableDsId;
- private JobId reusableJobId;
+ private final DatasetId reusableDsId;
+ private final JobId reusableJobId;
public LogPage(TransactionSubsystem txnSubsystem, int logPageSize, MutableLong flushLSN) {
this.txnSubsystem = txnSubsystem;
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/transaction/TransactionContext.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/transaction/TransactionContext.java
index b28964a..77c66a2 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/transaction/TransactionContext.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/transaction/TransactionContext.java
@@ -30,8 +30,8 @@
import edu.uci.ics.asterix.common.transactions.JobId;
import edu.uci.ics.asterix.common.transactions.LogRecord;
import edu.uci.ics.asterix.common.transactions.MutableLong;
-import edu.uci.ics.asterix.transaction.management.opcallbacks.PrimaryIndexModificationOperationCallback;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndex;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMOperationType;
@@ -44,7 +44,7 @@
public class TransactionContext implements ITransactionContext, Serializable {
private static final long serialVersionUID = -6105616785783310111L;
- private TransactionSubsystem transactionSubsystem;
+ private final TransactionSubsystem transactionSubsystem;
// jobId is set once and read concurrently.
private final JobId jobId;
@@ -54,18 +54,18 @@
// But readers and writers can be different threads,
// so both LSNs are atomic variables in order to be read and written
// atomically.
- private AtomicLong firstLSN;
- private AtomicLong lastLSN;
+ private final AtomicLong firstLSN;
+ private final AtomicLong lastLSN;
// txnState is read and written concurrently.
- private AtomicInteger txnState;
+ private final AtomicInteger txnState;
// isTimeout is read and written under the lockMgr's tableLatch
// Thus, no other synchronization is required separately.
private boolean isTimeout;
// isWriteTxn can be set concurrently by multiple threads.
- private AtomicBoolean isWriteTxn;
+ private final AtomicBoolean isWriteTxn;
// isMetadataTxn is accessed by a single thread since the metadata is not
// partitioned
@@ -73,21 +73,21 @@
// indexMap is concurrently accessed by multiple threads,
// so those threads are synchronized on indexMap object itself
- private Map<MutableLong, AbstractLSMIOOperationCallback> indexMap;
+ private final Map<MutableLong, AbstractLSMIOOperationCallback> indexMap;
// TODO: fix ComponentLSNs' issues.
// primaryIndex, primaryIndexCallback, and primaryIndexOptracker will be
// modified accordingly
// when the issues of componentLSNs are fixed.
private ILSMIndex primaryIndex;
- private PrimaryIndexModificationOperationCallback primaryIndexCallback;
+ private AbstractOperationCallback primaryIndexCallback;
private PrimaryIndexOperationTracker primaryIndexOpTracker;
// The following three variables are used as temporary variables in order to
// avoid object creations.
// Those are used in synchronized methods.
- private MutableLong tempResourceIdForRegister;
- private LogRecord logRecord;
+ private final MutableLong tempResourceIdForRegister;
+ private final LogRecord logRecord;
// TODO: implement transactionContext pool in order to avoid object
// creations.
@@ -108,12 +108,13 @@
logRecord = new LogRecord();
}
+ @Override
public void registerIndexAndCallback(long resourceId, ILSMIndex index, AbstractOperationCallback callback,
boolean isPrimaryIndex) {
synchronized (indexMap) {
if (isPrimaryIndex && primaryIndex == null) {
primaryIndex = index;
- primaryIndexCallback = (PrimaryIndexModificationOperationCallback) callback;
+ primaryIndexCallback = callback;
primaryIndexOpTracker = (PrimaryIndexOperationTracker) index.getOperationTracker();
}
tempResourceIdForRegister.set(resourceId);
@@ -138,18 +139,20 @@
if (isJobLevelCommit && isMetadataTxn) {
primaryIndexOpTracker.exclusiveJobCommitted();
} else if (!isJobLevelCommit) {
- primaryIndexOpTracker
- .completeOperation(null, LSMOperationType.MODIFICATION, null, primaryIndexCallback);
+ primaryIndexOpTracker.completeOperation(null, LSMOperationType.MODIFICATION, null,
+ (IModificationOperationCallback) primaryIndexCallback);
}
} catch (HyracksDataException e) {
throw new IllegalStateException(e);
}
}
+ @Override
public void setWriteTxn(boolean isWriteTxn) {
this.isWriteTxn.set(isWriteTxn);
}
+ @Override
public boolean isWriteTxn() {
return isWriteTxn.get();
}
@@ -164,22 +167,27 @@
return lastLSN.get();
}
+ @Override
public JobId getJobId() {
return jobId;
}
+ @Override
public void setTimeout(boolean isTimeout) {
this.isTimeout = isTimeout;
}
+ @Override
public boolean isTimeout() {
return isTimeout;
}
+ @Override
public void setTxnState(int txnState) {
this.txnState.set(txnState);
}
+ @Override
public int getTxnState() {
return txnState.get();
}
@@ -204,6 +212,7 @@
return isMetadataTxn;
}
+ @Override
public String prettyPrint() {
StringBuilder sb = new StringBuilder();
sb.append("\n" + jobId + "\n");