Support for full compaction command.
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlExpressionToPlanTranslator.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlExpressionToPlanTranslator.java
index 6f92d3a..7b12550 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlExpressionToPlanTranslator.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlExpressionToPlanTranslator.java
@@ -28,6 +28,7 @@
import edu.uci.ics.asterix.aql.base.Expression.Kind;
import edu.uci.ics.asterix.aql.expression.BeginFeedStatement;
import edu.uci.ics.asterix.aql.expression.CallExpr;
+import edu.uci.ics.asterix.aql.expression.CompactStatement;
import edu.uci.ics.asterix.aql.expression.ControlFeedStatement;
import edu.uci.ics.asterix.aql.expression.CreateDataverseStatement;
import edu.uci.ics.asterix.aql.expression.CreateFunctionStatement;
@@ -1430,4 +1431,11 @@
// TODO Auto-generated method stub
return null;
}
+
+ @Override
+ public Pair<ILogicalOperator, LogicalVariable> visitCompactStatement(CompactStatement del,
+ Mutable<ILogicalOperator> arg) throws AsterixException {
+ // TODO Auto-generated method stub
+ return null;
+ }
}
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlPlusExpressionToPlanTranslator.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlPlusExpressionToPlanTranslator.java
index d1f8053..4d11dbd 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlPlusExpressionToPlanTranslator.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlPlusExpressionToPlanTranslator.java
@@ -28,6 +28,7 @@
import edu.uci.ics.asterix.aql.base.Expression.Kind;
import edu.uci.ics.asterix.aql.expression.BeginFeedStatement;
import edu.uci.ics.asterix.aql.expression.CallExpr;
+import edu.uci.ics.asterix.aql.expression.CompactStatement;
import edu.uci.ics.asterix.aql.expression.ControlFeedStatement;
import edu.uci.ics.asterix.aql.expression.CreateDataverseStatement;
import edu.uci.ics.asterix.aql.expression.CreateFunctionStatement;
@@ -1448,4 +1449,11 @@
// TODO Auto-generated method stub
return null;
}
+
+ @Override
+ public Pair<ILogicalOperator, LogicalVariable> visitCompactStatement(CompactStatement del,
+ Mutable<ILogicalOperator> arg) throws AsterixException {
+ // TODO Auto-generated method stub
+ return null;
+ }
}
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/CompiledStatements.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/CompiledStatements.java
index 36f0824..d9aa047 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/CompiledStatements.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/CompiledStatements.java
@@ -39,7 +39,6 @@
import edu.uci.ics.asterix.common.functions.FunctionSignature;
import edu.uci.ics.asterix.metadata.declared.AqlMetadataProvider;
import edu.uci.ics.asterix.metadata.entities.Dataset;
-import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
import edu.uci.ics.asterix.om.types.ARecordType;
import edu.uci.ics.asterix.om.types.IAType;
import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -458,8 +457,8 @@
LiteralExpr argumentLiteral = new LiteralExpr(new StringLiteral(arg));
arguments.add(argumentLiteral);
- CallExpr callExpression = new CallExpr(new FunctionSignature(
- FunctionConstants.ASTERIX_NS, "dataset", 1), arguments);
+ CallExpr callExpression = new CallExpr(new FunctionSignature(FunctionConstants.ASTERIX_NS, "dataset", 1),
+ arguments);
List<Clause> clauseList = new ArrayList<Clause>();
Clause forClause = new ForClause(var, callExpression);
clauseList.add(forClause);
@@ -498,4 +497,61 @@
}
+ public static class CompiledCompactStatement implements ICompiledStatement {
+ private final String dataverseName;
+ private final String datasetName;
+
+ public CompiledCompactStatement(String dataverseName, String datasetName) {
+ this.dataverseName = dataverseName;
+ this.datasetName = datasetName;
+ }
+
+ public String getDataverseName() {
+ return dataverseName;
+ }
+
+ public String getDatasetName() {
+ return datasetName;
+ }
+
+ @Override
+ public Kind getKind() {
+ return Kind.COMPACT;
+ }
+ }
+
+ public static class CompiledIndexCompactStatement extends CompiledCompactStatement {
+ private final String indexName;
+ private final List<String> keyFields;
+ private final IndexType indexType;
+
+ // Specific to NGram index.
+ private final int gramLength;
+
+ public CompiledIndexCompactStatement(String dataverseName, String datasetName, String indexName,
+ List<String> keyFields, int gramLength, IndexType indexType) {
+ super(dataverseName, datasetName);
+ this.indexName = indexName;
+ this.keyFields = keyFields;
+ this.gramLength = gramLength;
+ this.indexType = indexType;
+ }
+
+ public String getIndexName() {
+ return indexName;
+ }
+
+ public List<String> getKeyFields() {
+ return keyFields;
+ }
+
+ public IndexType getIndexType() {
+ return indexType;
+ }
+
+ public int getGramLength() {
+ return gramLength;
+ }
+ }
+
}
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/UpdateAPIServlet.java b/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/UpdateAPIServlet.java
index cb1e9ad..772fad6 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/UpdateAPIServlet.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/UpdateAPIServlet.java
@@ -32,7 +32,7 @@
protected List<Statement.Kind> getAllowedStatements() {
Kind[] statementsArray = { Kind.DATAVERSE_DECL, Kind.DELETE, Kind.INSERT, Kind.UPDATE,
Kind.DML_CMD_LIST, Kind.LOAD_FROM_FILE, Kind.BEGIN_FEED,
- Kind.CONTROL_FEED };
+ Kind.CONTROL_FEED, Kind.COMPACT };
return Arrays.asList(statementsArray);
}
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
index f22d2fb..68a41a2 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
@@ -33,6 +33,7 @@
import edu.uci.ics.asterix.api.common.SessionConfig;
import edu.uci.ics.asterix.aql.base.Statement;
import edu.uci.ics.asterix.aql.expression.BeginFeedStatement;
+import edu.uci.ics.asterix.aql.expression.CompactStatement;
import edu.uci.ics.asterix.aql.expression.ControlFeedStatement;
import edu.uci.ics.asterix.aql.expression.CreateDataverseStatement;
import edu.uci.ics.asterix.aql.expression.CreateFunctionStatement;
@@ -97,6 +98,7 @@
import edu.uci.ics.asterix.translator.CompiledStatements.CompiledCreateIndexStatement;
import edu.uci.ics.asterix.translator.CompiledStatements.CompiledDatasetDropStatement;
import edu.uci.ics.asterix.translator.CompiledStatements.CompiledDeleteStatement;
+import edu.uci.ics.asterix.translator.CompiledStatements.CompiledIndexCompactStatement;
import edu.uci.ics.asterix.translator.CompiledStatements.CompiledIndexDropStatement;
import edu.uci.ics.asterix.translator.CompiledStatements.CompiledInsertStatement;
import edu.uci.ics.asterix.translator.CompiledStatements.CompiledLoadFromFileStatement;
@@ -273,6 +275,11 @@
break;
}
+ case COMPACT: {
+ handleCompactStatement(metadataProvider, stmt, hcc);
+ break;
+ }
+
case WRITE: {
Pair<IAWriterFactory, FileSplit> result = handleWriteStatement(metadataProvider, stmt);
if (result.first != null) {
@@ -1339,6 +1346,60 @@
}
}
+ private void handleCompactStatement(AqlMetadataProvider metadataProvider, Statement stmt,
+ IHyracksClientConnection hcc) throws Exception {
+ MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+ boolean bActiveTxn = true;
+ metadataProvider.setMetadataTxnContext(mdTxnCtx);
+ acquireReadLatch();
+
+ String dataverseName = null;
+ String datasetName = null;
+ List<JobSpecification> jobsToExecute = new ArrayList<JobSpecification>();
+ try {
+ CompactStatement compactStatement = (CompactStatement) stmt;
+ dataverseName = getActiveDataverseName(compactStatement.getDataverseName());
+ datasetName = compactStatement.getDatasetName().getValue();
+
+ Dataset ds = MetadataManager.INSTANCE.getDataset(mdTxnCtx, dataverseName, datasetName);
+ if (ds == null) {
+ throw new AlgebricksException("There is no dataset with this name " + datasetName + " in dataverse "
+ + dataverseName + ".");
+ } else if (ds.getDatasetType() != DatasetType.INTERNAL && ds.getDatasetType() != DatasetType.FEED) {
+ throw new AlgebricksException("Cannot compact the extrenal dataset " + datasetName + ".");
+ }
+
+ // Prepare jobs to compact the datatset and its indexes
+ List<Index> indexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName, datasetName);
+ for (int j = 0; j < indexes.size(); j++) {
+ if (indexes.get(j).isSecondaryIndex()) {
+ CompiledIndexCompactStatement cics = new CompiledIndexCompactStatement(dataverseName, datasetName,
+ indexes.get(j).getIndexName(), indexes.get(j).getKeyFieldNames(), indexes.get(j)
+ .getGramLength(), indexes.get(j).getIndexType());
+ jobsToExecute.add(IndexOperations.buildSecondaryIndexCompactJobSpec(cics, metadataProvider, ds));
+ }
+ }
+ Dataverse dataverse = MetadataManager.INSTANCE.getDataverse(metadataProvider.getMetadataTxnContext(),
+ dataverseName);
+ jobsToExecute.add(DatasetOperations.compactDatasetJobSpec(dataverse, datasetName, metadataProvider));
+
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ bActiveTxn = false;
+
+ //#. run the jobs
+ for (JobSpecification jobSpec : jobsToExecute) {
+ runJob(hcc, jobSpec, true);
+ }
+ } catch (Exception e) {
+ if (bActiveTxn) {
+ abort(e, e, mdTxnCtx);
+ }
+ throw e;
+ } finally {
+ releaseReadLatch();
+ }
+ }
+
private QueryResult handleQuery(AqlMetadataProvider metadataProvider, Query query, IHyracksClientConnection hcc,
IHyracksDataset hdc, boolean asyncResults) throws Exception {
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
index 3e41a77..edbae21 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
@@ -30,6 +30,7 @@
import edu.uci.ics.asterix.common.context.AsterixVirtualBufferCacheProvider;
import edu.uci.ics.asterix.common.exceptions.ACIDException;
import edu.uci.ics.asterix.common.exceptions.AsterixException;
+import edu.uci.ics.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory;
import edu.uci.ics.asterix.formats.base.IDataFormat;
import edu.uci.ics.asterix.metadata.MetadataManager;
import edu.uci.ics.asterix.metadata.MetadataTransactionContext;
@@ -42,6 +43,7 @@
import edu.uci.ics.asterix.om.types.IAType;
import edu.uci.ics.asterix.om.util.AsterixAppContextInfo;
import edu.uci.ics.asterix.transaction.management.opcallbacks.PrimaryIndexOperationTrackerProvider;
+import edu.uci.ics.asterix.transaction.management.opcallbacks.SecondaryIndexOperationTrackerProvider;
import edu.uci.ics.asterix.transaction.management.resource.LSMBTreeLocalResourceMetadata;
import edu.uci.ics.asterix.transaction.management.resource.PersistentLocalResourceFactoryProvider;
import edu.uci.ics.asterix.transaction.management.service.transaction.AsterixRuntimeComponentsProvider;
@@ -82,6 +84,7 @@
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexCreateOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
import edu.uci.ics.hyracks.storage.am.lsm.btree.dataflow.LSMBTreeDataflowHelperFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.dataflow.LSMTreeIndexCompactOperatorDescriptor;
import edu.uci.ics.hyracks.storage.common.file.ILocalResourceFactoryProvider;
import edu.uci.ics.hyracks.storage.common.file.LocalResource;
@@ -359,4 +362,47 @@
}
return new RecordDescriptor(recordFields);
}
+
+ public static JobSpecification compactDatasetJobSpec(Dataverse dataverse, String datasetName,
+ AqlMetadataProvider metadata) throws AsterixException, AlgebricksException {
+ String dataverseName = dataverse.getDataverseName();
+ IDataFormat format;
+ try {
+ format = (IDataFormat) Class.forName(dataverse.getDataFormat()).newInstance();
+ } catch (Exception e) {
+ throw new AsterixException(e);
+ }
+ Dataset dataset = metadata.findDataset(dataverseName, datasetName);
+ if (dataset == null) {
+ throw new AsterixException("Could not find dataset " + datasetName + " in datavetse " + dataverseName);
+ }
+ ARecordType itemType = (ARecordType) metadata.findType(dataverseName, dataset.getItemTypeName());
+ JobSpecification spec = JobSpecificationUtils.createJobSpecification();
+ IBinaryComparatorFactory[] comparatorFactories = DatasetUtils.computeKeysBinaryComparatorFactories(dataset,
+ itemType, format.getBinaryComparatorFactoryProvider());
+ ITypeTraits[] typeTraits = DatasetUtils.computeTupleTypeTraits(dataset, itemType);
+ int[] blooFilterKeyFields = DatasetUtils.createBloomFilterKeyFields(dataset);
+
+ Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadata
+ .splitProviderAndPartitionConstraintsForInternalOrFeedDataset(dataverseName, datasetName, datasetName);
+
+ AsterixStorageProperties storageProperties = AsterixAppContextInfo.getInstance().getStorageProperties();
+
+ LSMTreeIndexCompactOperatorDescriptor compactOp = new LSMTreeIndexCompactOperatorDescriptor(spec,
+ AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+ splitsAndConstraint.first, typeTraits, comparatorFactories, blooFilterKeyFields,
+ new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()),
+ AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, new SecondaryIndexOperationTrackerProvider(
+ LSMBTreeIOOperationCallbackFactory.INSTANCE, dataset.getDatasetId()),
+ AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+ AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+ storageProperties.getBloomFilterFalsePositiveRate()), NoOpOperationCallbackFactory.INSTANCE);
+ AlgebricksPartitionConstraintHelper
+ .setPartitionConstraintInJobSpec(spec, compactOp, splitsAndConstraint.second);
+
+ AlgebricksPartitionConstraintHelper
+ .setPartitionConstraintInJobSpec(spec, compactOp, splitsAndConstraint.second);
+ spec.addRoot(compactOp);
+ return spec;
+ }
}
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java
index af56894..60d7251 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java
@@ -26,6 +26,7 @@
import edu.uci.ics.asterix.transaction.management.opcallbacks.SecondaryIndexOperationTrackerProvider;
import edu.uci.ics.asterix.transaction.management.service.transaction.AsterixRuntimeComponentsProvider;
import edu.uci.ics.asterix.translator.CompiledStatements.CompiledCreateIndexStatement;
+import edu.uci.ics.asterix.translator.CompiledStatements.CompiledIndexCompactStatement;
import edu.uci.ics.asterix.translator.CompiledStatements.CompiledIndexDropStatement;
import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
@@ -44,16 +45,22 @@
public static JobSpecification buildSecondaryIndexCreationJobSpec(CompiledCreateIndexStatement createIndexStmt,
AqlMetadataProvider metadataProvider) throws AsterixException, AlgebricksException {
- SecondaryIndexCreator secondaryIndexCreator = SecondaryIndexCreator.createIndexCreator(createIndexStmt,
- metadataProvider, physicalOptimizationConfig);
- return secondaryIndexCreator.buildCreationJobSpec();
+ SecondaryIndexOperationsHelper secondaryIndexHelper = SecondaryIndexOperationsHelper
+ .createIndexOperationsHelper(createIndexStmt.getIndexType(), createIndexStmt.getDataverseName(),
+ createIndexStmt.getDatasetName(), createIndexStmt.getIndexName(),
+ createIndexStmt.getKeyFields(), createIndexStmt.getGramLength(), metadataProvider,
+ physicalOptimizationConfig);
+ return secondaryIndexHelper.buildCreationJobSpec();
}
public static JobSpecification buildSecondaryIndexLoadingJobSpec(CompiledCreateIndexStatement createIndexStmt,
AqlMetadataProvider metadataProvider) throws AsterixException, AlgebricksException {
- SecondaryIndexCreator secondaryIndexCreator = SecondaryIndexCreator.createIndexCreator(createIndexStmt,
- metadataProvider, physicalOptimizationConfig);
- return secondaryIndexCreator.buildLoadingJobSpec();
+ SecondaryIndexOperationsHelper secondaryIndexHelper = SecondaryIndexOperationsHelper
+ .createIndexOperationsHelper(createIndexStmt.getIndexType(), createIndexStmt.getDataverseName(),
+ createIndexStmt.getDatasetName(), createIndexStmt.getIndexName(),
+ createIndexStmt.getKeyFields(), createIndexStmt.getGramLength(), metadataProvider,
+ physicalOptimizationConfig);
+ return secondaryIndexHelper.buildLoadingJobSpec();
}
public static JobSpecification buildDropSecondaryIndexJobSpec(CompiledIndexDropStatement indexDropStmt,
@@ -81,4 +88,14 @@
return spec;
}
+
+ public static JobSpecification buildSecondaryIndexCompactJobSpec(CompiledIndexCompactStatement indexCompactStmt,
+ AqlMetadataProvider metadataProvider, Dataset dataset) throws AsterixException, AlgebricksException {
+ SecondaryIndexOperationsHelper secondaryIndexHelper = SecondaryIndexOperationsHelper
+ .createIndexOperationsHelper(indexCompactStmt.getIndexType(), indexCompactStmt.getDataverseName(),
+ indexCompactStmt.getDatasetName(), indexCompactStmt.getIndexName(),
+ indexCompactStmt.getKeyFields(), indexCompactStmt.getGramLength(), metadataProvider,
+ physicalOptimizationConfig);
+ return secondaryIndexHelper.buildCompactJobSpec();
+ }
}
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeCreator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeOperationsHelper.java
similarity index 81%
rename from asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeCreator.java
rename to asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeOperationsHelper.java
index e3832d4..71caad1 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeCreator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeOperationsHelper.java
@@ -39,12 +39,13 @@
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexCreateOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
import edu.uci.ics.hyracks.storage.am.lsm.btree.dataflow.LSMBTreeDataflowHelperFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.dataflow.LSMTreeIndexCompactOperatorDescriptor;
import edu.uci.ics.hyracks.storage.common.file.ILocalResourceFactoryProvider;
import edu.uci.ics.hyracks.storage.common.file.LocalResource;
-public class SecondaryBTreeCreator extends SecondaryIndexCreator {
+public class SecondaryBTreeOperationsHelper extends SecondaryIndexOperationsHelper {
- protected SecondaryBTreeCreator(PhysicalOptimizationConfig physOptConf,
+ protected SecondaryBTreeOperationsHelper(PhysicalOptimizationConfig physOptConf,
IAsterixPropertiesProvider propertiesProvider) {
super(physOptConf, propertiesProvider);
}
@@ -126,4 +127,25 @@
spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
return spec;
}
+
+ @Override
+ public JobSpecification buildCompactJobSpec() throws AsterixException, AlgebricksException {
+ JobSpecification spec = JobSpecificationUtils.createJobSpecification();
+
+ AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
+ LSMTreeIndexCompactOperatorDescriptor compactOp = new LSMTreeIndexCompactOperatorDescriptor(spec,
+ AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+ secondaryFileSplitProvider, secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories,
+ secondaryBloomFilterKeyFields, new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(
+ dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+ new SecondaryIndexOperationTrackerProvider(LSMBTreeIOOperationCallbackFactory.INSTANCE, dataset
+ .getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+ AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+ storageProperties.getBloomFilterFalsePositiveRate()), NoOpOperationCallbackFactory.INSTANCE);
+ AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, compactOp,
+ secondaryPartitionConstraint);
+ spec.addRoot(compactOp);
+ spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
+ return spec;
+ }
}
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexCreator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexOperationsHelper.java
similarity index 91%
rename from asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexCreator.java
rename to asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexOperationsHelper.java
index a9b3881..0a830d6 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexCreator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexOperationsHelper.java
@@ -21,6 +21,7 @@
import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
+import edu.uci.ics.asterix.common.config.DatasetConfig.IndexType;
import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
import edu.uci.ics.asterix.common.context.AsterixVirtualBufferCacheProvider;
import edu.uci.ics.asterix.common.context.ITransactionSubsystemProvider;
@@ -48,7 +49,6 @@
import edu.uci.ics.asterix.transaction.management.opcallbacks.PrimaryIndexOperationTrackerProvider;
import edu.uci.ics.asterix.transaction.management.service.transaction.AsterixRuntimeComponentsProvider;
import edu.uci.ics.asterix.transaction.management.service.transaction.JobIdFactory;
-import edu.uci.ics.asterix.translator.CompiledStatements.CompiledCreateIndexStatement;
import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -89,7 +89,7 @@
// TODO: We should eventually have a hierarchy of classes that can create all
// possible index job specs,
// not just for creation.
-public abstract class SecondaryIndexCreator {
+public abstract class SecondaryIndexOperationsHelper {
protected final PhysicalOptimizationConfig physOptConf;
protected int numPrimaryKeys;
@@ -119,52 +119,55 @@
protected IAsterixPropertiesProvider propertiesProvider;
// Prevent public construction. Should be created via createIndexCreator().
- protected SecondaryIndexCreator(PhysicalOptimizationConfig physOptConf,
+ protected SecondaryIndexOperationsHelper(PhysicalOptimizationConfig physOptConf,
IAsterixPropertiesProvider propertiesProvider) {
this.physOptConf = physOptConf;
this.propertiesProvider = propertiesProvider;
}
- public static SecondaryIndexCreator createIndexCreator(CompiledCreateIndexStatement createIndexStmt,
+ public static SecondaryIndexOperationsHelper createIndexOperationsHelper(IndexType indexType, String dataverseName,
+ String datasetName, String indexName, List<String> secondaryKeyFields, int gramLength,
AqlMetadataProvider metadataProvider, PhysicalOptimizationConfig physOptConf) throws AsterixException,
AlgebricksException {
IAsterixPropertiesProvider asterixPropertiesProvider = AsterixAppContextInfo.getInstance();
- SecondaryIndexCreator indexCreator = null;
- switch (createIndexStmt.getIndexType()) {
+ SecondaryIndexOperationsHelper indexOperationsHelper = null;
+ switch (indexType) {
case BTREE: {
- indexCreator = new SecondaryBTreeCreator(physOptConf, asterixPropertiesProvider);
+ indexOperationsHelper = new SecondaryBTreeOperationsHelper(physOptConf, asterixPropertiesProvider);
break;
}
case RTREE: {
- indexCreator = new SecondaryRTreeCreator(physOptConf, asterixPropertiesProvider);
+ indexOperationsHelper = new SecondaryRTreeOperationsHelper(physOptConf, asterixPropertiesProvider);
break;
}
case SINGLE_PARTITION_WORD_INVIX:
case SINGLE_PARTITION_NGRAM_INVIX:
case LENGTH_PARTITIONED_WORD_INVIX:
case LENGTH_PARTITIONED_NGRAM_INVIX: {
- indexCreator = new SecondaryInvertedIndexCreator(physOptConf, asterixPropertiesProvider);
+ indexOperationsHelper = new SecondaryInvertedIndexOperationsHelper(physOptConf, asterixPropertiesProvider);
break;
}
default: {
- throw new AsterixException("Unknown Index Type: " + createIndexStmt.getIndexType());
+ throw new AsterixException("Unknown Index Type: " + indexType);
}
}
- indexCreator.init(createIndexStmt, metadataProvider);
- return indexCreator;
+ indexOperationsHelper.init(indexType, dataverseName, datasetName, indexName, secondaryKeyFields, gramLength,
+ metadataProvider);
+ return indexOperationsHelper;
}
public abstract JobSpecification buildCreationJobSpec() throws AsterixException, AlgebricksException;
public abstract JobSpecification buildLoadingJobSpec() throws AsterixException, AlgebricksException;
- protected void init(CompiledCreateIndexStatement createIndexStmt, AqlMetadataProvider metadataProvider)
- throws AsterixException, AlgebricksException {
+ public abstract JobSpecification buildCompactJobSpec() throws AsterixException, AlgebricksException;
+
+ protected void init(IndexType indexType, String dvn, String dsn, String in, List<String> secondaryKeyFields,
+ int gramLength, AqlMetadataProvider metadataProvider) throws AsterixException, AlgebricksException {
this.metadataProvider = metadataProvider;
- dataverseName = createIndexStmt.getDataverseName() == null ? metadataProvider.getDefaultDataverseName()
- : createIndexStmt.getDataverseName();
- datasetName = createIndexStmt.getDatasetName();
- secondaryIndexName = createIndexStmt.getIndexName();
+ dataverseName = dvn == null ? metadataProvider.getDefaultDataverseName() : dvn;
+ datasetName = dsn;
+ secondaryIndexName = in;
dataset = metadataProvider.findDataset(dataverseName, datasetName);
if (dataset == null) {
throw new AsterixException("Unknown dataset " + datasetName);
@@ -175,7 +178,7 @@
itemType = (ARecordType) metadataProvider.findType(dataset.getDataverseName(), dataset.getItemTypeName());
payloadSerde = AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(itemType);
numPrimaryKeys = DatasetUtils.getPartitioningKeys(dataset).size();
- numSecondaryKeys = createIndexStmt.getKeyFields().size();
+ numSecondaryKeys = secondaryKeyFields.size();
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> primarySplitsAndConstraint = metadataProvider
.splitProviderAndPartitionConstraintsForInternalOrFeedDataset(dataverseName, datasetName, datasetName);
primaryFileSplitProvider = primarySplitsAndConstraint.first;
@@ -187,7 +190,7 @@
secondaryPartitionConstraint = secondarySplitsAndConstraint.second;
// Must be called in this order.
setPrimaryRecDescAndComparators();
- setSecondaryRecDescAndComparators(createIndexStmt, metadataProvider);
+ setSecondaryRecDescAndComparators(indexType, secondaryKeyFields, gramLength, metadataProvider);
numElementsHint = metadataProvider.getCardinalityPerPartitionHint(dataset);
}
@@ -217,9 +220,8 @@
primaryRecDesc = new RecordDescriptor(primaryRecFields, primaryTypeTraits);
}
- protected void setSecondaryRecDescAndComparators(CompiledCreateIndexStatement createIndexStmt,
- AqlMetadataProvider metadataProvider) throws AlgebricksException, AsterixException {
- List<String> secondaryKeyFields = createIndexStmt.getKeyFields();
+ protected void setSecondaryRecDescAndComparators(IndexType indexType, List<String> secondaryKeyFields,
+ int gramLength, AqlMetadataProvider metadataProvider) throws AlgebricksException, AsterixException {
secondaryFieldAccessEvalFactories = new ICopyEvaluatorFactory[numSecondaryKeys];
secondaryComparatorFactories = new IBinaryComparatorFactory[numSecondaryKeys + numPrimaryKeys];
secondaryBloomFilterKeyFields = new int[numSecondaryKeys];
@@ -391,4 +393,4 @@
primaryPartitionConstraint);
return asterixSelectOp;
}
-}
+}
\ No newline at end of file
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexOperationsHelper.java
similarity index 90%
rename from asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java
rename to asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexOperationsHelper.java
index 40e0aa9..c11122b 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexOperationsHelper.java
@@ -32,7 +32,6 @@
import edu.uci.ics.asterix.transaction.management.resource.LSMInvertedIndexLocalResourceMetadata;
import edu.uci.ics.asterix.transaction.management.resource.PersistentLocalResourceFactoryProvider;
import edu.uci.ics.asterix.transaction.management.service.transaction.AsterixRuntimeComponentsProvider;
-import edu.uci.ics.asterix.translator.CompiledStatements.CompiledCreateIndexStatement;
import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
@@ -58,6 +57,7 @@
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.dataflow.BinaryTokenizerOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.dataflow.LSMInvertedIndexBulkLoadOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.dataflow.LSMInvertedIndexCompactOperator;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.dataflow.LSMInvertedIndexCreateOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.dataflow.LSMInvertedIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.dataflow.PartitionedLSMInvertedIndexDataflowHelperFactory;
@@ -65,7 +65,7 @@
import edu.uci.ics.hyracks.storage.common.file.ILocalResourceFactoryProvider;
import edu.uci.ics.hyracks.storage.common.file.LocalResource;
-public class SecondaryInvertedIndexCreator extends SecondaryIndexCreator {
+public class SecondaryInvertedIndexOperationsHelper extends SecondaryIndexOperationsHelper {
private IAType secondaryKeyType;
private ITypeTraits[] invListsTypeTraits;
@@ -78,15 +78,15 @@
private RecordDescriptor tokenKeyPairRecDesc;
private boolean isPartitioned;
- protected SecondaryInvertedIndexCreator(PhysicalOptimizationConfig physOptConf,
+ protected SecondaryInvertedIndexOperationsHelper(PhysicalOptimizationConfig physOptConf,
IAsterixPropertiesProvider propertiesProvider) {
super(physOptConf, propertiesProvider);
}
@Override
@SuppressWarnings("rawtypes")
- protected void setSecondaryRecDescAndComparators(CompiledCreateIndexStatement createIndexStmt,
- AqlMetadataProvider metadata) throws AlgebricksException, AsterixException {
+ protected void setSecondaryRecDescAndComparators(IndexType indexType, List<String> secondaryKeyFields,
+ int gramLength, AqlMetadataProvider metadata) throws AlgebricksException, AsterixException {
// Sanity checks.
if (numPrimaryKeys > 1) {
throw new AsterixException("Cannot create inverted index on dataset with composite primary key.");
@@ -94,15 +94,14 @@
if (numSecondaryKeys > 1) {
throw new AsterixException("Cannot create composite inverted index on multiple fields.");
}
- if (createIndexStmt.getIndexType() == IndexType.LENGTH_PARTITIONED_WORD_INVIX
- || createIndexStmt.getIndexType() == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX) {
+ if (indexType == IndexType.LENGTH_PARTITIONED_WORD_INVIX
+ || indexType == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX) {
isPartitioned = true;
} else {
isPartitioned = false;
}
// Prepare record descriptor used in the assign op, and the optional
// select op.
- List<String> secondaryKeyFields = createIndexStmt.getKeyFields();
secondaryFieldAccessEvalFactories = new ICopyEvaluatorFactory[numSecondaryKeys];
ISerializerDeserializer[] secondaryRecFields = new ISerializerDeserializer[numPrimaryKeys + numSecondaryKeys];
ITypeTraits[] secondaryTypeTraits = new ITypeTraits[numSecondaryKeys + numPrimaryKeys];
@@ -133,8 +132,8 @@
// Set tokenizer factory.
// TODO: We might want to expose the hashing option at the AQL level,
// and add the choice to the index metadata.
- tokenizerFactory = NonTaggedFormatUtil.getBinaryTokenizerFactory(secondaryKeyType.getTypeTag(),
- createIndexStmt.getIndexType(), createIndexStmt.getGramLength());
+ tokenizerFactory = NonTaggedFormatUtil.getBinaryTokenizerFactory(secondaryKeyType.getTypeTag(), indexType,
+ gramLength);
// Type traits for inverted-list elements. Inverted lists contain
// primary keys.
invListsTypeTraits = new ITypeTraits[numPrimaryKeys];
@@ -296,4 +295,22 @@
storageProperties.getBloomFilterFalsePositiveRate());
}
}
+
+ @Override
+ public JobSpecification buildCompactJobSpec() throws AsterixException, AlgebricksException {
+ JobSpecification spec = JobSpecificationUtils.createJobSpecification();
+
+ IIndexDataflowHelperFactory dataflowHelperFactory = createDataflowHelperFactory();
+ LSMInvertedIndexCompactOperator compactOp = new LSMInvertedIndexCompactOperator(spec,
+ AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, secondaryFileSplitProvider,
+ AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, tokenTypeTraits, tokenComparatorFactories,
+ invListsTypeTraits, primaryComparatorFactories, tokenizerFactory, dataflowHelperFactory,
+ NoOpOperationCallbackFactory.INSTANCE);
+ AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, compactOp,
+ secondaryPartitionConstraint);
+
+ spec.addRoot(compactOp);
+ spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
+ return spec;
+ }
}
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeCreator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeOperationsHelper.java
similarity index 83%
rename from asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeCreator.java
rename to asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeOperationsHelper.java
index ec62068..bd4917c 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeCreator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeOperationsHelper.java
@@ -18,6 +18,7 @@
import edu.uci.ics.asterix.common.api.ILocalResourceMetadata;
import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
+import edu.uci.ics.asterix.common.config.DatasetConfig.IndexType;
import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
import edu.uci.ics.asterix.common.context.AsterixVirtualBufferCacheProvider;
import edu.uci.ics.asterix.common.exceptions.AsterixException;
@@ -35,7 +36,6 @@
import edu.uci.ics.asterix.transaction.management.resource.LSMRTreeLocalResourceMetadata;
import edu.uci.ics.asterix.transaction.management.resource.PersistentLocalResourceFactoryProvider;
import edu.uci.ics.asterix.transaction.management.service.transaction.AsterixRuntimeComponentsProvider;
-import edu.uci.ics.asterix.translator.CompiledStatements.CompiledCreateIndexStatement;
import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
@@ -55,19 +55,20 @@
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexCreateOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.dataflow.LSMTreeIndexCompactOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.lsm.rtree.dataflow.LSMRTreeDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreePolicyType;
import edu.uci.ics.hyracks.storage.common.file.ILocalResourceFactoryProvider;
import edu.uci.ics.hyracks.storage.common.file.LocalResource;
@SuppressWarnings("rawtypes")
-public class SecondaryRTreeCreator extends SecondaryIndexCreator {
+public class SecondaryRTreeOperationsHelper extends SecondaryIndexOperationsHelper {
protected IPrimitiveValueProviderFactory[] valueProviderFactories;
protected int numNestedSecondaryKeyFields;
protected ATypeTag keyType;
- protected SecondaryRTreeCreator(PhysicalOptimizationConfig physOptConf,
+ protected SecondaryRTreeOperationsHelper(PhysicalOptimizationConfig physOptConf,
IAsterixPropertiesProvider propertiesProvider) {
super(physOptConf, propertiesProvider);
}
@@ -105,9 +106,8 @@
}
@Override
- protected void setSecondaryRecDescAndComparators(CompiledCreateIndexStatement createIndexStmt,
- AqlMetadataProvider metadata) throws AlgebricksException, AsterixException {
- List<String> secondaryKeyFields = createIndexStmt.getKeyFields();
+ protected void setSecondaryRecDescAndComparators(IndexType indexType, List<String> secondaryKeyFields,
+ int gramLength, AqlMetadataProvider metadata) throws AlgebricksException, AsterixException {
int numSecondaryKeys = secondaryKeyFields.size();
if (numSecondaryKeys != 1) {
throw new AsterixException(
@@ -196,4 +196,27 @@
spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
return spec;
}
+
+ @Override
+ public JobSpecification buildCompactJobSpec() throws AsterixException, AlgebricksException {
+ JobSpecification spec = JobSpecificationUtils.createJobSpecification();
+
+ AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
+ LSMTreeIndexCompactOperatorDescriptor compactOp = new LSMTreeIndexCompactOperatorDescriptor(spec,
+ AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+ secondaryFileSplitProvider, secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories,
+ secondaryBloomFilterKeyFields, new LSMRTreeDataflowHelperFactory(valueProviderFactories,
+ RTreePolicyType.RTREE, primaryComparatorFactories, new AsterixVirtualBufferCacheProvider(
+ dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+ new SecondaryIndexOperationTrackerProvider(LSMRTreeIOOperationCallbackFactory.INSTANCE, dataset
+ .getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+ AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AqlMetadataProvider.proposeLinearizer(
+ keyType, secondaryComparatorFactories.length),
+ storageProperties.getBloomFilterFalsePositiveRate()), NoOpOperationCallbackFactory.INSTANCE);
+ AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, compactOp,
+ secondaryPartitionConstraint);
+ spec.addRoot(compactOp);
+ spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
+ return spec;
+ }
}
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/compact-dataset-and-its-indexes/compact-dataset-and-its-indexes.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/compact-dataset-and-its-indexes/compact-dataset-and-its-indexes.1.ddl.aql
new file mode 100644
index 0000000..2f54c58
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/compact-dataset-and-its-indexes/compact-dataset-and-its-indexes.1.ddl.aql
@@ -0,0 +1,35 @@
+/*
+ * Test case Name : compact-dataset-and-its-indexes.aql
+ * Description : This test is intended to test the compact statement which merge the disk components of a dataset and
+ * all of its indexes.
+ * Expected Result : Success
+ * Date : Sep 19 2013
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type LineItemType as closed {
+ l_orderkey: int32,
+ l_partkey: int32,
+ l_suppkey: int32,
+ l_linenumber: int32,
+ l_quantity: int32,
+ l_extendedprice: double,
+ l_discount: double,
+ l_tax: double,
+ l_returnflag: string,
+ l_linestatus: string,
+ l_shipdate: string,
+ l_commitdate: string,
+ l_receiptdate: string,
+ l_shipinstruct: string,
+ l_shipmode: string,
+ l_comment: string
+}
+
+create dataset LineItem(LineItemType)
+ primary key l_orderkey, l_linenumber;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/compact-dataset-and-its-indexes/compact-dataset-and-its-indexes.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/compact-dataset-and-its-indexes/compact-dataset-and-its-indexes.2.update.aql
new file mode 100644
index 0000000..169c6dc
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/compact-dataset-and-its-indexes/compact-dataset-and-its-indexes.2.update.aql
@@ -0,0 +1,6 @@
+use dataverse test;
+
+load dataset LineItem
+using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/tpch0.001/lineitem.tbl"),("format"="delimited-text"),("delimiter"="|")) pre-sorted;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/compact-dataset-and-its-indexes/compact-dataset-and-its-indexes.3.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/compact-dataset-and-its-indexes/compact-dataset-and-its-indexes.3.ddl.aql
new file mode 100644
index 0000000..a3572c4
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/compact-dataset-and-its-indexes/compact-dataset-and-its-indexes.3.ddl.aql
@@ -0,0 +1,5 @@
+use dataverse test;
+
+create index idx_LineItem_partkey on LineItem(l_linenumber);
+create index idx_LineItem_suppkey on LineItem(l_suppkey);
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/compact-dataset-and-its-indexes/compact-dataset-and-its-indexes.4.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/compact-dataset-and-its-indexes/compact-dataset-and-its-indexes.4.update.aql
new file mode 100644
index 0000000..c3bc7d7
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/compact-dataset-and-its-indexes/compact-dataset-and-its-indexes.4.update.aql
@@ -0,0 +1,5 @@
+use dataverse test;
+
+delete $l from dataset LineItem where $l.l_suppkey>=2 or $l.l_linenumber>1;
+
+compact dataset LineItem;
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/compact-dataset-and-its-indexes/compact-dataset-and-its-indexes.5.query.aql b/asterix-app/src/test/resources/runtimets/queries/dml/compact-dataset-and-its-indexes/compact-dataset-and-its-indexes.5.query.aql
new file mode 100644
index 0000000..69912f9
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/compact-dataset-and-its-indexes/compact-dataset-and-its-indexes.5.query.aql
@@ -0,0 +1,6 @@
+use dataverse test;
+
+for $c in dataset('LineItem')
+where $c.l_suppkey<150
+order by $c.l_orderkey, $c.l_linenumber
+return $c
diff --git a/asterix-app/src/test/resources/runtimets/results/dml/compact-dataset-and-its-indexes/compact-dataset-and-its-indexes.1.adm b/asterix-app/src/test/resources/runtimets/results/dml/compact-dataset-and-its-indexes/compact-dataset-and-its-indexes.1.adm
new file mode 100644
index 0000000..0d69984
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/dml/compact-dataset-and-its-indexes/compact-dataset-and-its-indexes.1.adm
@@ -0,0 +1,167 @@
+{ "l_orderkey": 36, "l_partkey": 120, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 42, "l_extendedprice": 42845.04d, "l_discount": 0.09d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-02-03", "l_commitdate": "1996-01-21", "l_receiptdate": "1996-02-23", "l_shipinstruct": "COLLECT COD", "l_shipmode": "SHIP", "l_comment": " careful courts. special " }
+{ "l_orderkey": 68, "l_partkey": 8, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 3, "l_extendedprice": 2724.0d, "l_discount": 0.05d, "l_tax": 0.02d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-07-04", "l_commitdate": "1998-06-05", "l_receiptdate": "1998-07-21", "l_shipinstruct": "NONE", "l_shipmode": "RAIL", "l_comment": "fully special instructions cajole. furious" }
+{ "l_orderkey": 162, "l_partkey": 190, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 2, "l_extendedprice": 2180.38d, "l_discount": 0.02d, "l_tax": 0.01d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-09-02", "l_commitdate": "1995-06-17", "l_receiptdate": "1995-09-08", "l_shipinstruct": "COLLECT COD", "l_shipmode": "FOB", "l_comment": "es! final somas integrate" }
+{ "l_orderkey": 192, "l_partkey": 98, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 23, "l_extendedprice": 22956.07d, "l_discount": 0.0d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-02-05", "l_commitdate": "1998-02-06", "l_receiptdate": "1998-03-07", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "AIR", "l_comment": "ly pending theodolites haggle quickly fluf" }
+{ "l_orderkey": 197, "l_partkey": 99, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 39, "l_extendedprice": 38964.51d, "l_discount": 0.02d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-07-21", "l_commitdate": "1995-07-01", "l_receiptdate": "1995-08-14", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "AIR", "l_comment": "press accounts. daringly sp" }
+{ "l_orderkey": 227, "l_partkey": 166, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 19, "l_extendedprice": 20257.04d, "l_discount": 0.05d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-12-10", "l_commitdate": "1996-01-30", "l_receiptdate": "1995-12-26", "l_shipinstruct": "NONE", "l_shipmode": "RAIL", "l_comment": "s cajole furiously a" }
+{ "l_orderkey": 290, "l_partkey": 6, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 35, "l_extendedprice": 31710.0d, "l_discount": 0.01d, "l_tax": 0.02d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-04-01", "l_commitdate": "1994-02-05", "l_receiptdate": "1994-04-27", "l_shipinstruct": "NONE", "l_shipmode": "MAIL", "l_comment": "ove the final foxes detect slyly fluffily" }
+{ "l_orderkey": 325, "l_partkey": 159, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 34, "l_extendedprice": 36011.1d, "l_discount": 0.09d, "l_tax": 0.04d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1993-10-28", "l_commitdate": "1993-12-13", "l_receiptdate": "1993-11-17", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "MAIL", "l_comment": "ly bold deposits. always iron" }
+{ "l_orderkey": 355, "l_partkey": 114, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 31, "l_extendedprice": 31437.41d, "l_discount": 0.09d, "l_tax": 0.07d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-07-13", "l_commitdate": "1994-08-18", "l_receiptdate": "1994-07-18", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "FOB", "l_comment": "y unusual, ironic" }
+{ "l_orderkey": 389, "l_partkey": 190, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 2, "l_extendedprice": 2180.38d, "l_discount": 0.09d, "l_tax": 0.0d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-04-13", "l_commitdate": "1994-04-10", "l_receiptdate": "1994-04-25", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": "fts. courts eat blithely even dependenc" }
+{ "l_orderkey": 391, "l_partkey": 122, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 14, "l_extendedprice": 14309.68d, "l_discount": 0.09d, "l_tax": 0.02d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1995-02-11", "l_commitdate": "1995-02-03", "l_receiptdate": "1995-02-13", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "TRUCK", "l_comment": " escapades sleep furiously about " }
+{ "l_orderkey": 417, "l_partkey": 40, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 39, "l_extendedprice": 36661.56d, "l_discount": 0.01d, "l_tax": 0.02d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-05-31", "l_commitdate": "1994-05-02", "l_receiptdate": "1994-06-06", "l_shipinstruct": "NONE", "l_shipmode": "SHIP", "l_comment": "y regular requests wake along " }
+{ "l_orderkey": 453, "l_partkey": 198, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 45, "l_extendedprice": 49418.55d, "l_discount": 0.01d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-06-30", "l_commitdate": "1997-08-20", "l_receiptdate": "1997-07-19", "l_shipinstruct": "COLLECT COD", "l_shipmode": "REG AIR", "l_comment": "ifts wake carefully." }
+{ "l_orderkey": 485, "l_partkey": 150, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 50, "l_extendedprice": 52507.5d, "l_discount": 0.01d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-03-28", "l_commitdate": "1997-05-26", "l_receiptdate": "1997-04-18", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "MAIL", "l_comment": "iously quick excuses. carefully final f" }
+{ "l_orderkey": 545, "l_partkey": 170, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 4, "l_extendedprice": 4280.68d, "l_discount": 0.02d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-02-23", "l_commitdate": "1995-12-16", "l_receiptdate": "1996-03-21", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "FOB", "l_comment": ", ironic grouches cajole over" }
+{ "l_orderkey": 581, "l_partkey": 64, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 41, "l_extendedprice": 39526.46d, "l_discount": 0.09d, "l_tax": 0.07d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-05-26", "l_commitdate": "1997-04-06", "l_receiptdate": "1997-06-10", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "MAIL", "l_comment": "nts. quickly" }
+{ "l_orderkey": 647, "l_partkey": 17, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 41, "l_extendedprice": 37597.41d, "l_discount": 0.08d, "l_tax": 0.08d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-11-19", "l_commitdate": "1997-09-24", "l_receiptdate": "1997-12-15", "l_shipinstruct": "COLLECT COD", "l_shipmode": "REG AIR", "l_comment": "r instructions. quickly unusu" }
+{ "l_orderkey": 704, "l_partkey": 190, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 40, "l_extendedprice": 43607.6d, "l_discount": 0.05d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-01-30", "l_commitdate": "1997-01-10", "l_receiptdate": "1997-02-20", "l_shipinstruct": "COLLECT COD", "l_shipmode": "AIR", "l_comment": "ggle quickly. r" }
+{ "l_orderkey": 738, "l_partkey": 198, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 34, "l_extendedprice": 37338.46d, "l_discount": 0.0d, "l_tax": 0.06d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-06-09", "l_commitdate": "1993-04-15", "l_receiptdate": "1993-07-09", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "TRUCK", "l_comment": "s against the ironic exc" }
+{ "l_orderkey": 773, "l_partkey": 100, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 5, "l_extendedprice": 5000.5d, "l_discount": 0.06d, "l_tax": 0.04d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1993-11-21", "l_commitdate": "1993-12-19", "l_receiptdate": "1993-12-21", "l_shipinstruct": "COLLECT COD", "l_shipmode": "MAIL", "l_comment": "ar requests. regular, thin packages u" }
+{ "l_orderkey": 800, "l_partkey": 72, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 38, "l_extendedprice": 36938.66d, "l_discount": 0.0d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-07-21", "l_commitdate": "1998-09-25", "l_receiptdate": "1998-08-07", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "TRUCK", "l_comment": "according to the bold, final dependencies " }
+{ "l_orderkey": 900, "l_partkey": 199, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 44, "l_extendedprice": 48364.36d, "l_discount": 0.01d, "l_tax": 0.06d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-12-15", "l_commitdate": "1994-12-03", "l_receiptdate": "1994-12-27", "l_shipinstruct": "COLLECT COD", "l_shipmode": "MAIL", "l_comment": " detect quick" }
+{ "l_orderkey": 931, "l_partkey": 40, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 18, "l_extendedprice": 16920.72d, "l_discount": 0.0d, "l_tax": 0.05d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1993-04-04", "l_commitdate": "1993-01-11", "l_receiptdate": "1993-04-13", "l_shipinstruct": "NONE", "l_shipmode": "RAIL", "l_comment": "slyly ironic re" }
+{ "l_orderkey": 932, "l_partkey": 44, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 41, "l_extendedprice": 38705.64d, "l_discount": 0.01d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-06-05", "l_commitdate": "1997-07-22", "l_receiptdate": "1997-06-26", "l_shipinstruct": "COLLECT COD", "l_shipmode": "RAIL", "l_comment": "foxes. ironic pl" }
+{ "l_orderkey": 965, "l_partkey": 108, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 20, "l_extendedprice": 20162.0d, "l_discount": 0.04d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "F", "l_shipdate": "1995-06-16", "l_commitdate": "1995-07-20", "l_receiptdate": "1995-07-06", "l_shipinstruct": "COLLECT COD", "l_shipmode": "MAIL", "l_comment": "kly. carefully pending requ" }
+{ "l_orderkey": 995, "l_partkey": 173, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 15, "l_extendedprice": 16097.55d, "l_discount": 0.08d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-06-30", "l_commitdate": "1995-08-04", "l_receiptdate": "1995-07-27", "l_shipinstruct": "NONE", "l_shipmode": "REG AIR", "l_comment": "uses. fluffily fina" }
+{ "l_orderkey": 1025, "l_partkey": 150, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 36, "l_extendedprice": 37805.4d, "l_discount": 0.03d, "l_tax": 0.04d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1995-05-15", "l_commitdate": "1995-07-05", "l_receiptdate": "1995-06-10", "l_shipinstruct": "COLLECT COD", "l_shipmode": "FOB", "l_comment": "e unusual, regular instr" }
+{ "l_orderkey": 1027, "l_partkey": 156, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 43, "l_extendedprice": 45414.45d, "l_discount": 0.07d, "l_tax": 0.08d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1992-06-17", "l_commitdate": "1992-08-28", "l_receiptdate": "1992-07-10", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "MAIL", "l_comment": "oxes. carefully regular deposits" }
+{ "l_orderkey": 1155, "l_partkey": 70, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 4, "l_extendedprice": 3880.28d, "l_discount": 0.09d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-10-19", "l_commitdate": "1997-12-09", "l_receiptdate": "1997-11-02", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "SHIP", "l_comment": "ic foxes according to the carefully final " }
+{ "l_orderkey": 1185, "l_partkey": 72, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 8, "l_extendedprice": 7776.56d, "l_discount": 0.01d, "l_tax": 0.06d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1992-12-05", "l_commitdate": "1992-10-05", "l_receiptdate": "1992-12-28", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "MAIL", "l_comment": "ely according to the furiously regular r" }
+{ "l_orderkey": 1216, "l_partkey": 97, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 8, "l_extendedprice": 7976.72d, "l_discount": 0.03d, "l_tax": 0.04d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-02-01", "l_commitdate": "1993-03-06", "l_receiptdate": "1993-02-08", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "TRUCK", "l_comment": " of the carefully express" }
+{ "l_orderkey": 1223, "l_partkey": 100, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 28, "l_extendedprice": 28002.8d, "l_discount": 0.1d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-08-07", "l_commitdate": "1996-07-24", "l_receiptdate": "1996-08-13", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "MAIL", "l_comment": " quickly ironic requests. furious" }
+{ "l_orderkey": 1381, "l_partkey": 144, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 47, "l_extendedprice": 49074.58d, "l_discount": 0.08d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-09-22", "l_commitdate": "1998-08-12", "l_receiptdate": "1998-10-12", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "AIR", "l_comment": "ly ironic deposits" }
+{ "l_orderkey": 1409, "l_partkey": 99, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 23, "l_extendedprice": 22979.07d, "l_discount": 0.01d, "l_tax": 0.03d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1993-04-18", "l_commitdate": "1993-02-25", "l_receiptdate": "1993-05-06", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "FOB", "l_comment": "ions. slyly ironic packages wake quick" }
+{ "l_orderkey": 1445, "l_partkey": 100, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 24, "l_extendedprice": 24002.4d, "l_discount": 0.01d, "l_tax": 0.0d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1995-02-21", "l_commitdate": "1995-02-22", "l_receiptdate": "1995-03-18", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "SHIP", "l_comment": "al accounts use furiously a" }
+{ "l_orderkey": 1477, "l_partkey": 72, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 31, "l_extendedprice": 30134.17d, "l_discount": 0.0d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-12-16", "l_commitdate": "1997-09-30", "l_receiptdate": "1997-12-17", "l_shipinstruct": "COLLECT COD", "l_shipmode": "RAIL", "l_comment": " requests. fluffily final " }
+{ "l_orderkey": 1540, "l_partkey": 173, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 38, "l_extendedprice": 40780.46d, "l_discount": 0.03d, "l_tax": 0.01d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1992-09-30", "l_commitdate": "1992-10-27", "l_receiptdate": "1992-10-12", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "SHIP", "l_comment": " final grouches bo" }
+{ "l_orderkey": 1568, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 36, "l_extendedprice": 35643.24d, "l_discount": 0.02d, "l_tax": 0.03d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-05-31", "l_commitdate": "1997-04-22", "l_receiptdate": "1997-06-21", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": "platelets-- furiously sly excu" }
+{ "l_orderkey": 1605, "l_partkey": 142, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 47, "l_extendedprice": 48980.58d, "l_discount": 0.0d, "l_tax": 0.01d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-04-29", "l_commitdate": "1998-06-12", "l_receiptdate": "1998-05-20", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "AIR", "l_comment": ". carefully r" }
+{ "l_orderkey": 1607, "l_partkey": 190, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 2, "l_extendedprice": 2180.38d, "l_discount": 0.02d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-01-11", "l_commitdate": "1996-02-15", "l_receiptdate": "1996-01-19", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "MAIL", "l_comment": "packages haggle. regular requests boost s" }
+{ "l_orderkey": 1635, "l_partkey": 71, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 3, "l_extendedprice": 2913.21d, "l_discount": 0.06d, "l_tax": 0.08d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-03-13", "l_commitdate": "1997-03-25", "l_receiptdate": "1997-03-27", "l_shipinstruct": "COLLECT COD", "l_shipmode": "FOB", "l_comment": " quickly ironic r" }
+{ "l_orderkey": 1700, "l_partkey": 140, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 38, "l_extendedprice": 39525.32d, "l_discount": 0.04d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-10-03", "l_commitdate": "1996-07-27", "l_receiptdate": "1996-10-22", "l_shipinstruct": "NONE", "l_shipmode": "RAIL", "l_comment": "ular dependencies engage slyly " }
+{ "l_orderkey": 1796, "l_partkey": 10, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 28, "l_extendedprice": 25480.28d, "l_discount": 0.08d, "l_tax": 0.04d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1992-12-01", "l_commitdate": "1993-01-01", "l_receiptdate": "1992-12-24", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "FOB", "l_comment": "y quickly ironic accounts." }
+{ "l_orderkey": 1825, "l_partkey": 156, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 43, "l_extendedprice": 45414.45d, "l_discount": 0.05d, "l_tax": 0.05d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-02-18", "l_commitdate": "1994-02-19", "l_receiptdate": "1994-03-02", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": " accounts breach fluffily spe" }
+{ "l_orderkey": 1827, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 47, "l_extendedprice": 46534.23d, "l_discount": 0.0d, "l_tax": 0.01d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-08-01", "l_commitdate": "1996-08-07", "l_receiptdate": "1996-08-23", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": ". pending courts about the even e" }
+{ "l_orderkey": 1893, "l_partkey": 99, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 43, "l_extendedprice": 42960.87d, "l_discount": 0.1d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-01-25", "l_commitdate": "1998-01-06", "l_receiptdate": "1998-02-14", "l_shipinstruct": "COLLECT COD", "l_shipmode": "SHIP", "l_comment": "he carefully regular " }
+{ "l_orderkey": 1924, "l_partkey": 73, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 7, "l_extendedprice": 6811.49d, "l_discount": 0.06d, "l_tax": 0.07d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-01-01", "l_commitdate": "1996-12-02", "l_receiptdate": "1997-01-08", "l_shipinstruct": "COLLECT COD", "l_shipmode": "SHIP", "l_comment": "osits. even accounts nag furious" }
+{ "l_orderkey": 1953, "l_partkey": 128, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 25, "l_extendedprice": 25703.0d, "l_discount": 0.07d, "l_tax": 0.06d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-01-07", "l_commitdate": "1994-01-28", "l_receiptdate": "1994-01-29", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": "ular, regular i" }
+{ "l_orderkey": 1985, "l_partkey": 28, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 33, "l_extendedprice": 30624.66d, "l_discount": 0.1d, "l_tax": 0.03d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-12-04", "l_commitdate": "1994-11-01", "l_receiptdate": "1994-12-05", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "FOB", "l_comment": "s are express packages. pendin" }
+{ "l_orderkey": 1988, "l_partkey": 72, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 36, "l_extendedprice": 34994.52d, "l_discount": 0.09d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-01-21", "l_commitdate": "1995-11-24", "l_receiptdate": "1996-01-27", "l_shipinstruct": "NONE", "l_shipmode": "RAIL", "l_comment": "gular theodolites. " }
+{ "l_orderkey": 2048, "l_partkey": 35, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 7, "l_extendedprice": 6545.21d, "l_discount": 0.06d, "l_tax": 0.01d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-12-07", "l_commitdate": "1994-01-31", "l_receiptdate": "1994-01-05", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "REG AIR", "l_comment": "lent platelets boost deposits. carefully sp" }
+{ "l_orderkey": 2086, "l_partkey": 60, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 22, "l_extendedprice": 21121.32d, "l_discount": 0.03d, "l_tax": 0.07d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-12-04", "l_commitdate": "1994-12-16", "l_receiptdate": "1994-12-20", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "RAIL", "l_comment": "idly busy acc" }
+{ "l_orderkey": 2118, "l_partkey": 160, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 24, "l_extendedprice": 25443.84d, "l_discount": 0.1d, "l_tax": 0.03d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-01-06", "l_commitdate": "1996-12-14", "l_receiptdate": "1997-01-14", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": "about the slyly bold depende" }
+{ "l_orderkey": 2183, "l_partkey": 71, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 29, "l_extendedprice": 28161.03d, "l_discount": 0.05d, "l_tax": 0.01d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-07-21", "l_commitdate": "1996-08-24", "l_receiptdate": "1996-08-15", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": "ly unusual deposits sleep carefully" }
+{ "l_orderkey": 2211, "l_partkey": 48, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 25, "l_extendedprice": 23701.0d, "l_discount": 0.04d, "l_tax": 0.01d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-10-09", "l_commitdate": "1994-08-04", "l_receiptdate": "1994-11-03", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": "deas. carefully special theodolites along" }
+{ "l_orderkey": 2215, "l_partkey": 73, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 33, "l_extendedprice": 32111.31d, "l_discount": 0.0d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-07-19", "l_commitdate": "1996-08-10", "l_receiptdate": "1996-07-30", "l_shipinstruct": "COLLECT COD", "l_shipmode": "RAIL", "l_comment": "dolites cajole b" }
+{ "l_orderkey": 2272, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 18, "l_extendedprice": 17821.62d, "l_discount": 0.04d, "l_tax": 0.0d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-08-01", "l_commitdate": "1993-07-06", "l_receiptdate": "1993-08-25", "l_shipinstruct": "NONE", "l_shipmode": "MAIL", "l_comment": "ons along the blithely e" }
+{ "l_orderkey": 2342, "l_partkey": 42, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 12, "l_extendedprice": 11304.48d, "l_discount": 0.0d, "l_tax": 0.08d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-07-31", "l_commitdate": "1996-07-26", "l_receiptdate": "1996-08-14", "l_shipinstruct": "NONE", "l_shipmode": "TRUCK", "l_comment": "print blithely even deposits. carefull" }
+{ "l_orderkey": 2343, "l_partkey": 110, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 27, "l_extendedprice": 27272.97d, "l_discount": 0.0d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-11-10", "l_commitdate": "1995-11-17", "l_receiptdate": "1995-12-10", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "SHIP", "l_comment": "old theodolites." }
+{ "l_orderkey": 2439, "l_partkey": 164, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 2, "l_extendedprice": 2128.32d, "l_discount": 0.09d, "l_tax": 0.03d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-04-14", "l_commitdate": "1997-06-11", "l_receiptdate": "1997-05-09", "l_shipinstruct": "COLLECT COD", "l_shipmode": "MAIL", "l_comment": "courts boos" }
+{ "l_orderkey": 2469, "l_partkey": 166, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 11, "l_extendedprice": 11727.76d, "l_discount": 0.0d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-02-09", "l_commitdate": "1997-01-26", "l_receiptdate": "1997-02-16", "l_shipinstruct": "NONE", "l_shipmode": "TRUCK", "l_comment": "ies wake carefully b" }
+{ "l_orderkey": 2592, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 7, "l_extendedprice": 6930.63d, "l_discount": 0.1d, "l_tax": 0.04d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-03-13", "l_commitdate": "1993-04-25", "l_receiptdate": "1993-04-01", "l_shipinstruct": "NONE", "l_shipmode": "REG AIR", "l_comment": " carefully special theodolites integrate " }
+{ "l_orderkey": 2625, "l_partkey": 20, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 42, "l_extendedprice": 38640.84d, "l_discount": 0.02d, "l_tax": 0.04d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1992-10-18", "l_commitdate": "1992-11-17", "l_receiptdate": "1992-10-23", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "AIR", "l_comment": " even accounts haggle furiously" }
+{ "l_orderkey": 2659, "l_partkey": 42, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 28, "l_extendedprice": 26377.12d, "l_discount": 0.08d, "l_tax": 0.05d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-03-17", "l_commitdate": "1994-01-24", "l_receiptdate": "1994-03-19", "l_shipinstruct": "NONE", "l_shipmode": "FOB", "l_comment": "idle tithes" }
+{ "l_orderkey": 2689, "l_partkey": 6, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 45, "l_extendedprice": 40770.0d, "l_discount": 0.02d, "l_tax": 0.04d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1992-04-29", "l_commitdate": "1992-06-22", "l_receiptdate": "1992-04-30", "l_shipinstruct": "COLLECT COD", "l_shipmode": "SHIP", "l_comment": "e quickly. carefully silent" }
+{ "l_orderkey": 2690, "l_partkey": 140, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 44, "l_extendedprice": 45766.16d, "l_discount": 0.05d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-05-30", "l_commitdate": "1996-05-19", "l_receiptdate": "1996-06-26", "l_shipinstruct": "NONE", "l_shipmode": "REG AIR", "l_comment": "ly alongside of th" }
+{ "l_orderkey": 2692, "l_partkey": 17, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 3, "l_extendedprice": 2751.03d, "l_discount": 0.1d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-02-25", "l_commitdate": "1998-01-29", "l_receiptdate": "1998-03-27", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "MAIL", "l_comment": "equests. bold, even foxes haggle slyl" }
+{ "l_orderkey": 2694, "l_partkey": 153, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 30, "l_extendedprice": 31594.5d, "l_discount": 0.02d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-06-20", "l_commitdate": "1996-06-01", "l_receiptdate": "1996-07-15", "l_shipinstruct": "NONE", "l_shipmode": "TRUCK", "l_comment": "oxes. never iro" }
+{ "l_orderkey": 2759, "l_partkey": 59, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 10, "l_extendedprice": 9590.5d, "l_discount": 0.1d, "l_tax": 0.03d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-12-14", "l_commitdate": "1994-01-08", "l_receiptdate": "1994-01-01", "l_shipinstruct": "COLLECT COD", "l_shipmode": "FOB", "l_comment": "s. busily ironic theodo" }
+{ "l_orderkey": 2819, "l_partkey": 70, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 17, "l_extendedprice": 16491.19d, "l_discount": 0.08d, "l_tax": 0.08d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-07-16", "l_commitdate": "1994-07-15", "l_receiptdate": "1994-07-17", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": "en deposits above the f" }
+{ "l_orderkey": 2850, "l_partkey": 97, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 43, "l_extendedprice": 42874.87d, "l_discount": 0.02d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-01-11", "l_commitdate": "1996-11-03", "l_receiptdate": "1997-02-01", "l_shipinstruct": "COLLECT COD", "l_shipmode": "REG AIR", "l_comment": "unusual accounts" }
+{ "l_orderkey": 2886, "l_partkey": 60, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 1, "l_extendedprice": 960.06d, "l_discount": 0.09d, "l_tax": 0.05d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1995-02-01", "l_commitdate": "1994-12-18", "l_receiptdate": "1995-02-28", "l_shipinstruct": "COLLECT COD", "l_shipmode": "REG AIR", "l_comment": "eposits fr" }
+{ "l_orderkey": 2912, "l_partkey": 122, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 8, "l_extendedprice": 8176.96d, "l_discount": 0.06d, "l_tax": 0.04d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1992-04-09", "l_commitdate": "1992-04-19", "l_receiptdate": "1992-04-26", "l_shipinstruct": "NONE", "l_shipmode": "RAIL", "l_comment": "hs cajole over the slyl" }
+{ "l_orderkey": 2944, "l_partkey": 120, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 44, "l_extendedprice": 44885.28d, "l_discount": 0.08d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-12-25", "l_commitdate": "1997-10-28", "l_receiptdate": "1998-01-21", "l_shipinstruct": "COLLECT COD", "l_shipmode": "AIR", "l_comment": "ickly special theodolit" }
+{ "l_orderkey": 2947, "l_partkey": 10, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 37, "l_extendedprice": 33670.37d, "l_discount": 0.09d, "l_tax": 0.07d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-08-09", "l_commitdate": "1995-07-05", "l_receiptdate": "1995-08-20", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "RAIL", "l_comment": "e accounts: expres" }
+{ "l_orderkey": 2950, "l_partkey": 130, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 32, "l_extendedprice": 32964.16d, "l_discount": 0.01d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-09-21", "l_commitdate": "1997-08-25", "l_receiptdate": "1997-10-08", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "REG AIR", "l_comment": "its wake carefully slyly final ideas." }
+{ "l_orderkey": 2978, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 29, "l_extendedprice": 28712.61d, "l_discount": 0.0d, "l_tax": 0.08d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1995-06-03", "l_commitdate": "1995-07-25", "l_receiptdate": "1995-06-06", "l_shipinstruct": "NONE", "l_shipmode": "SHIP", "l_comment": "ecial ideas promise slyly" }
+{ "l_orderkey": 3143, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 22, "l_extendedprice": 21781.98d, "l_discount": 0.02d, "l_tax": 0.0d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1993-05-11", "l_commitdate": "1993-03-26", "l_receiptdate": "1993-05-20", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "MAIL", "l_comment": "l, special instructions nag " }
+{ "l_orderkey": 3264, "l_partkey": 200, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 39, "l_extendedprice": 42907.8d, "l_discount": 0.06d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-11-07", "l_commitdate": "1996-12-12", "l_receiptdate": "1996-11-20", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "REG AIR", "l_comment": "sleep carefully after the slyly final" }
+{ "l_orderkey": 3266, "l_partkey": 64, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 31, "l_extendedprice": 29885.86d, "l_discount": 0.09d, "l_tax": 0.02d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-06-19", "l_commitdate": "1995-05-04", "l_receiptdate": "1995-07-06", "l_shipinstruct": "COLLECT COD", "l_shipmode": "MAIL", "l_comment": "grate among the quickly express deposits" }
+{ "l_orderkey": 3270, "l_partkey": 35, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 11, "l_extendedprice": 10285.33d, "l_discount": 0.07d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-07-29", "l_commitdate": "1997-08-11", "l_receiptdate": "1997-08-05", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "AIR", "l_comment": " solve at the regular deposits. " }
+{ "l_orderkey": 3364, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 49, "l_extendedprice": 48514.41d, "l_discount": 0.03d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-09-17", "l_commitdate": "1997-08-23", "l_receiptdate": "1997-10-06", "l_shipinstruct": "NONE", "l_shipmode": "SHIP", "l_comment": "d accounts? caref" }
+{ "l_orderkey": 3366, "l_partkey": 40, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 4, "l_extendedprice": 3760.16d, "l_discount": 0.07d, "l_tax": 0.01d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-05-20", "l_commitdate": "1997-06-25", "l_receiptdate": "1997-06-03", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "AIR", "l_comment": " carefully about " }
+{ "l_orderkey": 3425, "l_partkey": 120, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 11, "l_extendedprice": 11221.32d, "l_discount": 0.03d, "l_tax": 0.08d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-04-24", "l_commitdate": "1996-05-29", "l_receiptdate": "1996-05-23", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "FOB", "l_comment": "ckly final deposits use quickly?" }
+{ "l_orderkey": 3460, "l_partkey": 11, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 40, "l_extendedprice": 36440.4d, "l_discount": 0.1d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-12-28", "l_commitdate": "1995-12-14", "l_receiptdate": "1996-01-02", "l_shipinstruct": "NONE", "l_shipmode": "REG AIR", "l_comment": "odolites are slyly bold deposits" }
+{ "l_orderkey": 3494, "l_partkey": 117, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 40, "l_extendedprice": 40684.4d, "l_discount": 0.05d, "l_tax": 0.04d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-07-10", "l_commitdate": "1993-06-01", "l_receiptdate": "1993-07-25", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "TRUCK", "l_comment": "lites haggle furiously about the fin" }
+{ "l_orderkey": 3520, "l_partkey": 28, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 30, "l_extendedprice": 27840.6d, "l_discount": 0.04d, "l_tax": 0.02d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-11-11", "l_commitdate": "1997-10-02", "l_receiptdate": "1997-12-06", "l_shipinstruct": "COLLECT COD", "l_shipmode": "SHIP", "l_comment": "deas should solve blithely among the ironi" }
+{ "l_orderkey": 3559, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 29, "l_extendedprice": 28712.61d, "l_discount": 0.0d, "l_tax": 0.07d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1992-12-10", "l_commitdate": "1992-12-03", "l_receiptdate": "1992-12-20", "l_shipinstruct": "COLLECT COD", "l_shipmode": "REG AIR", "l_comment": "l, regular accounts wake flu" }
+{ "l_orderkey": 3585, "l_partkey": 122, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 21, "l_extendedprice": 21464.52d, "l_discount": 0.05d, "l_tax": 0.04d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-12-04", "l_commitdate": "1994-12-25", "l_receiptdate": "1995-01-01", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "TRUCK", "l_comment": "ounts use. express, final platelets us" }
+{ "l_orderkey": 3618, "l_partkey": 140, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 38, "l_extendedprice": 39525.32d, "l_discount": 0.08d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-12-22", "l_commitdate": "1998-02-23", "l_receiptdate": "1998-01-03", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "TRUCK", "l_comment": "nts haggle fluffily above the regular " }
+{ "l_orderkey": 3715, "l_partkey": 97, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 13, "l_extendedprice": 12962.17d, "l_discount": 0.0d, "l_tax": 0.03d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-05-11", "l_commitdate": "1996-04-25", "l_receiptdate": "1996-06-09", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "SHIP", "l_comment": "e quickly ironic" }
+{ "l_orderkey": 3844, "l_partkey": 135, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 2, "l_extendedprice": 2070.26d, "l_discount": 0.03d, "l_tax": 0.07d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1995-02-24", "l_commitdate": "1995-02-03", "l_receiptdate": "1995-03-18", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "AIR", "l_comment": "es haggle final acco" }
+{ "l_orderkey": 3878, "l_partkey": 200, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 6, "l_extendedprice": 6601.2d, "l_discount": 0.07d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-06-21", "l_commitdate": "1997-05-22", "l_receiptdate": "1997-07-01", "l_shipinstruct": "COLLECT COD", "l_shipmode": "FOB", "l_comment": "s. regular instru" }
+{ "l_orderkey": 3906, "l_partkey": 153, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 42, "l_extendedprice": 44232.3d, "l_discount": 0.0d, "l_tax": 0.04d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1992-09-03", "l_commitdate": "1992-07-22", "l_receiptdate": "1992-09-04", "l_shipinstruct": "COLLECT COD", "l_shipmode": "RAIL", "l_comment": "jole blithely after the furiously regular " }
+{ "l_orderkey": 3974, "l_partkey": 22, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 47, "l_extendedprice": 43334.94d, "l_discount": 0.1d, "l_tax": 0.03d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-06-03", "l_commitdate": "1996-05-08", "l_receiptdate": "1996-06-28", "l_shipinstruct": "NONE", "l_shipmode": "TRUCK", "l_comment": "dencies above the re" }
+{ "l_orderkey": 4001, "l_partkey": 106, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 26, "l_extendedprice": 26158.6d, "l_discount": 0.0d, "l_tax": 0.01d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-07-26", "l_commitdate": "1997-06-18", "l_receiptdate": "1997-08-08", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "RAIL", "l_comment": "tegrate blithely" }
+{ "l_orderkey": 4005, "l_partkey": 4, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 26, "l_extendedprice": 23504.0d, "l_discount": 0.09d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-12-01", "l_commitdate": "1997-02-03", "l_receiptdate": "1996-12-15", "l_shipinstruct": "NONE", "l_shipmode": "REG AIR", "l_comment": " to the quic" }
+{ "l_orderkey": 4033, "l_partkey": 110, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 27, "l_extendedprice": 27272.97d, "l_discount": 0.01d, "l_tax": 0.04d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-08-08", "l_commitdate": "1993-08-14", "l_receiptdate": "1993-08-09", "l_shipinstruct": "NONE", "l_shipmode": "AIR", "l_comment": "pinto beans" }
+{ "l_orderkey": 4034, "l_partkey": 190, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 48, "l_extendedprice": 52329.12d, "l_discount": 0.03d, "l_tax": 0.03d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-03-01", "l_commitdate": "1994-01-16", "l_receiptdate": "1994-03-16", "l_shipinstruct": "NONE", "l_shipmode": "RAIL", "l_comment": " blithely regular requests play carefull" }
+{ "l_orderkey": 4036, "l_partkey": 6, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 46, "l_extendedprice": 41676.0d, "l_discount": 0.09d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-06-21", "l_commitdate": "1997-05-29", "l_receiptdate": "1997-07-18", "l_shipinstruct": "NONE", "l_shipmode": "REG AIR", "l_comment": "usly across the even th" }
+{ "l_orderkey": 4064, "l_partkey": 199, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 3, "l_extendedprice": 3297.57d, "l_discount": 0.1d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-01-04", "l_commitdate": "1997-01-01", "l_receiptdate": "1997-01-23", "l_shipinstruct": "NONE", "l_shipmode": "SHIP", "l_comment": "its! quickly sp" }
+{ "l_orderkey": 4067, "l_partkey": 180, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 18, "l_extendedprice": 19443.24d, "l_discount": 0.03d, "l_tax": 0.08d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1993-01-24", "l_commitdate": "1992-12-23", "l_receiptdate": "1993-02-20", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "FOB", "l_comment": "e the slyly final packages d" }
+{ "l_orderkey": 4068, "l_partkey": 110, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 43, "l_extendedprice": 43434.73d, "l_discount": 0.05d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-11-28", "l_commitdate": "1996-11-16", "l_receiptdate": "1996-12-22", "l_shipinstruct": "NONE", "l_shipmode": "AIR", "l_comment": "ructions. regular, special packag" }
+{ "l_orderkey": 4098, "l_partkey": 200, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 46, "l_extendedprice": 50609.2d, "l_discount": 0.07d, "l_tax": 0.03d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-01-26", "l_commitdate": "1997-01-27", "l_receiptdate": "1997-02-13", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "SHIP", "l_comment": "e slyly blithely silent deposits. fluff" }
+{ "l_orderkey": 4192, "l_partkey": 11, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 36, "l_extendedprice": 32796.36d, "l_discount": 0.06d, "l_tax": 0.08d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-04-25", "l_commitdate": "1998-05-26", "l_receiptdate": "1998-05-03", "l_shipinstruct": "COLLECT COD", "l_shipmode": "TRUCK", "l_comment": "eodolites sleep" }
+{ "l_orderkey": 4194, "l_partkey": 197, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 43, "l_extendedprice": 47179.17d, "l_discount": 0.08d, "l_tax": 0.06d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-11-06", "l_commitdate": "1994-12-09", "l_receiptdate": "1994-11-16", "l_shipinstruct": "NONE", "l_shipmode": "TRUCK", "l_comment": "olites are after the exp" }
+{ "l_orderkey": 4261, "l_partkey": 110, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 12, "l_extendedprice": 12121.32d, "l_discount": 0.05d, "l_tax": 0.01d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1992-11-01", "l_commitdate": "1993-01-01", "l_receiptdate": "1992-11-12", "l_shipinstruct": "NONE", "l_shipmode": "FOB", "l_comment": "into beans " }
+{ "l_orderkey": 4418, "l_partkey": 35, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 32, "l_extendedprice": 29920.96d, "l_discount": 0.02d, "l_tax": 0.06d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1993-05-28", "l_commitdate": "1993-06-02", "l_receiptdate": "1993-05-30", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": "ly. bold pinto b" }
+{ "l_orderkey": 4422, "l_partkey": 135, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 5, "l_extendedprice": 5175.65d, "l_discount": 0.09d, "l_tax": 0.07d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-07-17", "l_commitdate": "1995-08-13", "l_receiptdate": "1995-07-25", "l_shipinstruct": "NONE", "l_shipmode": "SHIP", "l_comment": "e furiously about t" }
+{ "l_orderkey": 4486, "l_partkey": 135, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 46, "l_extendedprice": 47615.98d, "l_discount": 0.08d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-05-02", "l_commitdate": "1998-04-05", "l_receiptdate": "1998-05-08", "l_shipinstruct": "COLLECT COD", "l_shipmode": "MAIL", "l_comment": "ackages. specia" }
+{ "l_orderkey": 4512, "l_partkey": 162, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 30, "l_extendedprice": 31864.8d, "l_discount": 0.07d, "l_tax": 0.07d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-01-28", "l_commitdate": "1995-12-22", "l_receiptdate": "1996-02-22", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "TRUCK", "l_comment": "ly unusual package" }
+{ "l_orderkey": 4513, "l_partkey": 170, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 29, "l_extendedprice": 31034.93d, "l_discount": 0.03d, "l_tax": 0.01d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-05-18", "l_commitdate": "1996-05-23", "l_receiptdate": "1996-06-08", "l_shipinstruct": "NONE", "l_shipmode": "REG AIR", "l_comment": "cajole. regular packages boost. s" }
+{ "l_orderkey": 4545, "l_partkey": 173, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 38, "l_extendedprice": 40780.46d, "l_discount": 0.06d, "l_tax": 0.06d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-01-27", "l_commitdate": "1993-03-01", "l_receiptdate": "1993-02-04", "l_shipinstruct": "NONE", "l_shipmode": "TRUCK", "l_comment": "nts serve according to th" }
+{ "l_orderkey": 4549, "l_partkey": 159, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 44, "l_extendedprice": 46602.6d, "l_discount": 0.08d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-03-13", "l_commitdate": "1998-04-15", "l_receiptdate": "1998-03-27", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "TRUCK", "l_comment": "ding to the regular, silent requests" }
+{ "l_orderkey": 4551, "l_partkey": 11, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 6, "l_extendedprice": 5466.06d, "l_discount": 0.08d, "l_tax": 0.08d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-05-18", "l_commitdate": "1996-04-23", "l_receiptdate": "1996-06-13", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "TRUCK", "l_comment": "fily silent fo" }
+{ "l_orderkey": 4576, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 5, "l_extendedprice": 4950.45d, "l_discount": 0.09d, "l_tax": 0.03d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-08-23", "l_commitdate": "1996-11-08", "l_receiptdate": "1996-09-20", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "AIR", "l_comment": "ly express, special asymptote" }
+{ "l_orderkey": 4608, "l_partkey": 173, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 30, "l_extendedprice": 32195.1d, "l_discount": 0.08d, "l_tax": 0.05d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-10-08", "l_commitdate": "1994-07-18", "l_receiptdate": "1994-10-25", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "SHIP", "l_comment": "s cajole. slyly " }
+{ "l_orderkey": 4641, "l_partkey": 190, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 45, "l_extendedprice": 49058.55d, "l_discount": 0.07d, "l_tax": 0.03d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-05-11", "l_commitdate": "1993-04-19", "l_receiptdate": "1993-05-21", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "MAIL", "l_comment": " about the close " }
+{ "l_orderkey": 4679, "l_partkey": 190, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 7, "l_extendedprice": 7631.33d, "l_discount": 0.1d, "l_tax": 0.05d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-05-11", "l_commitdate": "1993-04-11", "l_receiptdate": "1993-05-16", "l_shipinstruct": "NONE", "l_shipmode": "TRUCK", "l_comment": "kages. bold, regular packa" }
+{ "l_orderkey": 4769, "l_partkey": 35, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 16, "l_extendedprice": 14960.48d, "l_discount": 0.08d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-07-16", "l_commitdate": "1995-07-05", "l_receiptdate": "1995-07-22", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "FOB", "l_comment": " deposits. slyly even asymptote" }
+{ "l_orderkey": 4802, "l_partkey": 40, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 6, "l_extendedprice": 5640.24d, "l_discount": 0.0d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-04-16", "l_commitdate": "1997-03-25", "l_receiptdate": "1997-04-21", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "SHIP", "l_comment": "unusual accounts wake blithely. b" }
+{ "l_orderkey": 4804, "l_partkey": 128, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 44, "l_extendedprice": 45237.28d, "l_discount": 0.06d, "l_tax": 0.08d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1992-05-02", "l_commitdate": "1992-03-24", "l_receiptdate": "1992-05-28", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "AIR", "l_comment": "aggle quickly among the slyly fi" }
+{ "l_orderkey": 4805, "l_partkey": 150, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 7, "l_extendedprice": 7351.05d, "l_discount": 0.09d, "l_tax": 0.03d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1992-05-01", "l_commitdate": "1992-07-09", "l_receiptdate": "1992-05-09", "l_shipinstruct": "NONE", "l_shipmode": "FOB", "l_comment": " requests. regular deposit" }
+{ "l_orderkey": 4807, "l_partkey": 122, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 9, "l_extendedprice": 9199.08d, "l_discount": 0.04d, "l_tax": 0.08d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-04-23", "l_commitdate": "1997-03-01", "l_receiptdate": "1997-05-15", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "TRUCK", "l_comment": "may are blithely. carefully even pinto b" }
+{ "l_orderkey": 4836, "l_partkey": 162, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 22, "l_extendedprice": 23367.52d, "l_discount": 0.01d, "l_tax": 0.03d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-03-03", "l_commitdate": "1997-02-23", "l_receiptdate": "1997-03-04", "l_shipinstruct": "NONE", "l_shipmode": "SHIP", "l_comment": "al pinto beans. care" }
+{ "l_orderkey": 4837, "l_partkey": 42, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 16, "l_extendedprice": 15072.64d, "l_discount": 0.09d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-08-12", "l_commitdate": "1998-06-06", "l_receiptdate": "1998-08-26", "l_shipinstruct": "COLLECT COD", "l_shipmode": "FOB", "l_comment": "ing requests are blithely regular instructi" }
+{ "l_orderkey": 4898, "l_partkey": 72, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 44, "l_extendedprice": 42771.08d, "l_discount": 0.07d, "l_tax": 0.02d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-09-13", "l_commitdate": "1994-08-18", "l_receiptdate": "1994-09-16", "l_shipinstruct": "NONE", "l_shipmode": "FOB", "l_comment": "y regular grouches about" }
+{ "l_orderkey": 4928, "l_partkey": 100, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 4, "l_extendedprice": 4000.4d, "l_discount": 0.04d, "l_tax": 0.02d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-10-25", "l_commitdate": "1993-12-24", "l_receiptdate": "1993-11-16", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "REG AIR", "l_comment": "bout the slyly final accounts. carefull" }
+{ "l_orderkey": 4929, "l_partkey": 14, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 20, "l_extendedprice": 18280.2d, "l_discount": 0.0d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-03-12", "l_commitdate": "1996-05-23", "l_receiptdate": "1996-03-20", "l_shipinstruct": "COLLECT COD", "l_shipmode": "REG AIR", "l_comment": " final pinto beans detect. final," }
+{ "l_orderkey": 4967, "l_partkey": 71, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 50, "l_extendedprice": 48553.5d, "l_discount": 0.07d, "l_tax": 0.01d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-05-27", "l_commitdate": "1997-05-13", "l_receiptdate": "1997-06-12", "l_shipinstruct": "NONE", "l_shipmode": "REG AIR", "l_comment": "kages. final, unusual accounts c" }
+{ "l_orderkey": 4996, "l_partkey": 56, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 35, "l_extendedprice": 33461.75d, "l_discount": 0.07d, "l_tax": 0.01d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1992-10-30", "l_commitdate": "1992-10-27", "l_receiptdate": "1992-11-05", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "SHIP", "l_comment": "s. unusual, regular dolphins integrate care" }
+{ "l_orderkey": 5028, "l_partkey": 14, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 15, "l_extendedprice": 13710.15d, "l_discount": 0.07d, "l_tax": 0.07d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1992-07-17", "l_commitdate": "1992-07-16", "l_receiptdate": "1992-08-05", "l_shipinstruct": "COLLECT COD", "l_shipmode": "REG AIR", "l_comment": "es are quickly final pains. furiously pend" }
+{ "l_orderkey": 5031, "l_partkey": 50, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 15, "l_extendedprice": 14250.75d, "l_discount": 0.02d, "l_tax": 0.05d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1995-04-01", "l_commitdate": "1995-02-24", "l_receiptdate": "1995-04-12", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "AIR", "l_comment": "yly pending theodolites." }
+{ "l_orderkey": 5092, "l_partkey": 164, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 30, "l_extendedprice": 31924.8d, "l_discount": 0.06d, "l_tax": 0.0d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-12-27", "l_commitdate": "1995-12-08", "l_receiptdate": "1996-01-09", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "MAIL", "l_comment": "ss, ironic deposits. furiously stea" }
+{ "l_orderkey": 5153, "l_partkey": 35, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 42, "l_extendedprice": 39271.26d, "l_discount": 0.03d, "l_tax": 0.01d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-10-03", "l_commitdate": "1995-11-09", "l_receiptdate": "1995-10-11", "l_shipinstruct": "COLLECT COD", "l_shipmode": "RAIL", "l_comment": "re thinly. ironic" }
+{ "l_orderkey": 5154, "l_partkey": 190, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 11, "l_extendedprice": 11992.09d, "l_discount": 0.02d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-08-06", "l_commitdate": "1997-06-30", "l_receiptdate": "1997-09-04", "l_shipinstruct": "NONE", "l_shipmode": "RAIL", "l_comment": "luffily bold foxes. final" }
+{ "l_orderkey": 5185, "l_partkey": 197, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 37, "l_extendedprice": 40596.03d, "l_discount": 0.0d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-08-08", "l_commitdate": "1997-09-08", "l_receiptdate": "1997-08-14", "l_shipinstruct": "COLLECT COD", "l_shipmode": "SHIP", "l_comment": "gainst the courts dazzle care" }
+{ "l_orderkey": 5187, "l_partkey": 11, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 49, "l_extendedprice": 44639.49d, "l_discount": 0.04d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-10-20", "l_commitdate": "1997-10-12", "l_receiptdate": "1997-10-26", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "AIR", "l_comment": "l, regular platelets instead of the foxes w" }
+{ "l_orderkey": 5190, "l_partkey": 56, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 43, "l_extendedprice": 41110.15d, "l_discount": 0.09d, "l_tax": 0.06d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1992-08-19", "l_commitdate": "1992-06-10", "l_receiptdate": "1992-09-01", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "FOB", "l_comment": "encies use fluffily unusual requests? hoc" }
+{ "l_orderkey": 5217, "l_partkey": 80, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 50, "l_extendedprice": 49004.0d, "l_discount": 0.05d, "l_tax": 0.02d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-12-26", "l_commitdate": "1995-11-21", "l_receiptdate": "1996-01-24", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "MAIL", "l_comment": "s. express, express accounts c" }
+{ "l_orderkey": 5281, "l_partkey": 114, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 37, "l_extendedprice": 37522.07d, "l_discount": 0.05d, "l_tax": 0.02d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-11-10", "l_commitdate": "1996-01-31", "l_receiptdate": "1995-11-22", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "MAIL", "l_comment": "ronic dependencies. fluffily final p" }
+{ "l_orderkey": 5284, "l_partkey": 173, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 16, "l_extendedprice": 17170.72d, "l_discount": 0.04d, "l_tax": 0.02d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-08-17", "l_commitdate": "1995-08-23", "l_receiptdate": "1995-08-26", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "TRUCK", "l_comment": "unts detect furiously even d" }
+{ "l_orderkey": 5286, "l_partkey": 199, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 1, "l_extendedprice": 1099.19d, "l_discount": 0.01d, "l_tax": 0.07d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-11-25", "l_commitdate": "1997-11-07", "l_receiptdate": "1997-12-17", "l_shipinstruct": "COLLECT COD", "l_shipmode": "REG AIR", "l_comment": "ly! furiously final pack" }
+{ "l_orderkey": 5313, "l_partkey": 17, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 34, "l_extendedprice": 31178.34d, "l_discount": 0.1d, "l_tax": 0.02d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-08-07", "l_commitdate": "1997-08-12", "l_receiptdate": "1997-08-24", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "FOB", "l_comment": "ccording to the blithely final account" }
+{ "l_orderkey": 5315, "l_partkey": 35, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 12, "l_extendedprice": 11220.36d, "l_discount": 0.08d, "l_tax": 0.06d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1992-12-18", "l_commitdate": "1993-01-16", "l_receiptdate": "1993-01-10", "l_shipinstruct": "NONE", "l_shipmode": "AIR", "l_comment": "ccounts. furiously ironi" }
+{ "l_orderkey": 5316, "l_partkey": 108, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 29, "l_extendedprice": 29234.9d, "l_discount": 0.1d, "l_tax": 0.05d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-03-28", "l_commitdate": "1994-04-29", "l_receiptdate": "1994-04-09", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "REG AIR", "l_comment": "ckly unusual foxes bo" }
+{ "l_orderkey": 5379, "l_partkey": 199, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 40, "l_extendedprice": 43967.6d, "l_discount": 0.01d, "l_tax": 0.08d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-10-01", "l_commitdate": "1995-10-19", "l_receiptdate": "1995-10-30", "l_shipinstruct": "COLLECT COD", "l_shipmode": "MAIL", "l_comment": "carefully final accounts haggle blithely. " }
+{ "l_orderkey": 5441, "l_partkey": 164, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 3, "l_extendedprice": 3192.48d, "l_discount": 0.0d, "l_tax": 0.02d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-08-12", "l_commitdate": "1994-10-14", "l_receiptdate": "1994-09-01", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "REG AIR", "l_comment": "are. unusual, " }
+{ "l_orderkey": 5445, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 33, "l_extendedprice": 32672.97d, "l_discount": 0.08d, "l_tax": 0.07d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1993-10-21", "l_commitdate": "1993-10-14", "l_receiptdate": "1993-10-28", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "REG AIR", "l_comment": "ests. final instructions" }
+{ "l_orderkey": 5446, "l_partkey": 190, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 27, "l_extendedprice": 29435.13d, "l_discount": 0.05d, "l_tax": 0.07d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-07-21", "l_commitdate": "1994-08-25", "l_receiptdate": "1994-08-17", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": "ously across the quic" }
+{ "l_orderkey": 5447, "l_partkey": 99, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 31, "l_extendedprice": 30971.79d, "l_discount": 0.09d, "l_tax": 0.03d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-07-14", "l_commitdate": "1996-05-07", "l_receiptdate": "1996-07-17", "l_shipinstruct": "COLLECT COD", "l_shipmode": "SHIP", "l_comment": " foxes sleep. blithely unusual accounts det" }
+{ "l_orderkey": 5476, "l_partkey": 48, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 13, "l_extendedprice": 12324.52d, "l_discount": 0.01d, "l_tax": 0.04d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-12-27", "l_commitdate": "1997-12-08", "l_receiptdate": "1997-12-29", "l_shipinstruct": "COLLECT COD", "l_shipmode": "TRUCK", "l_comment": "iously special ac" }
+{ "l_orderkey": 5506, "l_partkey": 140, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 2, "l_extendedprice": 2080.28d, "l_discount": 0.0d, "l_tax": 0.03d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-02-04", "l_commitdate": "1994-01-13", "l_receiptdate": "1994-02-17", "l_shipinstruct": "COLLECT COD", "l_shipmode": "MAIL", "l_comment": "onic theodolites are fluffil" }
+{ "l_orderkey": 5536, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 14, "l_extendedprice": 13861.26d, "l_discount": 0.08d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-05-18", "l_commitdate": "1998-05-08", "l_receiptdate": "1998-06-05", "l_shipinstruct": "COLLECT COD", "l_shipmode": "MAIL", "l_comment": "instructions sleep " }
+{ "l_orderkey": 5572, "l_partkey": 22, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 24, "l_extendedprice": 22128.48d, "l_discount": 0.08d, "l_tax": 0.08d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-10-30", "l_commitdate": "1994-10-02", "l_receiptdate": "1994-11-27", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "MAIL", "l_comment": "ests cajole. evenly ironic exc" }
+{ "l_orderkey": 5664, "l_partkey": 122, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 25, "l_extendedprice": 25553.0d, "l_discount": 0.0d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-10-29", "l_commitdate": "1998-09-23", "l_receiptdate": "1998-11-25", "l_shipinstruct": "COLLECT COD", "l_shipmode": "FOB", "l_comment": "eposits: furiously ironic grouch" }
+{ "l_orderkey": 5670, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 27, "l_extendedprice": 26732.43d, "l_discount": 0.1d, "l_tax": 0.06d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-05-09", "l_commitdate": "1993-05-30", "l_receiptdate": "1993-06-06", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "REG AIR", "l_comment": " ideas promise bli" }
+{ "l_orderkey": 5728, "l_partkey": 44, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 47, "l_extendedprice": 44369.88d, "l_discount": 0.1d, "l_tax": 0.05d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-12-13", "l_commitdate": "1995-01-25", "l_receiptdate": "1994-12-25", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "MAIL", "l_comment": "nd the bravely final deposits. final ideas" }
+{ "l_orderkey": 5735, "l_partkey": 60, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 41, "l_extendedprice": 39362.46d, "l_discount": 0.01d, "l_tax": 0.01d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-12-23", "l_commitdate": "1995-02-10", "l_receiptdate": "1995-01-22", "l_shipinstruct": "COLLECT COD", "l_shipmode": "MAIL", "l_comment": "lthily ruthless i" }
+{ "l_orderkey": 5826, "l_partkey": 144, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 4, "l_extendedprice": 4176.56d, "l_discount": 0.03d, "l_tax": 0.06d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1998-07-31", "l_commitdate": "1998-09-10", "l_receiptdate": "1998-08-27", "l_shipinstruct": "NONE", "l_shipmode": "AIR", "l_comment": " packages across the fluffily spec" }
+{ "l_orderkey": 5829, "l_partkey": 40, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 4, "l_extendedprice": 3760.16d, "l_discount": 0.01d, "l_tax": 0.02d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-03-01", "l_commitdate": "1997-02-17", "l_receiptdate": "1997-03-22", "l_shipinstruct": "NONE", "l_shipmode": "TRUCK", "l_comment": "ithely; accounts cajole ideas. regular foxe" }
+{ "l_orderkey": 5856, "l_partkey": 4, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 1, "l_extendedprice": 904.0d, "l_discount": 0.03d, "l_tax": 0.02d, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-12-29", "l_commitdate": "1995-01-07", "l_receiptdate": "1995-01-10", "l_shipinstruct": "TAKE BACK RETURN", "l_shipmode": "MAIL", "l_comment": "tly. special deposits wake blithely even" }
+{ "l_orderkey": 5926, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 8, "l_extendedprice": 7920.72d, "l_discount": 0.02d, "l_tax": 0.0d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-07-17", "l_commitdate": "1994-07-20", "l_receiptdate": "1994-08-11", "l_shipinstruct": "COLLECT COD", "l_shipmode": "MAIL", "l_comment": "gle furiously express foxes. bo" }
+{ "l_orderkey": 5927, "l_partkey": 90, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 44, "l_extendedprice": 43563.96d, "l_discount": 0.04d, "l_tax": 0.05d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-11-29", "l_commitdate": "1997-11-21", "l_receiptdate": "1997-12-13", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "TRUCK", "l_comment": "rding to the special, final decoy" }
+{ "l_orderkey": 5955, "l_partkey": 140, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 14, "l_extendedprice": 14561.96d, "l_discount": 0.08d, "l_tax": 0.08d, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1995-06-22", "l_commitdate": "1995-05-23", "l_receiptdate": "1995-06-24", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "TRUCK", "l_comment": " unusual, bold theodolit" }
+{ "l_orderkey": 5959, "l_partkey": 135, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 49, "l_extendedprice": 50721.37d, "l_discount": 0.07d, "l_tax": 0.03d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1992-07-16", "l_commitdate": "1992-08-09", "l_receiptdate": "1992-08-14", "l_shipinstruct": "DELIVER IN PERSON", "l_shipmode": "SHIP", "l_comment": "usual packages haggle slyly pi" }
+{ "l_orderkey": 5988, "l_partkey": 172, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 41, "l_extendedprice": 43958.97d, "l_discount": 0.08d, "l_tax": 0.03d, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-01-20", "l_commitdate": "1994-02-06", "l_receiptdate": "1994-02-10", "l_shipinstruct": "COLLECT COD", "l_shipmode": "AIR", "l_comment": "the pending, express reque" }
diff --git a/asterix-app/src/test/resources/runtimets/testsuite.xml b/asterix-app/src/test/resources/runtimets/testsuite.xml
index 2d07061..55e5588 100644
--- a/asterix-app/src/test/resources/runtimets/testsuite.xml
+++ b/asterix-app/src/test/resources/runtimets/testsuite.xml
@@ -909,6 +909,11 @@
-->
</test-group>
<test-group name="dml">
+ <test-case FilePath="dml">
+ <compilation-unit name="compact-dataset-and-its-indexes">
+ <output-dir compare="Text">compact-dataset-and-its-indexes</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="dml">
<compilation-unit name="query-issue382">
<output-dir compare="Text">query-issue382</output-dir>
diff --git a/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/base/Statement.java b/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/base/Statement.java
index 98f8387..34693e4 100644
--- a/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/base/Statement.java
+++ b/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/base/Statement.java
@@ -40,7 +40,8 @@
BEGIN_FEED,
CONTROL_FEED,
CREATE_FUNCTION,
- FUNCTION_DROP
+ FUNCTION_DROP,
+ COMPACT
}
public abstract Kind getKind();
diff --git a/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/CompactStatement.java b/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/CompactStatement.java
new file mode 100644
index 0000000..a729306
--- /dev/null
+++ b/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/CompactStatement.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.aql.expression;
+
+import edu.uci.ics.asterix.aql.base.Statement;
+import edu.uci.ics.asterix.aql.expression.visitor.IAqlExpressionVisitor;
+import edu.uci.ics.asterix.aql.expression.visitor.IAqlVisitorWithVoidReturn;
+import edu.uci.ics.asterix.common.exceptions.AsterixException;
+
+public class CompactStatement implements Statement {
+
+ private final Identifier dataverseName;
+ private final Identifier datasetName;
+
+ public CompactStatement(Identifier dataverseName, Identifier datasetName) {
+ this.dataverseName = dataverseName;
+ this.datasetName = datasetName;
+ }
+
+ @Override
+ public Kind getKind() {
+ return Kind.COMPACT;
+ }
+
+ public Identifier getDataverseName() {
+ return dataverseName;
+ }
+
+ public Identifier getDatasetName() {
+ return datasetName;
+ }
+
+ @Override
+ public <R, T> R accept(IAqlExpressionVisitor<R, T> visitor, T arg) throws AsterixException {
+ return visitor.visitCompactStatement(this, arg);
+ }
+
+ @Override
+ public <T> void accept(IAqlVisitorWithVoidReturn<T> visitor, T arg) throws AsterixException {
+ visitor.visit(this, arg);
+ }
+}
diff --git a/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/visitor/AQLPrintVisitor.java b/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/visitor/AQLPrintVisitor.java
index cdbd86f..aac2b6b 100644
--- a/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/visitor/AQLPrintVisitor.java
+++ b/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/visitor/AQLPrintVisitor.java
@@ -23,6 +23,7 @@
import edu.uci.ics.asterix.aql.base.Literal;
import edu.uci.ics.asterix.aql.expression.BeginFeedStatement;
import edu.uci.ics.asterix.aql.expression.CallExpr;
+import edu.uci.ics.asterix.aql.expression.CompactStatement;
import edu.uci.ics.asterix.aql.expression.ControlFeedStatement;
import edu.uci.ics.asterix.aql.expression.CreateDataverseStatement;
import edu.uci.ics.asterix.aql.expression.CreateFunctionStatement;
@@ -533,4 +534,10 @@
}
+ @Override
+ public void visit(CompactStatement fds, Integer arg) throws AsterixException {
+ // TODO Auto-generated method stub
+
+ }
+
}
diff --git a/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/visitor/IAqlExpressionVisitor.java b/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/visitor/IAqlExpressionVisitor.java
index 7fbf342..d9a92b4 100644
--- a/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/visitor/IAqlExpressionVisitor.java
+++ b/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/visitor/IAqlExpressionVisitor.java
@@ -16,6 +16,7 @@
import edu.uci.ics.asterix.aql.expression.BeginFeedStatement;
import edu.uci.ics.asterix.aql.expression.CallExpr;
+import edu.uci.ics.asterix.aql.expression.CompactStatement;
import edu.uci.ics.asterix.aql.expression.ControlFeedStatement;
import edu.uci.ics.asterix.aql.expression.CreateDataverseStatement;
import edu.uci.ics.asterix.aql.expression.CreateFunctionStatement;
@@ -162,4 +163,6 @@
R visitFunctionDropStatement(FunctionDropStatement del, T arg) throws AsterixException;
+ R visitCompactStatement(CompactStatement del, T arg) throws AsterixException;
+
}
diff --git a/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/visitor/IAqlVisitorWithVoidReturn.java b/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/visitor/IAqlVisitorWithVoidReturn.java
index b936c63..61a7183 100644
--- a/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/visitor/IAqlVisitorWithVoidReturn.java
+++ b/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/visitor/IAqlVisitorWithVoidReturn.java
@@ -16,6 +16,7 @@
import edu.uci.ics.asterix.aql.expression.BeginFeedStatement;
import edu.uci.ics.asterix.aql.expression.CallExpr;
+import edu.uci.ics.asterix.aql.expression.CompactStatement;
import edu.uci.ics.asterix.aql.expression.ControlFeedStatement;
import edu.uci.ics.asterix.aql.expression.CreateDataverseStatement;
import edu.uci.ics.asterix.aql.expression.CreateFunctionStatement;
@@ -161,4 +162,6 @@
void visit(CreateFunctionStatement cfs, T arg) throws AsterixException;
void visit(FunctionDropStatement fds, T arg) throws AsterixException;
+
+ void visit(CompactStatement fds, T arg) throws AsterixException;
}
diff --git a/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/rewrites/AqlRewriter.java b/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/rewrites/AqlRewriter.java
index a9d8585..203758d 100644
--- a/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/rewrites/AqlRewriter.java
+++ b/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/rewrites/AqlRewriter.java
@@ -26,6 +26,7 @@
import edu.uci.ics.asterix.aql.base.Expression.Kind;
import edu.uci.ics.asterix.aql.expression.BeginFeedStatement;
import edu.uci.ics.asterix.aql.expression.CallExpr;
+import edu.uci.ics.asterix.aql.expression.CompactStatement;
import edu.uci.ics.asterix.aql.expression.ControlFeedStatement;
import edu.uci.ics.asterix.aql.expression.CreateDataverseStatement;
import edu.uci.ics.asterix.aql.expression.CreateFunctionStatement;
@@ -547,5 +548,11 @@
return null;
}
+ @Override
+ public Void visitCompactStatement(CompactStatement del, Void arg) throws AsterixException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
}
}
diff --git a/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/rewrites/CloneAndSubstituteVariablesVisitor.java b/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/rewrites/CloneAndSubstituteVariablesVisitor.java
index dcb4a66..d860b0c 100644
--- a/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/rewrites/CloneAndSubstituteVariablesVisitor.java
+++ b/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/rewrites/CloneAndSubstituteVariablesVisitor.java
@@ -23,6 +23,7 @@
import edu.uci.ics.asterix.aql.base.IAqlExpression;
import edu.uci.ics.asterix.aql.expression.BeginFeedStatement;
import edu.uci.ics.asterix.aql.expression.CallExpr;
+import edu.uci.ics.asterix.aql.expression.CompactStatement;
import edu.uci.ics.asterix.aql.expression.ControlFeedStatement;
import edu.uci.ics.asterix.aql.expression.CreateDataverseStatement;
import edu.uci.ics.asterix.aql.expression.CreateFunctionStatement;
@@ -582,4 +583,11 @@
// TODO Auto-generated method stub
return null;
}
+
+ @Override
+ public Pair<IAqlExpression, List<VariableSubstitution>> visitCompactStatement(CompactStatement del,
+ List<VariableSubstitution> arg) throws AsterixException {
+ // TODO Auto-generated method stub
+ return null;
+ }
}
diff --git a/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/rewrites/InlineUdfsVisitor.java b/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/rewrites/InlineUdfsVisitor.java
index ae5d7d9..fc7acfe 100644
--- a/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/rewrites/InlineUdfsVisitor.java
+++ b/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/rewrites/InlineUdfsVisitor.java
@@ -24,6 +24,7 @@
import edu.uci.ics.asterix.aql.base.IAqlExpression;
import edu.uci.ics.asterix.aql.expression.BeginFeedStatement;
import edu.uci.ics.asterix.aql.expression.CallExpr;
+import edu.uci.ics.asterix.aql.expression.CompactStatement;
import edu.uci.ics.asterix.aql.expression.ControlFeedStatement;
import edu.uci.ics.asterix.aql.expression.CreateDataverseStatement;
import edu.uci.ics.asterix.aql.expression.CreateFunctionStatement;
@@ -522,4 +523,10 @@
// TODO Auto-generated method stub
return null;
}
+
+ @Override
+ public Boolean visitCompactStatement(CompactStatement del, List<FunctionDecl> arg) throws AsterixException {
+ // TODO Auto-generated method stub
+ return null;
+ }
}
diff --git a/asterix-aql/src/main/javacc/AQL.jj b/asterix-aql/src/main/javacc/AQL.jj
index cb6336b..8ef6db6 100644
--- a/asterix-aql/src/main/javacc/AQL.jj
+++ b/asterix-aql/src/main/javacc/AQL.jj
@@ -193,6 +193,7 @@
| stmt = DeleteStatement()
| stmt = UpdateStatement()
| stmt = FeedStatement()
+ | stmt = CompactStatement()
| stmt = Query()
)
{
@@ -296,11 +297,13 @@
String typeName = null;
String adapterName = null;
Map<String,String> properties = null;
+ Map<String,String> compactionProperties = null;
FunctionSignature appliedFunction = null;
List<String> primaryKeyFields = null;
String nodeGroupName = null;
Map<String,String> hints = new HashMap<String,String>();
DatasetDecl dsetDecl = null;
+ String compactionPolicy = null;
}
{
(
@@ -309,6 +312,7 @@
ifNotExists = IfNotExists()
"using" adapterName = AdapterName() properties = Configuration()
( "hints" hints = Properties() )?
+ ( "using" "compaction" "policy" compactionPolicy = CompactionPolicy() compactionProperties = Configuration() )?
{
ExternalDetailsDecl edd = new ExternalDetailsDecl();
edd.setAdapter(adapterName);
@@ -329,6 +333,7 @@
(appliedFunction = ApplyFunction())? primaryKeyFields = PrimaryKey()
( "on" nodeGroupName = Identifier() )?
( "hints" hints = Properties() )?
+ ( "using" "compaction" "policy" compactionPolicy = CompactionPolicy() compactionProperties = Configuration() )?
{
FeedDetailsDecl fdd = new FeedDetailsDecl(adapterName,
properties,
@@ -350,6 +355,7 @@
ifNotExists = IfNotExists()
primaryKeyFields = PrimaryKey() ("on" nodeGroupName = Identifier() )?
( "hints" hints = Properties() )?
+ ( "using" "compaction" "policy" compactionPolicy = CompactionPolicy() compactionProperties = Configuration() )?
{
InternalDetailsDecl idd = new InternalDetailsDecl(nodeGroupName != null
? new Identifier(nodeGroupName)
@@ -376,7 +382,9 @@
String fieldExpr = null;
boolean ifNotExists = false;
Pair<Identifier,Identifier> nameComponents = null;
+ Map<String,String> compactionProperties = null;
IndexParams indexType = null;
+ String compactionPolicy = null;
}
{
"index" indexName = Identifier()
@@ -391,6 +399,7 @@
cis.addFieldExpr(fieldExpr);
}
)* <RIGHTPAREN> ( "type" indexType = IndexType() )?
+ ( "using" "compaction" "policy" compactionPolicy = CompactionPolicy() compactionProperties = Configuration() )?
{
cis.setIndexName(new Identifier(indexName));
cis.setIfNotExists(ifNotExists);
@@ -404,6 +413,17 @@
}
}
+String CompactionPolicy() throws ParseException :
+{
+ String compactionPolicy = null;
+}
+{
+ compactionPolicy = Identifier()
+ {
+ return compactionPolicy;
+ }
+}
+
IndexParams IndexType() throws ParseException:
{
IndexType type = null;
@@ -772,6 +792,21 @@
}
}
+Statement CompactStatement() throws ParseException:
+{
+ Pair<Identifier,Identifier> nameComponents = null;
+ Statement stmt = null;
+}
+{
+ "compact" <DATASET> nameComponents = QualifiedName()
+ {
+ stmt = new CompactStatement(nameComponents.first, nameComponents.second);
+ }
+ {
+ return stmt;
+ }
+}
+
Statement FeedStatement() throws ParseException:
{
Pair<Identifier,Identifier> nameComponents = null;
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/context/ConstantMergePolicy.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/context/ConstantMergePolicy.java
index 4e1239f..44f64b8 100644
--- a/asterix-common/src/main/java/edu/uci/ics/asterix/common/context/ConstantMergePolicy.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/context/ConstantMergePolicy.java
@@ -15,7 +15,6 @@
package edu.uci.ics.asterix.common.context;
-import java.util.ArrayList;
import java.util.List;
import edu.uci.ics.asterix.common.api.IAsterixAppRuntimeContext;
@@ -23,6 +22,7 @@
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent.ComponentState;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndex;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
@@ -39,12 +39,24 @@
}
@Override
- public void diskComponentAdded(final ILSMIndex index) throws HyracksDataException, IndexException {
- List<ILSMComponent> immutableComponents = new ArrayList<ILSMComponent>(index.getImmutableComponents());
- if (!ctx.isShuttingdown() && immutableComponents.size() >= threshold) {
- ILSMIndexAccessor accessor = (ILSMIndexAccessor) index.createAccessor(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
- accessor.scheduleMerge(NoOpIOOperationCallback.INSTANCE, immutableComponents);
+ public void diskComponentAdded(final ILSMIndex index, boolean fullMergeIsRequested) throws HyracksDataException,
+ IndexException {
+ if (!ctx.isShuttingdown()) {
+ List<ILSMComponent> immutableComponents = index.getImmutableComponents();
+ for (ILSMComponent c : immutableComponents) {
+ if (c.getState() != ComponentState.READABLE_UNWRITABLE) {
+ return;
+ }
+ }
+ if (fullMergeIsRequested) {
+ ILSMIndexAccessor accessor = (ILSMIndexAccessor) index.createAccessor(NoOpOperationCallback.INSTANCE,
+ NoOpOperationCallback.INSTANCE);
+ accessor.scheduleFullMerge(NoOpIOOperationCallback.INSTANCE);
+ } else if (immutableComponents.size() >= threshold) {
+ ILSMIndexAccessor accessor = (ILSMIndexAccessor) index.createAccessor(NoOpOperationCallback.INSTANCE,
+ NoOpOperationCallback.INSTANCE);
+ accessor.scheduleMerge(NoOpIOOperationCallback.INSTANCE, immutableComponents);
+ }
}
}
}
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/context/PrefixMergePolicy.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/context/PrefixMergePolicy.java
index 059572b..5bd774b 100644
--- a/asterix-common/src/main/java/edu/uci/ics/asterix/common/context/PrefixMergePolicy.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/context/PrefixMergePolicy.java
@@ -23,6 +23,7 @@
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent.ComponentState;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndex;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
@@ -43,13 +44,25 @@
}
@Override
- public void diskComponentAdded(final ILSMIndex index) throws HyracksDataException, IndexException {
+ public void diskComponentAdded(final ILSMIndex index, boolean fullMergeIsRequested) throws HyracksDataException,
+ IndexException {
if (!ctx.isShuttingdown()) {
// 1. Look at the candidate components for merging in oldest-first order. If one exists, identify the prefix of the sequence of
// all such components for which the sum of their sizes exceeds MaxMrgCompSz. Schedule a merge of those components into a new component.
// 2. If a merge from 1 doesn't happen, see if the set of candidate components for merging exceeds MaxTolCompCnt. If so, schedule
// a merge all of the current candidates into a new single component.
- List<ILSMComponent> immutableComponents = new ArrayList<ILSMComponent>(index.getImmutableComponents());
+ List<ILSMComponent> immutableComponents = index.getImmutableComponents();
+ for (ILSMComponent c : immutableComponents) {
+ if (c.getState() != ComponentState.READABLE_UNWRITABLE) {
+ return;
+ }
+ }
+ if (fullMergeIsRequested) {
+ ILSMIndexAccessor accessor = (ILSMIndexAccessor) index.createAccessor(NoOpOperationCallback.INSTANCE,
+ NoOpOperationCallback.INSTANCE);
+ accessor.scheduleFullMerge(NoOpIOOperationCallback.INSTANCE);
+ return;
+ }
long totalSize = 0;
int startIndex = -1;
for (int i = 0; i < immutableComponents.size(); i++) {